@gscdump/engine 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/_chunks/coerce.mjs +1 -1
- package/dist/_chunks/engine.mjs +1 -1
- package/dist/_chunks/entities.mjs +3 -91
- package/dist/_chunks/index.d.mts +44 -95
- package/dist/_chunks/resolver.mjs +105 -105
- package/dist/_chunks/schema2.mjs +2 -1
- package/dist/_chunks/sink.d.mts +2 -1
- package/dist/_chunks/snapshot.d.mts +27 -1
- package/dist/adapters/node.d.mts +6 -2
- package/dist/adapters/node.mjs +93 -2
- package/dist/entities.d.mts +3 -36
- package/dist/entities.mjs +4 -2
- package/dist/entity-keys.d.mts +27 -0
- package/dist/entity-keys.mjs +93 -0
- package/dist/iceberg/index.mjs +7 -7
- package/dist/index.d.mts +6 -107
- package/dist/index.mjs +101 -132
- package/dist/ingest-accumulator.d.mts +97 -0
- package/dist/ingest-accumulator.mjs +123 -0
- package/dist/period/index.d.mts +1 -1
- package/dist/planner.d.mts +14 -2
- package/dist/report/index.d.mts +1 -1
- package/dist/resolver/index.d.mts +53 -1
- package/dist/rollups.mjs +3 -2
- package/dist/source/index.d.mts +103 -2
- package/dist/source/index.mjs +142 -1
- package/package.json +14 -4
- package/dist/_chunks/duckdb.d.mts +0 -27
- package/dist/_chunks/index2.d.mts +0 -53
- package/dist/_chunks/pg-adapter.d.mts +0 -56
- package/dist/_chunks/planner.d.mts +0 -15
- package/dist/_chunks/source.mjs +0 -143
- package/dist/adapters/duckdb-node.d.mts +0 -7
- package/dist/adapters/duckdb-node.mjs +0 -94
- package/dist/schedule.mjs +0 -100
- /package/dist/{schedule.d.mts → _chunks/schedule.d.mts} +0 -0
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
import { arrowToRows } from "../arrow-utils.mjs";
|
|
2
|
-
import { createRequire } from "node:module";
|
|
3
|
-
import process from "node:process";
|
|
4
|
-
import { unlinkSync } from "node:fs";
|
|
5
|
-
import { tmpdir } from "node:os";
|
|
6
|
-
import { join } from "node:path";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
|
|
9
|
-
const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
|
|
10
|
-
let singleton = null;
|
|
11
|
-
let singletonOpts = null;
|
|
12
|
-
function bundles() {
|
|
13
|
-
return {
|
|
14
|
-
mvp: {
|
|
15
|
-
mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
|
|
16
|
-
mainWorker: null
|
|
17
|
-
},
|
|
18
|
-
eh: {
|
|
19
|
-
mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
|
|
20
|
-
mainWorker: null
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
async function initialize(opts) {
|
|
25
|
-
const logger = opts.verbose ? new ConsoleLogger() : new VoidLogger();
|
|
26
|
-
const db = await createDuckDB(bundles(), logger, NODE_RUNTIME);
|
|
27
|
-
await db.instantiate();
|
|
28
|
-
return {
|
|
29
|
-
db,
|
|
30
|
-
conn: db.connect()
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
function getSingleton(opts) {
|
|
34
|
-
if (!singleton) {
|
|
35
|
-
singleton = initialize(opts);
|
|
36
|
-
singletonOpts = opts;
|
|
37
|
-
}
|
|
38
|
-
return singleton;
|
|
39
|
-
}
|
|
40
|
-
function createNodeDuckDBHandle(opts = {}) {
|
|
41
|
-
if (singleton && opts.verbose !== void 0 && opts.verbose !== (singletonOpts?.verbose ?? false)) console.warn(`[gscdump] createNodeDuckDBHandle: ignoring verbose=${opts.verbose} — a shared DuckDB instance was already initialized with verbose=${singletonOpts?.verbose ?? false}. Call resetNodeDuckDB() before re-initializing to change it.`);
|
|
42
|
-
getSingleton(opts);
|
|
43
|
-
return {
|
|
44
|
-
async query(sql, params) {
|
|
45
|
-
const { conn } = await getSingleton(opts);
|
|
46
|
-
if (!params || params.length === 0) return arrowToRows(conn.query(sql));
|
|
47
|
-
const stmt = conn.prepare(sql);
|
|
48
|
-
try {
|
|
49
|
-
return arrowToRows(stmt.query(...params));
|
|
50
|
-
} finally {
|
|
51
|
-
stmt.close();
|
|
52
|
-
}
|
|
53
|
-
},
|
|
54
|
-
async registerFileBuffer(name, bytes) {
|
|
55
|
-
const { db } = await getSingleton(opts);
|
|
56
|
-
db.registerFileBuffer(name, bytes);
|
|
57
|
-
},
|
|
58
|
-
async copyFileToBuffer(name) {
|
|
59
|
-
const { db } = await getSingleton(opts);
|
|
60
|
-
return db.copyFileToBuffer(name);
|
|
61
|
-
},
|
|
62
|
-
async dropFiles(names) {
|
|
63
|
-
const { db } = await getSingleton(opts);
|
|
64
|
-
for (const name of names) {
|
|
65
|
-
try {
|
|
66
|
-
db.dropFile(name);
|
|
67
|
-
} catch (error) {
|
|
68
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
69
|
-
if (!/not found|does not exist|unknown file/i.test(message)) throw error;
|
|
70
|
-
}
|
|
71
|
-
try {
|
|
72
|
-
unlinkSync(name);
|
|
73
|
-
} catch (error) {
|
|
74
|
-
if (error.code !== "ENOENT") throw error;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
},
|
|
78
|
-
makeTempPath(ext) {
|
|
79
|
-
return join(tmpdir(), `gscdump-${Math.random().toString(36).slice(2, 10)}.${ext}`);
|
|
80
|
-
}
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
function resetNodeDuckDB() {
|
|
84
|
-
const pending = singleton;
|
|
85
|
-
singleton = null;
|
|
86
|
-
singletonOpts = null;
|
|
87
|
-
pending?.then(({ db, conn }) => {
|
|
88
|
-
conn.close();
|
|
89
|
-
db.reset();
|
|
90
|
-
}).catch((err) => {
|
|
91
|
-
console.warn("[gscdump] resetNodeDuckDB: failed to release DuckDB instance", err);
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
export { createNodeDuckDBHandle, resetNodeDuckDB };
|
package/dist/schedule.mjs
DELETED
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
const DAY = 24 * (3600 * 1e3);
|
|
2
|
-
function isDue(state, now) {
|
|
3
|
-
return now >= state.nextAt;
|
|
4
|
-
}
|
|
5
|
-
function sitemapCadenceMs(consecutiveUnchanged) {
|
|
6
|
-
if (consecutiveUnchanged >= 7) return 30 * DAY;
|
|
7
|
-
if (consecutiveUnchanged >= 3) return 7 * DAY;
|
|
8
|
-
return DAY;
|
|
9
|
-
}
|
|
10
|
-
const SITEMAP_VERSION = 1;
|
|
11
|
-
const sitemapPolicy = {
|
|
12
|
-
version: SITEMAP_VERSION,
|
|
13
|
-
initial(now) {
|
|
14
|
-
return {
|
|
15
|
-
nextAt: now + DAY,
|
|
16
|
-
consecutiveUnchanged: 0,
|
|
17
|
-
policyVersion: SITEMAP_VERSION
|
|
18
|
-
};
|
|
19
|
-
},
|
|
20
|
-
observe(prev, evt) {
|
|
21
|
-
if (prev.policyVersion !== SITEMAP_VERSION) return {
|
|
22
|
-
nextAt: evt.at + sitemapCadenceMs(0),
|
|
23
|
-
consecutiveUnchanged: 0,
|
|
24
|
-
policyVersion: SITEMAP_VERSION
|
|
25
|
-
};
|
|
26
|
-
if (evt.changed) return {
|
|
27
|
-
nextAt: evt.at + DAY,
|
|
28
|
-
consecutiveUnchanged: 0,
|
|
29
|
-
policyVersion: SITEMAP_VERSION
|
|
30
|
-
};
|
|
31
|
-
const next = prev.consecutiveUnchanged + 1;
|
|
32
|
-
return {
|
|
33
|
-
nextAt: evt.at + sitemapCadenceMs(next),
|
|
34
|
-
consecutiveUnchanged: next,
|
|
35
|
-
policyVersion: SITEMAP_VERSION
|
|
36
|
-
};
|
|
37
|
-
},
|
|
38
|
-
isDue
|
|
39
|
-
};
|
|
40
|
-
const INSPECTION_VERSION = 1;
|
|
41
|
-
function inspectionCadenceMs(verdict) {
|
|
42
|
-
if (verdict === "PASS") return 30 * DAY;
|
|
43
|
-
if (verdict === "FAIL") return 7 * DAY;
|
|
44
|
-
return 14 * DAY;
|
|
45
|
-
}
|
|
46
|
-
function inspectionPolicy(verdict) {
|
|
47
|
-
const cadence = inspectionCadenceMs(verdict);
|
|
48
|
-
return {
|
|
49
|
-
version: INSPECTION_VERSION,
|
|
50
|
-
initial(now) {
|
|
51
|
-
return {
|
|
52
|
-
nextAt: now + cadence,
|
|
53
|
-
consecutiveUnchanged: 0,
|
|
54
|
-
policyVersion: INSPECTION_VERSION
|
|
55
|
-
};
|
|
56
|
-
},
|
|
57
|
-
observe(prev, evt) {
|
|
58
|
-
if (prev.policyVersion !== INSPECTION_VERSION) return {
|
|
59
|
-
nextAt: evt.at + cadence,
|
|
60
|
-
consecutiveUnchanged: 0,
|
|
61
|
-
policyVersion: INSPECTION_VERSION
|
|
62
|
-
};
|
|
63
|
-
const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
|
|
64
|
-
return {
|
|
65
|
-
nextAt: evt.at + cadence,
|
|
66
|
-
consecutiveUnchanged: next,
|
|
67
|
-
policyVersion: INSPECTION_VERSION
|
|
68
|
-
};
|
|
69
|
-
},
|
|
70
|
-
isDue
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
const FIXED_VERSION = 1;
|
|
74
|
-
function fixedPolicy(intervalMs) {
|
|
75
|
-
return {
|
|
76
|
-
version: FIXED_VERSION,
|
|
77
|
-
initial(now) {
|
|
78
|
-
return {
|
|
79
|
-
nextAt: now + intervalMs,
|
|
80
|
-
consecutiveUnchanged: 0,
|
|
81
|
-
policyVersion: FIXED_VERSION
|
|
82
|
-
};
|
|
83
|
-
},
|
|
84
|
-
observe(prev, evt) {
|
|
85
|
-
if (prev.policyVersion !== FIXED_VERSION) return {
|
|
86
|
-
nextAt: evt.at + intervalMs,
|
|
87
|
-
consecutiveUnchanged: 0,
|
|
88
|
-
policyVersion: FIXED_VERSION
|
|
89
|
-
};
|
|
90
|
-
const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
|
|
91
|
-
return {
|
|
92
|
-
nextAt: evt.at + intervalMs,
|
|
93
|
-
consecutiveUnchanged: next,
|
|
94
|
-
policyVersion: FIXED_VERSION
|
|
95
|
-
};
|
|
96
|
-
},
|
|
97
|
-
isDue
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
export { fixedPolicy, inspectionPolicy, sitemapPolicy };
|
|
File without changes
|