@gscdump/cloudflare 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server-tail/duckdb-iceberg-executor.d.mts +2 -3
- package/dist/server-tail/duckdb-iceberg-executor.mjs +1 -1
- package/dist/server-tail/index.d.mts +2 -2
- package/dist/server-tail/index.mjs +2 -2
- package/dist/server-tail/r2-sql-client.d.mts +1 -7
- package/package.json +5 -14
- package/dist/arrow.mjs +0 -51
- package/dist/engine.d.mts +0 -17
- package/dist/engine.mjs +0 -39
- package/dist/env.d.mts +0 -45
- package/dist/index.d.mts +0 -5
- package/dist/index.mjs +0 -4
- package/dist/workers-duckdb.d.mts +0 -19
- package/dist/workers-duckdb.mjs +0 -362
|
@@ -4,9 +4,8 @@ import { Result } from "gscdump/result";
|
|
|
4
4
|
/** Row returned by the DuckDB sibling. */
|
|
5
5
|
type DuckDbIcebergRow = Record<string, string | number | null>;
|
|
6
6
|
/**
|
|
7
|
-
* The minimal `DUCKDB_SVC` shape this executor needs
|
|
8
|
-
*
|
|
9
|
-
* satisfies it.
|
|
7
|
+
* The minimal `DUCKDB_SVC` shape this executor needs. Any binding with
|
|
8
|
+
* `runSQL` satisfies it.
|
|
10
9
|
*/
|
|
11
10
|
interface DuckDbSvc {
|
|
12
11
|
runSQL: (args: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { TABLE_PLACEHOLDER, buildArchetypeSql } from "./archetype-sql.mjs";
|
|
2
|
-
import { bindLiterals } from "@gscdump/engine/sql";
|
|
3
2
|
import { err, ok, unwrapResult } from "gscdump/result";
|
|
3
|
+
import { bindLiterals } from "@gscdump/engine/sql";
|
|
4
4
|
var DuckDbIcebergError = class extends Error {
|
|
5
5
|
name = "DuckDbIcebergError";
|
|
6
6
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ArchetypeSqlPlan, BuildArchetypeSqlOptions, TABLE_PLACEHOLDER, buildArchetypeSql } from "./archetype-sql.mjs";
|
|
2
2
|
import { DuckDbIcebergError, DuckDbIcebergExecutor, DuckDbIcebergExecutorConfig, DuckDbIcebergQueryError, DuckDbIcebergResult, DuckDbIcebergRow, DuckDbIcebergTimeoutError, DuckDbSvc, createDuckDbIcebergExecutor } from "./duckdb-iceberg-executor.mjs";
|
|
3
|
-
import { R2SqlClient, R2SqlClientConfig, R2SqlError, R2SqlQueryError, R2SqlResult, R2SqlRow, R2SqlTimeoutError, createR2SqlClient,
|
|
3
|
+
import { R2SqlClient, R2SqlClientConfig, R2SqlError, R2SqlQueryError, R2SqlResult, R2SqlRow, R2SqlTimeoutError, createR2SqlClient, inlineParams } from "./r2-sql-client.mjs";
|
|
4
4
|
import { ServerTailDispatcher, ServerTailDispatcherConfig, ServerTailEngine, ServerTailRoutingError, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult } from "./dispatcher.mjs";
|
|
5
5
|
import { R2SqlTransport, R2SqlTransportConfig, R2SqlTransportMetrics, R2SqlTransportResult, R2SqlTransportRow, createR2SqlTransport } from "./r2-sql-transport.mjs";
|
|
6
|
-
export { type ArchetypeSqlPlan, type BuildArchetypeSqlOptions, DuckDbIcebergError, type DuckDbIcebergExecutor, type DuckDbIcebergExecutorConfig, type DuckDbIcebergQueryError, type DuckDbIcebergResult, type DuckDbIcebergRow, DuckDbIcebergTimeoutError, type DuckDbSvc, type R2SqlClient, type R2SqlClientConfig, R2SqlError, type R2SqlQueryError, type R2SqlResult, type R2SqlRow, R2SqlTimeoutError, type R2SqlTransport, type R2SqlTransportConfig, type R2SqlTransportMetrics, type R2SqlTransportResult, type R2SqlTransportRow, type ServerTailDispatcher, type ServerTailDispatcherConfig, type ServerTailEngine, ServerTailRoutingError, TABLE_PLACEHOLDER, buildArchetypeSql, createDuckDbIcebergExecutor, createR2SqlClient, createR2SqlTransport, createServerTailDispatcher,
|
|
6
|
+
export { type ArchetypeSqlPlan, type BuildArchetypeSqlOptions, DuckDbIcebergError, type DuckDbIcebergExecutor, type DuckDbIcebergExecutorConfig, type DuckDbIcebergQueryError, type DuckDbIcebergResult, type DuckDbIcebergRow, DuckDbIcebergTimeoutError, type DuckDbSvc, type R2SqlClient, type R2SqlClientConfig, R2SqlError, type R2SqlQueryError, type R2SqlResult, type R2SqlRow, R2SqlTimeoutError, type R2SqlTransport, type R2SqlTransportConfig, type R2SqlTransportMetrics, type R2SqlTransportResult, type R2SqlTransportRow, type ServerTailDispatcher, type ServerTailDispatcherConfig, type ServerTailEngine, ServerTailRoutingError, TABLE_PLACEHOLDER, buildArchetypeSql, createDuckDbIcebergExecutor, createR2SqlClient, createR2SqlTransport, createServerTailDispatcher, inlineParams, resolveServerTailEngine, resolveServerTailEngineResult };
|
|
@@ -2,5 +2,5 @@ import { TABLE_PLACEHOLDER, buildArchetypeSql } from "./archetype-sql.mjs";
|
|
|
2
2
|
import { ServerTailRoutingError, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult } from "./dispatcher.mjs";
|
|
3
3
|
import { DuckDbIcebergError, DuckDbIcebergTimeoutError, createDuckDbIcebergExecutor } from "./duckdb-iceberg-executor.mjs";
|
|
4
4
|
import { createR2SqlTransport } from "./r2-sql-transport.mjs";
|
|
5
|
-
import { R2SqlError, R2SqlTimeoutError, createR2SqlClient,
|
|
6
|
-
export { DuckDbIcebergError, DuckDbIcebergTimeoutError, R2SqlError, R2SqlTimeoutError, ServerTailRoutingError, TABLE_PLACEHOLDER, buildArchetypeSql, createDuckDbIcebergExecutor, createR2SqlClient, createR2SqlTransport, createServerTailDispatcher,
|
|
5
|
+
import { R2SqlError, R2SqlTimeoutError, createR2SqlClient, inlineParams } from "./r2-sql-client.mjs";
|
|
6
|
+
export { DuckDbIcebergError, DuckDbIcebergTimeoutError, R2SqlError, R2SqlTimeoutError, ServerTailRoutingError, TABLE_PLACEHOLDER, buildArchetypeSql, createDuckDbIcebergExecutor, createR2SqlClient, createR2SqlTransport, createServerTailDispatcher, inlineParams, resolveServerTailEngine, resolveServerTailEngineResult };
|
|
@@ -63,12 +63,6 @@ declare class R2SqlTimeoutError extends Error {
|
|
|
63
63
|
* modelled here; they keep throwing `R2SqlError` synchronously.
|
|
64
64
|
*/
|
|
65
65
|
type R2SqlQueryError = R2SqlError | R2SqlTimeoutError;
|
|
66
|
-
/**
|
|
67
|
-
* Escape a JS value for inline embedding in R2 SQL. R2 SQL has no bound-param
|
|
68
|
-
* channel, so `buildArchetypeSql`'s `?` placeholders are substituted here.
|
|
69
|
-
* Numbers go in bare; strings are single-quote-escaped; null → `NULL`.
|
|
70
|
-
*/
|
|
71
|
-
declare function escapeSqlValue(value: unknown): string;
|
|
72
66
|
/**
|
|
73
67
|
* Inline a plan's `?`-bound params into its SQL, in order. R2 SQL accepts only
|
|
74
68
|
* a literal query string. Quote-aware so a `?` inside a string literal is not
|
|
@@ -98,4 +92,4 @@ interface R2SqlClient {
|
|
|
98
92
|
* production; tests inject `fetchImpl` returning a recorded envelope.
|
|
99
93
|
*/
|
|
100
94
|
declare function createR2SqlClient(config: R2SqlClientConfig): R2SqlClient;
|
|
101
|
-
export { R2SqlClient, R2SqlClientConfig, R2SqlError, R2SqlQueryError, R2SqlResult, R2SqlRow, R2SqlTimeoutError, createR2SqlClient,
|
|
95
|
+
export { R2SqlClient, R2SqlClientConfig, R2SqlError, R2SqlQueryError, R2SqlResult, R2SqlRow, R2SqlTimeoutError, createR2SqlClient, inlineParams };
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/cloudflare",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "3.
|
|
5
|
-
"description": "Cloudflare
|
|
4
|
+
"version": "3.1.0",
|
|
5
|
+
"description": "Cloudflare Workers helpers for server-tail queries and in-flight request deduplication.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
@@ -21,11 +21,6 @@
|
|
|
21
21
|
},
|
|
22
22
|
"sideEffects": false,
|
|
23
23
|
"exports": {
|
|
24
|
-
".": {
|
|
25
|
-
"types": "./dist/index.d.mts",
|
|
26
|
-
"import": "./dist/index.mjs",
|
|
27
|
-
"default": "./dist/index.mjs"
|
|
28
|
-
},
|
|
29
24
|
"./server-tail": {
|
|
30
25
|
"types": "./dist/server-tail/index.d.mts",
|
|
31
26
|
"import": "./dist/server-tail/index.mjs",
|
|
@@ -37,8 +32,6 @@
|
|
|
37
32
|
"default": "./dist/inflight-dedupe.mjs"
|
|
38
33
|
}
|
|
39
34
|
},
|
|
40
|
-
"main": "./dist/index.mjs",
|
|
41
|
-
"types": "./dist/index.d.mts",
|
|
42
35
|
"files": [
|
|
43
36
|
"dist"
|
|
44
37
|
],
|
|
@@ -46,11 +39,9 @@
|
|
|
46
39
|
"node": ">=22"
|
|
47
40
|
},
|
|
48
41
|
"dependencies": {
|
|
49
|
-
"@
|
|
50
|
-
"
|
|
51
|
-
"@gscdump/
|
|
52
|
-
"@gscdump/engine-sqlite": "^3.0.0",
|
|
53
|
-
"gscdump": "^3.0.0"
|
|
42
|
+
"@gscdump/engine": "^3.1.0",
|
|
43
|
+
"gscdump": "^3.1.0",
|
|
44
|
+
"@gscdump/contracts": "^3.1.0"
|
|
54
45
|
},
|
|
55
46
|
"devDependencies": {
|
|
56
47
|
"@cloudflare/vitest-pool-workers": "^0.21.3",
|
package/dist/arrow.mjs
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { float64, int32, int64, tableFromArrays, tableToIPC, utf8 } from "@uwdata/flechette";
|
|
2
|
-
function arrowTypeForColumn(type) {
|
|
3
|
-
switch (type) {
|
|
4
|
-
case "VARCHAR":
|
|
5
|
-
case "DATE": return utf8();
|
|
6
|
-
case "BIGINT": return int64();
|
|
7
|
-
case "INTEGER": return int32();
|
|
8
|
-
case "DOUBLE": return float64();
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
function rowsToArrowIPC(rows, schemaColumns) {
|
|
12
|
-
const colNames = [];
|
|
13
|
-
const seen = /* @__PURE__ */ new Set();
|
|
14
|
-
const add = (name) => {
|
|
15
|
-
if (!seen.has(name)) {
|
|
16
|
-
seen.add(name);
|
|
17
|
-
colNames.push(name);
|
|
18
|
-
}
|
|
19
|
-
};
|
|
20
|
-
if (schemaColumns) for (const c of schemaColumns) add(c.name);
|
|
21
|
-
for (const row of rows) for (const key in row) add(key);
|
|
22
|
-
const schemaType = /* @__PURE__ */ new Map();
|
|
23
|
-
if (schemaColumns) for (const c of schemaColumns) schemaType.set(c.name, c.type);
|
|
24
|
-
const data = {};
|
|
25
|
-
const types = {};
|
|
26
|
-
for (const name of colNames) {
|
|
27
|
-
const values = rows.map((r) => r[name] ?? null);
|
|
28
|
-
data[name] = values;
|
|
29
|
-
const known = schemaType.get(name);
|
|
30
|
-
if (known) {
|
|
31
|
-
types[name] = arrowTypeForColumn(known);
|
|
32
|
-
continue;
|
|
33
|
-
}
|
|
34
|
-
let hasString = false;
|
|
35
|
-
let hasValue = false;
|
|
36
|
-
for (const v of values) {
|
|
37
|
-
if (v === null || v === void 0) continue;
|
|
38
|
-
hasValue = true;
|
|
39
|
-
if (typeof v === "string") {
|
|
40
|
-
hasString = true;
|
|
41
|
-
break;
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (hasString || !hasValue) types[name] = utf8();
|
|
45
|
-
}
|
|
46
|
-
const table = tableFromArrays(data, { types });
|
|
47
|
-
const ipc = tableToIPC(table, { format: "stream" });
|
|
48
|
-
if (!ipc) throw new Error("rowsToArrowIPC: tableToIPC returned null");
|
|
49
|
-
return ipc;
|
|
50
|
-
}
|
|
51
|
-
export { rowsToArrowIPC };
|
package/dist/engine.d.mts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { AnalyticsEnv } from "./env.mjs";
|
|
2
|
-
import { createStorageEngine } from "@gscdump/engine";
|
|
3
|
-
/**
|
|
4
|
-
* Optional per-request hooks for telemetry / tracing. Hosts wire these in
|
|
5
|
-
* to bridge engine activity into their own metrics pipeline.
|
|
6
|
-
*/
|
|
7
|
-
interface AnalyticsEngineHooks {
|
|
8
|
-
/** Called once per R2 PUT, with the byte size of the payload. */
|
|
9
|
-
onR2Write?: (byteLength: number) => void;
|
|
10
|
-
/** Override the interactive 22-second DuckDB RPC budget for background work. */
|
|
11
|
-
duckdbRpcTimeoutMs?: number;
|
|
12
|
-
}
|
|
13
|
-
interface AnalyticsEngineRuntime {
|
|
14
|
-
getEngine: (env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks) => ReturnType<typeof createStorageEngine> | null;
|
|
15
|
-
}
|
|
16
|
-
declare function getAnalyticsEngine(env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks): ReturnType<typeof createStorageEngine> | null;
|
|
17
|
-
export { AnalyticsEngineHooks, AnalyticsEngineRuntime, getAnalyticsEngine };
|
package/dist/engine.mjs
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
|
|
2
|
-
import { createStorageEngine } from "@gscdump/engine";
|
|
3
|
-
import { createD1ManifestStore } from "@gscdump/engine-sqlite";
|
|
4
|
-
import { createR2DataSource } from "@gscdump/engine/r2";
|
|
5
|
-
function createAnalyticsEngineRuntime() {
|
|
6
|
-
const rowCache = createDucklingsRowCache();
|
|
7
|
-
return { getEngine(env, db, hooks = {}) {
|
|
8
|
-
if (!env.R2_DATA) return null;
|
|
9
|
-
const baseDataSource = createR2DataSource({
|
|
10
|
-
bucket: env.R2_DATA,
|
|
11
|
-
bucketName: env.R2_BUCKET_NAME
|
|
12
|
-
});
|
|
13
|
-
const dataSource = hooks.onR2Write ? {
|
|
14
|
-
...baseDataSource,
|
|
15
|
-
async write(key, bytes) {
|
|
16
|
-
hooks.onR2Write(bytes.byteLength);
|
|
17
|
-
return baseDataSource.write(key, bytes);
|
|
18
|
-
}
|
|
19
|
-
} : baseDataSource;
|
|
20
|
-
const manifestStore = createD1ManifestStore(db);
|
|
21
|
-
const codec = createDucklingsCodec(env);
|
|
22
|
-
const executor = createDucklingsExecutor(env, {
|
|
23
|
-
rowCache,
|
|
24
|
-
...hooks.duckdbRpcTimeoutMs !== void 0 ? { rpcTimeoutMs: hooks.duckdbRpcTimeoutMs } : {}
|
|
25
|
-
});
|
|
26
|
-
return createStorageEngine({
|
|
27
|
-
dataSource,
|
|
28
|
-
manifestStore,
|
|
29
|
-
codec,
|
|
30
|
-
executor
|
|
31
|
-
});
|
|
32
|
-
} };
|
|
33
|
-
}
|
|
34
|
-
let sharedRuntime;
|
|
35
|
-
function getAnalyticsEngine(env, db, hooks = {}) {
|
|
36
|
-
sharedRuntime ??= createAnalyticsEngineRuntime();
|
|
37
|
-
return sharedRuntime.getEngine(env, db, hooks);
|
|
38
|
-
}
|
|
39
|
-
export { getAnalyticsEngine };
|
package/dist/env.d.mts
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
interface AnalyticsEnv {
|
|
2
|
-
/** R2 bucket holding parquet + rollup + entity data. Required in origin mode. */
|
|
3
|
-
R2_DATA?: R2Bucket;
|
|
4
|
-
/** Human name of the R2 bucket (used by the presigner when building public URLs). */
|
|
5
|
-
R2_BUCKET_NAME?: string;
|
|
6
|
-
/** Optional: D1 database holding the r2_manifest mirror + sync state. */
|
|
7
|
-
DB?: D1Database;
|
|
8
|
-
/**
|
|
9
|
-
* Optional: DuckDB service binding (Workers RPC) for server-side execution.
|
|
10
|
-
* Structural shape — any binding with `runSQL` + `ping` satisfies it, so
|
|
11
|
-
* hosts can declare their own binding interface without coupling to this one.
|
|
12
|
-
* Request tables use Arrow IPC chunks. Small queries pass tables inline to
|
|
13
|
-
* `runSQL`; larger queries stage chunks via `stageArrowTable`, then call
|
|
14
|
-
* `runSQL` with only SQL, and finally `dropTables`.
|
|
15
|
-
*/
|
|
16
|
-
DUCKDB_SVC?: {
|
|
17
|
-
runSQL: (args: {
|
|
18
|
-
sql: string;
|
|
19
|
-
tables?: Record<string, {
|
|
20
|
-
ipc: Uint8Array;
|
|
21
|
-
}>;
|
|
22
|
-
deadlineAt?: number;
|
|
23
|
-
}) => Promise<{
|
|
24
|
-
rows: unknown[];
|
|
25
|
-
sql: string;
|
|
26
|
-
}>;
|
|
27
|
-
stageArrowTable?: (args: {
|
|
28
|
-
table: string;
|
|
29
|
-
ipc: Uint8Array;
|
|
30
|
-
}) => Promise<void>;
|
|
31
|
-
dropTables?: (args: {
|
|
32
|
-
tables: string[];
|
|
33
|
-
}) => Promise<void>;
|
|
34
|
-
ping: () => Promise<string>;
|
|
35
|
-
};
|
|
36
|
-
/** Route override: force D1 as the manifest source even if R2 is bound. */
|
|
37
|
-
ANALYTICS_FORCE_D1?: string;
|
|
38
|
-
/** Route override: disable R2 reads entirely (consumer / legacy mode). */
|
|
39
|
-
R2_READS_ENABLED?: string;
|
|
40
|
-
/** R2 S3-API credentials for presigning (origin mode when the worker can't proxy). */
|
|
41
|
-
R2_ACCESS_KEY_ID?: string;
|
|
42
|
-
R2_SECRET_ACCESS_KEY?: string;
|
|
43
|
-
CLOUDFLARE_ACCOUNT_ID?: string;
|
|
44
|
-
}
|
|
45
|
-
export { AnalyticsEnv };
|
package/dist/index.d.mts
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import { AnalyticsEnv } from "./env.mjs";
|
|
2
|
-
import { AnalyticsEngineHooks, AnalyticsEngineRuntime, getAnalyticsEngine } from "./engine.mjs";
|
|
3
|
-
import { HostedR2QueryKeyInput, InflightDedupe, createInflightDedupe, getHostedR2QueryKey } from "./inflight-dedupe.mjs";
|
|
4
|
-
import { DucklingsRowCache, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
|
|
5
|
-
export { type AnalyticsEngineHooks, type AnalyticsEngineRuntime, type AnalyticsEnv, type DucklingsRowCache, type HostedR2QueryKeyInput, type InflightDedupe, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey };
|
package/dist/index.mjs
DELETED
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
|
|
2
|
-
import { getAnalyticsEngine } from "./engine.mjs";
|
|
3
|
-
import { createInflightDedupe, getHostedR2QueryKey } from "./inflight-dedupe.mjs";
|
|
4
|
-
export { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey };
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { AnalyticsEnv } from "./env.mjs";
|
|
2
|
-
import { ParquetCodec, QueryExecutor, Row } from "@gscdump/engine";
|
|
3
|
-
declare function createDucklingsCodec(_env: AnalyticsEnv): ParquetCodec;
|
|
4
|
-
interface DucklingsRowCache {
|
|
5
|
-
clear: () => void;
|
|
6
|
-
get: (key: string) => Row[] | undefined;
|
|
7
|
-
put: (key: string, rows: Row[]) => void;
|
|
8
|
-
}
|
|
9
|
-
declare function createDucklingsRowCache(maxBytes?: number): DucklingsRowCache;
|
|
10
|
-
interface DucklingsExecutorOptions {
|
|
11
|
-
ipcChunkBytes?: number;
|
|
12
|
-
ipcDirectCallBytes?: number;
|
|
13
|
-
ipcTotalBytes?: number;
|
|
14
|
-
/** Per-RPC wall/queue budget. Interactive default is 22 seconds. */
|
|
15
|
-
rpcTimeoutMs?: number;
|
|
16
|
-
rowCache?: DucklingsRowCache;
|
|
17
|
-
}
|
|
18
|
-
declare function createDucklingsExecutor(env: AnalyticsEnv, opts?: DucklingsExecutorOptions): QueryExecutor;
|
|
19
|
-
export { DucklingsExecutorOptions, DucklingsRowCache, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache };
|
package/dist/workers-duckdb.mjs
DELETED
|
@@ -1,362 +0,0 @@
|
|
|
1
|
-
import { rowsToArrowIPC } from "./arrow.mjs";
|
|
2
|
-
import { coerceRow } from "@gscdump/engine";
|
|
3
|
-
import { createHyparquetCodec, decodeParquetToRows } from "@gscdump/engine/hyparquet";
|
|
4
|
-
import { SCHEMAS } from "@gscdump/engine/schema";
|
|
5
|
-
import { bindLiterals } from "@gscdump/engine/sql";
|
|
6
|
-
function resolveSvc(env) {
|
|
7
|
-
const svc = env.DUCKDB_SVC;
|
|
8
|
-
if (!svc) throw new Error("DUCKDB_SVC service binding is not configured");
|
|
9
|
-
return svc;
|
|
10
|
-
}
|
|
11
|
-
const DUCKDB_RPC_TIMEOUT_MS = 22e3;
|
|
12
|
-
const WORKER_R2_MAX_FILES = 96;
|
|
13
|
-
const WORKER_R2_MAX_BYTES = 67108864;
|
|
14
|
-
const WORKER_R2_DECODE_CONCURRENCY = 2;
|
|
15
|
-
const WORKER_R2_HEAD_CONCURRENCY = 4;
|
|
16
|
-
const IPC_CHUNK_BUDGET = 8388608;
|
|
17
|
-
const IPC_DIRECT_CALL_BUDGET = 31457280;
|
|
18
|
-
const IPC_STAGED_TOTAL_BUDGET = 67108864;
|
|
19
|
-
var DuckDBServiceTimeoutError = class extends Error {
|
|
20
|
-
name = "DuckDBServiceTimeoutError";
|
|
21
|
-
constructor(timeoutMs) {
|
|
22
|
-
super(`DUCKDB_SVC.runSQL exceeded ${timeoutMs}ms deadline`);
|
|
23
|
-
}
|
|
24
|
-
};
|
|
25
|
-
function withDuckDBDeadline(op, timeoutMs, signal) {
|
|
26
|
-
if (signal?.aborted) return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
27
|
-
return new Promise((resolve, reject) => {
|
|
28
|
-
let settled = false;
|
|
29
|
-
let timer;
|
|
30
|
-
let onAbort;
|
|
31
|
-
const cleanup = () => {
|
|
32
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
33
|
-
if (onAbort) signal?.removeEventListener("abort", onAbort);
|
|
34
|
-
};
|
|
35
|
-
const settle = (complete, value) => {
|
|
36
|
-
if (settled) return;
|
|
37
|
-
settled = true;
|
|
38
|
-
cleanup();
|
|
39
|
-
complete(value);
|
|
40
|
-
};
|
|
41
|
-
onAbort = () => settle(reject, signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
42
|
-
timer = setTimeout(() => settle(reject, new DuckDBServiceTimeoutError(timeoutMs)), timeoutMs);
|
|
43
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
44
|
-
op.then((value) => settle(resolve, value), (error) => settle(reject, error));
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
function runSQLWithDeadline(svc, args, timeoutMs, signal) {
|
|
48
|
-
const deadlineAt = Date.now() + timeoutMs;
|
|
49
|
-
return withDuckDBDeadline(svc.runSQL({
|
|
50
|
-
...args,
|
|
51
|
-
deadlineAt
|
|
52
|
-
}), timeoutMs, signal);
|
|
53
|
-
}
|
|
54
|
-
function createDucklingsCodec(_env) {
|
|
55
|
-
return createHyparquetCodec();
|
|
56
|
-
}
|
|
57
|
-
const READ_PARQUET_PLACEHOLDER = /read_parquet\(\{\{(\w+)\}\}(?:\s*,[^)]*)?\)/g;
|
|
58
|
-
function tmpTableName(placeholder) {
|
|
59
|
-
const uuid = crypto.randomUUID().replace(/-/g, "_");
|
|
60
|
-
return `tmp_${placeholder.toLowerCase()}_${uuid}`;
|
|
61
|
-
}
|
|
62
|
-
async function mapLimit(items, concurrency, fn) {
|
|
63
|
-
const limit = Math.max(1, Math.floor(concurrency));
|
|
64
|
-
const out = Array.from({ length: items.length });
|
|
65
|
-
let next = 0;
|
|
66
|
-
let failed = false;
|
|
67
|
-
let failure;
|
|
68
|
-
async function worker() {
|
|
69
|
-
while (!failed && next < items.length) {
|
|
70
|
-
const index = next++;
|
|
71
|
-
try {
|
|
72
|
-
out[index] = await fn(items[index], index);
|
|
73
|
-
} catch (error) {
|
|
74
|
-
failed = true;
|
|
75
|
-
failure = error;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
80
|
-
if (failed) throw failure;
|
|
81
|
-
return out;
|
|
82
|
-
}
|
|
83
|
-
function assertWorkerReadBudget(opts) {
|
|
84
|
-
const maxFiles = opts.maxFiles ?? WORKER_R2_MAX_FILES;
|
|
85
|
-
const maxBytes = opts.maxBytes ?? WORKER_R2_MAX_BYTES;
|
|
86
|
-
const entries = Object.entries(opts.fileKeys);
|
|
87
|
-
const totalFiles = entries.reduce((acc, [, keys]) => acc + keys.length, 0);
|
|
88
|
-
if (totalFiles > maxFiles) throw new Error(`createDucklingsExecutor: planned read spans ${totalFiles} files, exceeding the ${maxFiles} file Worker budget. Narrow the date range or route through a background/windowed query.`);
|
|
89
|
-
if (!opts.sizes) return;
|
|
90
|
-
let totalBytes = 0;
|
|
91
|
-
for (const [, keys] of entries) for (const key of keys) {
|
|
92
|
-
const bytes = opts.sizes[key];
|
|
93
|
-
if (bytes !== void 0) totalBytes += Math.max(0, bytes);
|
|
94
|
-
}
|
|
95
|
-
if (totalBytes > maxBytes) throw new Error(`createDucklingsExecutor: planned read spans ${totalBytes} bytes, exceeding the ${maxBytes} byte Worker budget. Narrow the date range or route through a background/windowed query.`);
|
|
96
|
-
}
|
|
97
|
-
const ROW_CACHE_MAX_BYTES = 16777216;
|
|
98
|
-
function estimateRowsBytes(rows) {
|
|
99
|
-
if (rows.length === 0) return 0;
|
|
100
|
-
const cols = Object.keys(rows[0]).length;
|
|
101
|
-
return rows.length * cols * 64;
|
|
102
|
-
}
|
|
103
|
-
function addInferredValue(inference, value) {
|
|
104
|
-
if (value === null || value === void 0) return;
|
|
105
|
-
inference.hasValue = true;
|
|
106
|
-
if (typeof value === "string") {
|
|
107
|
-
inference.hasString = true;
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
if (typeof value === "bigint") {
|
|
111
|
-
inference.hasBigInt = true;
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
if (typeof value === "number") {
|
|
115
|
-
if (!Number.isInteger(value)) inference.hasFloat = true;
|
|
116
|
-
if (value > 2147483647 || value < -2147483648) inference.hasBigInt = true;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
function inferredColumnType(inference) {
|
|
120
|
-
if (!inference.hasValue || inference.hasString) return "VARCHAR";
|
|
121
|
-
if (inference.hasFloat) return "DOUBLE";
|
|
122
|
-
return inference.hasBigInt ? "BIGINT" : "INTEGER";
|
|
123
|
-
}
|
|
124
|
-
function chunkSchemaColumns(rows, schemaColumns) {
|
|
125
|
-
const columns = schemaColumns ? [...schemaColumns] : [];
|
|
126
|
-
const seen = new Set(columns.map((c) => c.name));
|
|
127
|
-
const extraColumns = /* @__PURE__ */ new Map();
|
|
128
|
-
for (const row of rows) for (const key in row) {
|
|
129
|
-
if (seen.has(key)) continue;
|
|
130
|
-
let inference = extraColumns.get(key);
|
|
131
|
-
if (!inference) {
|
|
132
|
-
inference = {
|
|
133
|
-
hasValue: false,
|
|
134
|
-
hasString: false,
|
|
135
|
-
hasFloat: false,
|
|
136
|
-
hasBigInt: false
|
|
137
|
-
};
|
|
138
|
-
extraColumns.set(key, inference);
|
|
139
|
-
}
|
|
140
|
-
addInferredValue(inference, row[key]);
|
|
141
|
-
}
|
|
142
|
-
if (extraColumns.size === 0) return schemaColumns ? columns : void 0;
|
|
143
|
-
for (const [name, inference] of extraColumns) columns.push({
|
|
144
|
-
name,
|
|
145
|
-
type: inferredColumnType(inference),
|
|
146
|
-
nullable: true
|
|
147
|
-
});
|
|
148
|
-
return columns;
|
|
149
|
-
}
|
|
150
|
-
function rowsToArrowIPCChunks(rows, schemaColumns, opts = {}) {
|
|
151
|
-
const maxChunkBytes = Math.max(1, Math.floor(opts.maxChunkBytes ?? IPC_CHUNK_BUDGET));
|
|
152
|
-
const placeholder = opts.placeholder ? `{{${opts.placeholder}}}` : "placeholder";
|
|
153
|
-
const chunkColumns = chunkSchemaColumns(rows, schemaColumns);
|
|
154
|
-
if (rows.length === 0) {
|
|
155
|
-
const ipc = rowsToArrowIPC([], chunkColumns);
|
|
156
|
-
if (ipc.byteLength > maxChunkBytes) throw new Error(`createDucklingsExecutor: empty ${placeholder} Arrow IPC schema encoded to ${ipc.byteLength} bytes, exceeding the ${maxChunkBytes}-byte service-binding chunk budget.`);
|
|
157
|
-
return [{
|
|
158
|
-
ipc,
|
|
159
|
-
rows: 0
|
|
160
|
-
}];
|
|
161
|
-
}
|
|
162
|
-
const estimatedPerRow = Math.max(1, Math.ceil(estimateRowsBytes(rows) / rows.length));
|
|
163
|
-
let targetRows = Math.max(1, Math.min(rows.length, Math.floor(maxChunkBytes * .75 / estimatedPerRow)));
|
|
164
|
-
const chunks = [];
|
|
165
|
-
let index = 0;
|
|
166
|
-
while (index < rows.length) {
|
|
167
|
-
let take = Math.min(targetRows, rows.length - index);
|
|
168
|
-
while (true) {
|
|
169
|
-
const slice = rows.slice(index, index + take);
|
|
170
|
-
const ipc = rowsToArrowIPC(slice, chunkColumns);
|
|
171
|
-
if (ipc.byteLength <= maxChunkBytes) {
|
|
172
|
-
chunks.push({
|
|
173
|
-
ipc,
|
|
174
|
-
rows: slice.length
|
|
175
|
-
});
|
|
176
|
-
index += take;
|
|
177
|
-
if (ipc.byteLength < maxChunkBytes * .4 && take === targetRows) targetRows = Math.min(rows.length - index || targetRows, targetRows * 2);
|
|
178
|
-
break;
|
|
179
|
-
}
|
|
180
|
-
if (take === 1) throw new Error(`createDucklingsExecutor: one ${placeholder} row encoded to ${ipc.byteLength} bytes of Arrow IPC, exceeding the ${maxChunkBytes}-byte service-binding chunk budget. Narrow the query or route through a background/windowed query.`);
|
|
181
|
-
take = Math.max(1, Math.floor(take / 2));
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
return chunks;
|
|
185
|
-
}
|
|
186
|
-
function createDucklingsRowCache(maxBytes = ROW_CACHE_MAX_BYTES) {
|
|
187
|
-
let totalBytes = 0;
|
|
188
|
-
const entries = /* @__PURE__ */ new Map();
|
|
189
|
-
return {
|
|
190
|
-
clear() {
|
|
191
|
-
entries.clear();
|
|
192
|
-
totalBytes = 0;
|
|
193
|
-
},
|
|
194
|
-
get(key) {
|
|
195
|
-
const hit = entries.get(key);
|
|
196
|
-
if (!hit) return void 0;
|
|
197
|
-
entries.delete(key);
|
|
198
|
-
entries.set(key, hit);
|
|
199
|
-
return hit.rows;
|
|
200
|
-
},
|
|
201
|
-
put(key, rows) {
|
|
202
|
-
const bytes = estimateRowsBytes(rows);
|
|
203
|
-
if (bytes > maxBytes) return;
|
|
204
|
-
const existing = entries.get(key);
|
|
205
|
-
if (existing) {
|
|
206
|
-
entries.delete(key);
|
|
207
|
-
totalBytes -= existing.bytes;
|
|
208
|
-
}
|
|
209
|
-
while (totalBytes + bytes > maxBytes) {
|
|
210
|
-
const oldest = entries.keys().next().value;
|
|
211
|
-
if (oldest === void 0) break;
|
|
212
|
-
const evicted = entries.get(oldest);
|
|
213
|
-
entries.delete(oldest);
|
|
214
|
-
totalBytes -= evicted.bytes;
|
|
215
|
-
}
|
|
216
|
-
entries.set(key, {
|
|
217
|
-
rows,
|
|
218
|
-
bytes
|
|
219
|
-
});
|
|
220
|
-
totalBytes += bytes;
|
|
221
|
-
}
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
|
-
function createDucklingsExecutor(env, opts = {}) {
|
|
225
|
-
const rowCache = opts.rowCache ?? createDucklingsRowCache();
|
|
226
|
-
const rpcTimeoutMs = opts.rpcTimeoutMs ?? DUCKDB_RPC_TIMEOUT_MS;
|
|
227
|
-
if (!Number.isFinite(rpcTimeoutMs) || rpcTimeoutMs <= 0) throw new TypeError("createDucklingsExecutor: rpcTimeoutMs must be a positive finite number");
|
|
228
|
-
return { async execute({ sql, params, fileKeys, placeholderTables, pushdownFilters, dataSource, signal, table }) {
|
|
229
|
-
signal?.throwIfAborted();
|
|
230
|
-
const svc = resolveSvc(env);
|
|
231
|
-
assertWorkerReadBudget({ fileKeys });
|
|
232
|
-
const cachedUnfiltered = /* @__PURE__ */ new Map();
|
|
233
|
-
const scheduledUnfiltered = /* @__PURE__ */ new Set();
|
|
234
|
-
const plannedReadKeys = [];
|
|
235
|
-
for (const [placeholder, keys] of Object.entries(fileKeys)) {
|
|
236
|
-
const filter = pushdownFilters?.[placeholder];
|
|
237
|
-
for (const key of keys) {
|
|
238
|
-
if (filter) {
|
|
239
|
-
plannedReadKeys.push(key);
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
const cached = cachedUnfiltered.get(key) ?? rowCache.get(key);
|
|
243
|
-
if (cached !== void 0) {
|
|
244
|
-
cachedUnfiltered.set(key, cached);
|
|
245
|
-
continue;
|
|
246
|
-
}
|
|
247
|
-
if (!scheduledUnfiltered.has(key)) {
|
|
248
|
-
scheduledUnfiltered.add(key);
|
|
249
|
-
plannedReadKeys.push(key);
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
if (dataSource.head && plannedReadKeys.length > 0) {
|
|
254
|
-
const uniqueKeys = [...new Set(plannedReadKeys)];
|
|
255
|
-
const sizes = {};
|
|
256
|
-
await mapLimit(uniqueKeys, WORKER_R2_HEAD_CONCURRENCY, async (key) => {
|
|
257
|
-
signal?.throwIfAborted();
|
|
258
|
-
sizes[key] = (await dataSource.head(key))?.bytes;
|
|
259
|
-
});
|
|
260
|
-
assertWorkerReadBudget({
|
|
261
|
-
fileKeys: { READS: plannedReadKeys },
|
|
262
|
-
sizes
|
|
263
|
-
});
|
|
264
|
-
}
|
|
265
|
-
const tempNames = {};
|
|
266
|
-
const tableChunks = {};
|
|
267
|
-
let totalIpcBytes = 0;
|
|
268
|
-
const maxChunkBytes = opts.ipcChunkBytes ?? IPC_CHUNK_BUDGET;
|
|
269
|
-
const maxDirectCallBytes = opts.ipcDirectCallBytes ?? IPC_DIRECT_CALL_BUDGET;
|
|
270
|
-
const maxTotalBytes = opts.ipcTotalBytes ?? IPC_STAGED_TOTAL_BUDGET;
|
|
271
|
-
const unfilteredLoads = /* @__PURE__ */ new Map();
|
|
272
|
-
for (const [key, rows] of cachedUnfiltered) unfilteredLoads.set(key, Promise.resolve(rows));
|
|
273
|
-
const loadUnfiltered = (key) => {
|
|
274
|
-
let loading = unfilteredLoads.get(key);
|
|
275
|
-
if (!loading) {
|
|
276
|
-
loading = (async () => {
|
|
277
|
-
signal?.throwIfAborted();
|
|
278
|
-
const bytes = await dataSource.read(key, void 0, signal);
|
|
279
|
-
signal?.throwIfAborted();
|
|
280
|
-
const rows = await decodeParquetToRows(bytes);
|
|
281
|
-
rowCache.put(key, rows);
|
|
282
|
-
return rows;
|
|
283
|
-
})();
|
|
284
|
-
unfilteredLoads.set(key, loading);
|
|
285
|
-
}
|
|
286
|
-
return loading;
|
|
287
|
-
};
|
|
288
|
-
for (const [placeholder, keys] of Object.entries(fileKeys)) {
|
|
289
|
-
signal?.throwIfAborted();
|
|
290
|
-
const filter = pushdownFilters?.[placeholder];
|
|
291
|
-
const perFile = await mapLimit(keys, WORKER_R2_DECODE_CONCURRENCY, async (key) => {
|
|
292
|
-
if (!filter) return loadUnfiltered(key);
|
|
293
|
-
signal?.throwIfAborted();
|
|
294
|
-
const bytes = await dataSource.read(key, void 0, signal);
|
|
295
|
-
signal?.throwIfAborted();
|
|
296
|
-
return await decodeParquetToRows(bytes, filter ? { filter } : {});
|
|
297
|
-
});
|
|
298
|
-
const merged = [];
|
|
299
|
-
for (const rows of perFile) merged.push(...rows);
|
|
300
|
-
const placeholderTable = placeholderTables?.[placeholder] ?? table;
|
|
301
|
-
const chunks = rowsToArrowIPCChunks(merged, SCHEMAS[placeholderTable]?.columns, {
|
|
302
|
-
maxChunkBytes,
|
|
303
|
-
placeholder
|
|
304
|
-
});
|
|
305
|
-
signal?.throwIfAborted();
|
|
306
|
-
totalIpcBytes += chunks.reduce((acc, chunk) => acc + chunk.ipc.byteLength, 0);
|
|
307
|
-
if (totalIpcBytes > maxTotalBytes) throw new Error(`createDucklingsExecutor: query encoded to ${totalIpcBytes} bytes of Arrow IPC across ${Object.keys(tableChunks).length + 1} placeholders, exceeding the ${maxTotalBytes}-byte service-binding transport budget. Window the query (chunk partitions / narrow the range).`);
|
|
308
|
-
const tmp = tmpTableName(placeholder);
|
|
309
|
-
tempNames[placeholder] = tmp;
|
|
310
|
-
tableChunks[tmp] = chunks;
|
|
311
|
-
}
|
|
312
|
-
signal?.throwIfAborted();
|
|
313
|
-
const rewritten = sql.replace(READ_PARQUET_PLACEHOLDER, (_, placeholder) => {
|
|
314
|
-
const tmp = tempNames[placeholder];
|
|
315
|
-
if (!tmp) throw new Error(`createDucklingsExecutor: SQL references {{${placeholder}}} but no fileKeys entry provided`);
|
|
316
|
-
return tmp;
|
|
317
|
-
});
|
|
318
|
-
const finalSql = bindLiterals(rewritten, params);
|
|
319
|
-
const canInlineTables = totalIpcBytes <= maxDirectCallBytes && Object.values(tableChunks).every((chunks) => chunks.length === 1);
|
|
320
|
-
let result;
|
|
321
|
-
if (canInlineTables) {
|
|
322
|
-
const tables = {};
|
|
323
|
-
for (const [name, chunks] of Object.entries(tableChunks)) tables[name] = { ipc: chunks[0].ipc };
|
|
324
|
-
result = await runSQLWithDeadline(svc, {
|
|
325
|
-
sql: finalSql,
|
|
326
|
-
tables
|
|
327
|
-
}, rpcTimeoutMs, signal);
|
|
328
|
-
} else {
|
|
329
|
-
if (!svc.stageArrowTable || !svc.dropTables) throw new Error("createDucklingsExecutor: DUCKDB_SVC does not support chunked Arrow IPC staging. Deploy the gscdump-duckdb worker with stageArrowTable/dropTables support.");
|
|
330
|
-
const staged = /* @__PURE__ */ new Set();
|
|
331
|
-
let primaryError;
|
|
332
|
-
let cleanupError;
|
|
333
|
-
try {
|
|
334
|
-
for (const [name, chunks] of Object.entries(tableChunks)) for (const chunk of chunks) {
|
|
335
|
-
signal?.throwIfAborted();
|
|
336
|
-
staged.add(name);
|
|
337
|
-
await withDuckDBDeadline(svc.stageArrowTable({
|
|
338
|
-
table: name,
|
|
339
|
-
ipc: chunk.ipc
|
|
340
|
-
}), rpcTimeoutMs, signal);
|
|
341
|
-
}
|
|
342
|
-
result = await runSQLWithDeadline(svc, { sql: finalSql }, rpcTimeoutMs, signal);
|
|
343
|
-
} catch (error) {
|
|
344
|
-
primaryError = error;
|
|
345
|
-
} finally {
|
|
346
|
-
if (staged.size > 0) try {
|
|
347
|
-
await withDuckDBDeadline(svc.dropTables({ tables: [...staged] }), rpcTimeoutMs);
|
|
348
|
-
} catch (error) {
|
|
349
|
-
cleanupError = error;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
if (primaryError) throw primaryError;
|
|
353
|
-
if (cleanupError) throw cleanupError;
|
|
354
|
-
result = result;
|
|
355
|
-
}
|
|
356
|
-
return {
|
|
357
|
-
rows: result.rows.map(coerceRow),
|
|
358
|
-
sql: result.sql
|
|
359
|
-
};
|
|
360
|
-
} };
|
|
361
|
-
}
|
|
362
|
-
export { DuckDBServiceTimeoutError, assertWorkerReadBudget, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, mapLimit, rowsToArrowIPCChunks, withDuckDBDeadline };
|