@gscdump/cloudflare 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.mjs +50 -0
- package/dist/engine.d.mts +17 -0
- package/dist/engine.mjs +35 -0
- package/dist/env.d.mts +45 -0
- package/dist/index.d.mts +4 -89
- package/dist/index.mjs +3 -475
- package/dist/inflight-dedupe.d.mts +15 -0
- package/dist/inflight-dedupe.mjs +37 -0
- package/dist/server-tail/archetype-sql.d.mts +38 -0
- package/dist/server-tail/archetype-sql.mjs +345 -0
- package/dist/server-tail/dispatcher.d.mts +47 -0
- package/dist/server-tail/dispatcher.mjs +87 -0
- package/dist/server-tail/duckdb-iceberg-executor.d.mts +88 -0
- package/dist/server-tail/duckdb-iceberg-executor.mjs +77 -0
- package/dist/server-tail/index.d.mts +4 -249
- package/dist/server-tail/index.mjs +4 -660
- package/dist/server-tail/r2-sql-client.d.mts +89 -0
- package/dist/server-tail/r2-sql-client.mjs +159 -0
- package/dist/workers-duckdb.d.mts +19 -0
- package/dist/workers-duckdb.mjs +360 -0
- package/package.json +10 -5
package/dist/index.mjs
CHANGED
|
@@ -1,476 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { createHyparquetCodec, decodeParquetToRows } from "@gscdump/engine/hyparquet";
|
|
5
|
-
import { SCHEMAS } from "@gscdump/engine/schema";
|
|
6
|
-
import { bindLiterals } from "@gscdump/engine/sql";
|
|
7
|
-
import { float64, int32, int64, tableFromArrays, tableToIPC, utf8 } from "@uwdata/flechette";
|
|
8
|
-
function arrowTypeForColumn(type) {
|
|
9
|
-
switch (type) {
|
|
10
|
-
case "VARCHAR":
|
|
11
|
-
case "DATE": return utf8();
|
|
12
|
-
case "BIGINT": return int64();
|
|
13
|
-
case "INTEGER": return int32();
|
|
14
|
-
case "DOUBLE": return float64();
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
function rowsToArrowIPC(rows, schemaColumns) {
|
|
18
|
-
const colNames = [];
|
|
19
|
-
const seen = /* @__PURE__ */ new Set();
|
|
20
|
-
const add = (name) => {
|
|
21
|
-
if (!seen.has(name)) {
|
|
22
|
-
seen.add(name);
|
|
23
|
-
colNames.push(name);
|
|
24
|
-
}
|
|
25
|
-
};
|
|
26
|
-
if (schemaColumns) for (const c of schemaColumns) add(c.name);
|
|
27
|
-
for (const row of rows) for (const key in row) add(key);
|
|
28
|
-
const schemaType = /* @__PURE__ */ new Map();
|
|
29
|
-
if (schemaColumns) for (const c of schemaColumns) schemaType.set(c.name, c.type);
|
|
30
|
-
const data = {};
|
|
31
|
-
const types = {};
|
|
32
|
-
for (const name of colNames) {
|
|
33
|
-
const values = rows.map((r) => r[name] ?? null);
|
|
34
|
-
data[name] = values;
|
|
35
|
-
const known = schemaType.get(name);
|
|
36
|
-
if (known) {
|
|
37
|
-
types[name] = arrowTypeForColumn(known);
|
|
38
|
-
continue;
|
|
39
|
-
}
|
|
40
|
-
let hasString = false;
|
|
41
|
-
let hasValue = false;
|
|
42
|
-
for (const v of values) {
|
|
43
|
-
if (v === null || v === void 0) continue;
|
|
44
|
-
hasValue = true;
|
|
45
|
-
if (typeof v === "string") {
|
|
46
|
-
hasString = true;
|
|
47
|
-
break;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
if (hasString || !hasValue) types[name] = utf8();
|
|
51
|
-
}
|
|
52
|
-
const ipc = tableToIPC(tableFromArrays(data, { types }), { format: "stream" });
|
|
53
|
-
if (!ipc) throw new Error("rowsToArrowIPC: tableToIPC returned null");
|
|
54
|
-
return ipc;
|
|
55
|
-
}
|
|
56
|
-
function resolveSvc(env) {
|
|
57
|
-
const svc = env.DUCKDB_SVC;
|
|
58
|
-
if (!svc) throw new Error("DUCKDB_SVC service binding is not configured");
|
|
59
|
-
return svc;
|
|
60
|
-
}
|
|
61
|
-
const DUCKDB_RPC_TIMEOUT_MS = 22e3;
|
|
62
|
-
const WORKER_R2_MAX_FILES = 96;
|
|
63
|
-
const WORKER_R2_MAX_BYTES = 64 * 1024 * 1024;
|
|
64
|
-
const WORKER_R2_DECODE_CONCURRENCY = 2;
|
|
65
|
-
const WORKER_R2_HEAD_CONCURRENCY = 4;
|
|
66
|
-
const IPC_CHUNK_BUDGET = 8 * 1024 * 1024;
|
|
67
|
-
const IPC_DIRECT_CALL_BUDGET = 30 * 1024 * 1024;
|
|
68
|
-
const IPC_STAGED_TOTAL_BUDGET = 64 * 1024 * 1024;
|
|
69
|
-
var DuckDBServiceTimeoutError = class extends Error {
|
|
70
|
-
name = "DuckDBServiceTimeoutError";
|
|
71
|
-
constructor(timeoutMs) {
|
|
72
|
-
super(`DUCKDB_SVC.runSQL exceeded ${timeoutMs}ms deadline`);
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
function withDuckDBDeadline(op, timeoutMs, signal) {
|
|
76
|
-
if (signal?.aborted) return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
77
|
-
return new Promise((resolve, reject) => {
|
|
78
|
-
let settled = false;
|
|
79
|
-
let timer;
|
|
80
|
-
let onAbort;
|
|
81
|
-
const cleanup = () => {
|
|
82
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
83
|
-
if (onAbort) signal?.removeEventListener("abort", onAbort);
|
|
84
|
-
};
|
|
85
|
-
const settle = (complete, value) => {
|
|
86
|
-
if (settled) return;
|
|
87
|
-
settled = true;
|
|
88
|
-
cleanup();
|
|
89
|
-
complete(value);
|
|
90
|
-
};
|
|
91
|
-
onAbort = () => settle(reject, signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
92
|
-
timer = setTimeout(() => settle(reject, new DuckDBServiceTimeoutError(timeoutMs)), timeoutMs);
|
|
93
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
94
|
-
op.then((value) => settle(resolve, value), (error) => settle(reject, error));
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
function runSQLWithDeadline(svc, args, timeoutMs, signal) {
|
|
98
|
-
const deadlineAt = Date.now() + timeoutMs;
|
|
99
|
-
return withDuckDBDeadline(svc.runSQL({
|
|
100
|
-
...args,
|
|
101
|
-
deadlineAt
|
|
102
|
-
}), timeoutMs, signal);
|
|
103
|
-
}
|
|
104
|
-
function createDucklingsCodec(_env) {
|
|
105
|
-
return createHyparquetCodec();
|
|
106
|
-
}
|
|
107
|
-
const READ_PARQUET_PLACEHOLDER = /read_parquet\(\{\{(\w+)\}\}(?:\s*,[^)]*)?\)/g;
|
|
108
|
-
function tmpTableName(placeholder) {
|
|
109
|
-
const uuid = crypto.randomUUID().replace(/-/g, "_");
|
|
110
|
-
return `tmp_${placeholder.toLowerCase()}_${uuid}`;
|
|
111
|
-
}
|
|
112
|
-
async function mapLimit(items, concurrency, fn) {
|
|
113
|
-
const limit = Math.max(1, Math.floor(concurrency));
|
|
114
|
-
const out = Array.from({ length: items.length });
|
|
115
|
-
let next = 0;
|
|
116
|
-
let failed = false;
|
|
117
|
-
let failure;
|
|
118
|
-
async function worker() {
|
|
119
|
-
while (!failed && next < items.length) {
|
|
120
|
-
const index = next++;
|
|
121
|
-
try {
|
|
122
|
-
out[index] = await fn(items[index], index);
|
|
123
|
-
} catch (error) {
|
|
124
|
-
failed = true;
|
|
125
|
-
failure = error;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
130
|
-
if (failed) throw failure;
|
|
131
|
-
return out;
|
|
132
|
-
}
|
|
133
|
-
function assertWorkerReadBudget(opts) {
|
|
134
|
-
const maxFiles = opts.maxFiles ?? WORKER_R2_MAX_FILES;
|
|
135
|
-
const maxBytes = opts.maxBytes ?? WORKER_R2_MAX_BYTES;
|
|
136
|
-
const entries = Object.entries(opts.fileKeys);
|
|
137
|
-
const totalFiles = entries.reduce((acc, [, keys]) => acc + keys.length, 0);
|
|
138
|
-
if (totalFiles > maxFiles) throw new Error(`createDucklingsExecutor: planned read spans ${totalFiles} files, exceeding the ${maxFiles} file Worker budget. Narrow the date range or route through a background/windowed query.`);
|
|
139
|
-
if (!opts.sizes) return;
|
|
140
|
-
let totalBytes = 0;
|
|
141
|
-
for (const [, keys] of entries) for (const key of keys) {
|
|
142
|
-
const bytes = opts.sizes[key];
|
|
143
|
-
if (bytes !== void 0) totalBytes += Math.max(0, bytes);
|
|
144
|
-
}
|
|
145
|
-
if (totalBytes > maxBytes) throw new Error(`createDucklingsExecutor: planned read spans ${totalBytes} bytes, exceeding the ${maxBytes} byte Worker budget. Narrow the date range or route through a background/windowed query.`);
|
|
146
|
-
}
|
|
147
|
-
const ROW_CACHE_MAX_BYTES = 16 * 1024 * 1024;
|
|
148
|
-
function estimateRowsBytes(rows) {
|
|
149
|
-
if (rows.length === 0) return 0;
|
|
150
|
-
const cols = Object.keys(rows[0]).length;
|
|
151
|
-
return rows.length * cols * 64;
|
|
152
|
-
}
|
|
153
|
-
function addInferredValue(inference, value) {
|
|
154
|
-
if (value === null || value === void 0) return;
|
|
155
|
-
inference.hasValue = true;
|
|
156
|
-
if (typeof value === "string") {
|
|
157
|
-
inference.hasString = true;
|
|
158
|
-
return;
|
|
159
|
-
}
|
|
160
|
-
if (typeof value === "bigint") {
|
|
161
|
-
inference.hasBigInt = true;
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
if (typeof value === "number") {
|
|
165
|
-
if (!Number.isInteger(value)) inference.hasFloat = true;
|
|
166
|
-
if (value > 2147483647 || value < -2147483648) inference.hasBigInt = true;
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
function inferredColumnType(inference) {
|
|
170
|
-
if (!inference.hasValue || inference.hasString) return "VARCHAR";
|
|
171
|
-
if (inference.hasFloat) return "DOUBLE";
|
|
172
|
-
return inference.hasBigInt ? "BIGINT" : "INTEGER";
|
|
173
|
-
}
|
|
174
|
-
function chunkSchemaColumns(rows, schemaColumns) {
|
|
175
|
-
const columns = schemaColumns ? [...schemaColumns] : [];
|
|
176
|
-
const seen = new Set(columns.map((c) => c.name));
|
|
177
|
-
const extraColumns = /* @__PURE__ */ new Map();
|
|
178
|
-
for (const row of rows) for (const key in row) {
|
|
179
|
-
if (seen.has(key)) continue;
|
|
180
|
-
let inference = extraColumns.get(key);
|
|
181
|
-
if (!inference) {
|
|
182
|
-
inference = {
|
|
183
|
-
hasValue: false,
|
|
184
|
-
hasString: false,
|
|
185
|
-
hasFloat: false,
|
|
186
|
-
hasBigInt: false
|
|
187
|
-
};
|
|
188
|
-
extraColumns.set(key, inference);
|
|
189
|
-
}
|
|
190
|
-
addInferredValue(inference, row[key]);
|
|
191
|
-
}
|
|
192
|
-
if (extraColumns.size === 0) return schemaColumns ? columns : void 0;
|
|
193
|
-
for (const [name, inference] of extraColumns) columns.push({
|
|
194
|
-
name,
|
|
195
|
-
type: inferredColumnType(inference),
|
|
196
|
-
nullable: true
|
|
197
|
-
});
|
|
198
|
-
return columns;
|
|
199
|
-
}
|
|
200
|
-
function rowsToArrowIPCChunks(rows, schemaColumns, opts = {}) {
|
|
201
|
-
const maxChunkBytes = Math.max(1, Math.floor(opts.maxChunkBytes ?? IPC_CHUNK_BUDGET));
|
|
202
|
-
const placeholder = opts.placeholder ? `{{${opts.placeholder}}}` : "placeholder";
|
|
203
|
-
const chunkColumns = chunkSchemaColumns(rows, schemaColumns);
|
|
204
|
-
if (rows.length === 0) {
|
|
205
|
-
const ipc = rowsToArrowIPC([], chunkColumns);
|
|
206
|
-
if (ipc.byteLength > maxChunkBytes) throw new Error(`createDucklingsExecutor: empty ${placeholder} Arrow IPC schema encoded to ${ipc.byteLength} bytes, exceeding the ${maxChunkBytes}-byte service-binding chunk budget.`);
|
|
207
|
-
return [{
|
|
208
|
-
ipc,
|
|
209
|
-
rows: 0
|
|
210
|
-
}];
|
|
211
|
-
}
|
|
212
|
-
const estimatedPerRow = Math.max(1, Math.ceil(estimateRowsBytes(rows) / rows.length));
|
|
213
|
-
let targetRows = Math.max(1, Math.min(rows.length, Math.floor(maxChunkBytes * .75 / estimatedPerRow)));
|
|
214
|
-
const chunks = [];
|
|
215
|
-
let index = 0;
|
|
216
|
-
while (index < rows.length) {
|
|
217
|
-
let take = Math.min(targetRows, rows.length - index);
|
|
218
|
-
while (true) {
|
|
219
|
-
const slice = rows.slice(index, index + take);
|
|
220
|
-
const ipc = rowsToArrowIPC(slice, chunkColumns);
|
|
221
|
-
if (ipc.byteLength <= maxChunkBytes) {
|
|
222
|
-
chunks.push({
|
|
223
|
-
ipc,
|
|
224
|
-
rows: slice.length
|
|
225
|
-
});
|
|
226
|
-
index += take;
|
|
227
|
-
if (ipc.byteLength < maxChunkBytes * .4 && take === targetRows) targetRows = Math.min(rows.length - index || targetRows, targetRows * 2);
|
|
228
|
-
break;
|
|
229
|
-
}
|
|
230
|
-
if (take === 1) throw new Error(`createDucklingsExecutor: one ${placeholder} row encoded to ${ipc.byteLength} bytes of Arrow IPC, exceeding the ${maxChunkBytes}-byte service-binding chunk budget. Narrow the query or route through a background/windowed query.`);
|
|
231
|
-
take = Math.max(1, Math.floor(take / 2));
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
return chunks;
|
|
235
|
-
}
|
|
236
|
-
function createDucklingsRowCache(maxBytes = ROW_CACHE_MAX_BYTES) {
|
|
237
|
-
let totalBytes = 0;
|
|
238
|
-
const entries = /* @__PURE__ */ new Map();
|
|
239
|
-
return {
|
|
240
|
-
clear() {
|
|
241
|
-
entries.clear();
|
|
242
|
-
totalBytes = 0;
|
|
243
|
-
},
|
|
244
|
-
get(key) {
|
|
245
|
-
const hit = entries.get(key);
|
|
246
|
-
if (!hit) return void 0;
|
|
247
|
-
entries.delete(key);
|
|
248
|
-
entries.set(key, hit);
|
|
249
|
-
return hit.rows;
|
|
250
|
-
},
|
|
251
|
-
put(key, rows) {
|
|
252
|
-
const bytes = estimateRowsBytes(rows);
|
|
253
|
-
if (bytes > maxBytes) return;
|
|
254
|
-
const existing = entries.get(key);
|
|
255
|
-
if (existing) {
|
|
256
|
-
entries.delete(key);
|
|
257
|
-
totalBytes -= existing.bytes;
|
|
258
|
-
}
|
|
259
|
-
while (totalBytes + bytes > maxBytes) {
|
|
260
|
-
const oldest = entries.keys().next().value;
|
|
261
|
-
if (oldest === void 0) break;
|
|
262
|
-
const evicted = entries.get(oldest);
|
|
263
|
-
entries.delete(oldest);
|
|
264
|
-
totalBytes -= evicted.bytes;
|
|
265
|
-
}
|
|
266
|
-
entries.set(key, {
|
|
267
|
-
rows,
|
|
268
|
-
bytes
|
|
269
|
-
});
|
|
270
|
-
totalBytes += bytes;
|
|
271
|
-
}
|
|
272
|
-
};
|
|
273
|
-
}
|
|
274
|
-
function createDucklingsExecutor(env, opts = {}) {
|
|
275
|
-
const rowCache = opts.rowCache ?? createDucklingsRowCache();
|
|
276
|
-
const rpcTimeoutMs = opts.rpcTimeoutMs ?? DUCKDB_RPC_TIMEOUT_MS;
|
|
277
|
-
if (!Number.isFinite(rpcTimeoutMs) || rpcTimeoutMs <= 0) throw new TypeError("createDucklingsExecutor: rpcTimeoutMs must be a positive finite number");
|
|
278
|
-
return { async execute({ sql, params, fileKeys, placeholderTables, pushdownFilters, dataSource, signal, table }) {
|
|
279
|
-
signal?.throwIfAborted();
|
|
280
|
-
const svc = resolveSvc(env);
|
|
281
|
-
assertWorkerReadBudget({ fileKeys });
|
|
282
|
-
const cachedUnfiltered = /* @__PURE__ */ new Map();
|
|
283
|
-
const scheduledUnfiltered = /* @__PURE__ */ new Set();
|
|
284
|
-
const plannedReadKeys = [];
|
|
285
|
-
for (const [placeholder, keys] of Object.entries(fileKeys)) {
|
|
286
|
-
const filter = pushdownFilters?.[placeholder];
|
|
287
|
-
for (const key of keys) {
|
|
288
|
-
if (filter) {
|
|
289
|
-
plannedReadKeys.push(key);
|
|
290
|
-
continue;
|
|
291
|
-
}
|
|
292
|
-
const cached = cachedUnfiltered.get(key) ?? rowCache.get(key);
|
|
293
|
-
if (cached !== void 0) {
|
|
294
|
-
cachedUnfiltered.set(key, cached);
|
|
295
|
-
continue;
|
|
296
|
-
}
|
|
297
|
-
if (!scheduledUnfiltered.has(key)) {
|
|
298
|
-
scheduledUnfiltered.add(key);
|
|
299
|
-
plannedReadKeys.push(key);
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
if (dataSource.head && plannedReadKeys.length > 0) {
|
|
304
|
-
const uniqueKeys = [...new Set(plannedReadKeys)];
|
|
305
|
-
const sizes = {};
|
|
306
|
-
await mapLimit(uniqueKeys, WORKER_R2_HEAD_CONCURRENCY, async (key) => {
|
|
307
|
-
signal?.throwIfAborted();
|
|
308
|
-
sizes[key] = (await dataSource.head(key))?.bytes;
|
|
309
|
-
});
|
|
310
|
-
assertWorkerReadBudget({
|
|
311
|
-
fileKeys: { READS: plannedReadKeys },
|
|
312
|
-
sizes
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
const tempNames = {};
|
|
316
|
-
const tableChunks = {};
|
|
317
|
-
let totalIpcBytes = 0;
|
|
318
|
-
const maxChunkBytes = opts.ipcChunkBytes ?? IPC_CHUNK_BUDGET;
|
|
319
|
-
const maxDirectCallBytes = opts.ipcDirectCallBytes ?? IPC_DIRECT_CALL_BUDGET;
|
|
320
|
-
const maxTotalBytes = opts.ipcTotalBytes ?? IPC_STAGED_TOTAL_BUDGET;
|
|
321
|
-
const unfilteredLoads = /* @__PURE__ */ new Map();
|
|
322
|
-
for (const [key, rows] of cachedUnfiltered) unfilteredLoads.set(key, Promise.resolve(rows));
|
|
323
|
-
const loadUnfiltered = (key) => {
|
|
324
|
-
let loading = unfilteredLoads.get(key);
|
|
325
|
-
if (!loading) {
|
|
326
|
-
loading = (async () => {
|
|
327
|
-
signal?.throwIfAborted();
|
|
328
|
-
const bytes = await dataSource.read(key, void 0, signal);
|
|
329
|
-
signal?.throwIfAborted();
|
|
330
|
-
const rows = await decodeParquetToRows(bytes);
|
|
331
|
-
rowCache.put(key, rows);
|
|
332
|
-
return rows;
|
|
333
|
-
})();
|
|
334
|
-
unfilteredLoads.set(key, loading);
|
|
335
|
-
}
|
|
336
|
-
return loading;
|
|
337
|
-
};
|
|
338
|
-
for (const [placeholder, keys] of Object.entries(fileKeys)) {
|
|
339
|
-
signal?.throwIfAborted();
|
|
340
|
-
const filter = pushdownFilters?.[placeholder];
|
|
341
|
-
const perFile = await mapLimit(keys, WORKER_R2_DECODE_CONCURRENCY, async (key) => {
|
|
342
|
-
if (!filter) return loadUnfiltered(key);
|
|
343
|
-
signal?.throwIfAborted();
|
|
344
|
-
const bytes = await dataSource.read(key, void 0, signal);
|
|
345
|
-
signal?.throwIfAborted();
|
|
346
|
-
return await decodeParquetToRows(bytes, filter ? { filter } : {});
|
|
347
|
-
});
|
|
348
|
-
const merged = [];
|
|
349
|
-
for (const rows of perFile) merged.push(...rows);
|
|
350
|
-
const chunks = rowsToArrowIPCChunks(merged, SCHEMAS[placeholderTables?.[placeholder] ?? table]?.columns, {
|
|
351
|
-
maxChunkBytes,
|
|
352
|
-
placeholder
|
|
353
|
-
});
|
|
354
|
-
signal?.throwIfAborted();
|
|
355
|
-
totalIpcBytes += chunks.reduce((acc, chunk) => acc + chunk.ipc.byteLength, 0);
|
|
356
|
-
if (totalIpcBytes > maxTotalBytes) throw new Error(`createDucklingsExecutor: query encoded to ${totalIpcBytes} bytes of Arrow IPC across ${Object.keys(tableChunks).length + 1} placeholders, exceeding the ${maxTotalBytes}-byte service-binding transport budget. Window the query (chunk partitions / narrow the range).`);
|
|
357
|
-
const tmp = tmpTableName(placeholder);
|
|
358
|
-
tempNames[placeholder] = tmp;
|
|
359
|
-
tableChunks[tmp] = chunks;
|
|
360
|
-
}
|
|
361
|
-
signal?.throwIfAborted();
|
|
362
|
-
const finalSql = bindLiterals(sql.replace(READ_PARQUET_PLACEHOLDER, (_, placeholder) => {
|
|
363
|
-
const tmp = tempNames[placeholder];
|
|
364
|
-
if (!tmp) throw new Error(`createDucklingsExecutor: SQL references {{${placeholder}}} but no fileKeys entry provided`);
|
|
365
|
-
return tmp;
|
|
366
|
-
}), params);
|
|
367
|
-
const canInlineTables = totalIpcBytes <= maxDirectCallBytes && Object.values(tableChunks).every((chunks) => chunks.length === 1);
|
|
368
|
-
let result;
|
|
369
|
-
if (canInlineTables) {
|
|
370
|
-
const tables = {};
|
|
371
|
-
for (const [name, chunks] of Object.entries(tableChunks)) tables[name] = { ipc: chunks[0].ipc };
|
|
372
|
-
result = await runSQLWithDeadline(svc, {
|
|
373
|
-
sql: finalSql,
|
|
374
|
-
tables
|
|
375
|
-
}, rpcTimeoutMs, signal);
|
|
376
|
-
} else {
|
|
377
|
-
if (!svc.stageArrowTable || !svc.dropTables) throw new Error("createDucklingsExecutor: DUCKDB_SVC does not support chunked Arrow IPC staging. Deploy the gscdump-duckdb worker with stageArrowTable/dropTables support.");
|
|
378
|
-
const staged = /* @__PURE__ */ new Set();
|
|
379
|
-
let primaryError;
|
|
380
|
-
let cleanupError;
|
|
381
|
-
try {
|
|
382
|
-
for (const [name, chunks] of Object.entries(tableChunks)) for (const chunk of chunks) {
|
|
383
|
-
signal?.throwIfAborted();
|
|
384
|
-
staged.add(name);
|
|
385
|
-
await withDuckDBDeadline(svc.stageArrowTable({
|
|
386
|
-
table: name,
|
|
387
|
-
ipc: chunk.ipc
|
|
388
|
-
}), rpcTimeoutMs, signal);
|
|
389
|
-
}
|
|
390
|
-
result = await runSQLWithDeadline(svc, { sql: finalSql }, rpcTimeoutMs, signal);
|
|
391
|
-
} catch (error) {
|
|
392
|
-
primaryError = error;
|
|
393
|
-
} finally {
|
|
394
|
-
if (staged.size > 0) try {
|
|
395
|
-
await withDuckDBDeadline(svc.dropTables({ tables: [...staged] }), rpcTimeoutMs);
|
|
396
|
-
} catch (error) {
|
|
397
|
-
cleanupError = error;
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
if (primaryError) throw primaryError;
|
|
401
|
-
if (cleanupError) throw cleanupError;
|
|
402
|
-
result = result;
|
|
403
|
-
}
|
|
404
|
-
return {
|
|
405
|
-
rows: result.rows.map(coerceRow),
|
|
406
|
-
sql: result.sql
|
|
407
|
-
};
|
|
408
|
-
} };
|
|
409
|
-
}
|
|
410
|
-
function createAnalyticsEngineRuntime() {
|
|
411
|
-
const rowCache = createDucklingsRowCache();
|
|
412
|
-
return { getEngine(env, db, hooks = {}) {
|
|
413
|
-
if (!env.R2_DATA) return null;
|
|
414
|
-
const baseDataSource = createR2DataSource({
|
|
415
|
-
bucket: env.R2_DATA,
|
|
416
|
-
bucketName: env.R2_BUCKET_NAME
|
|
417
|
-
});
|
|
418
|
-
return createStorageEngine({
|
|
419
|
-
dataSource: hooks.onR2Write ? {
|
|
420
|
-
...baseDataSource,
|
|
421
|
-
async write(key, bytes) {
|
|
422
|
-
hooks.onR2Write(bytes.byteLength);
|
|
423
|
-
return baseDataSource.write(key, bytes);
|
|
424
|
-
}
|
|
425
|
-
} : baseDataSource,
|
|
426
|
-
manifestStore: createD1ManifestStore(db),
|
|
427
|
-
codec: createDucklingsCodec(env),
|
|
428
|
-
executor: createDucklingsExecutor(env, {
|
|
429
|
-
rowCache,
|
|
430
|
-
...hooks.duckdbRpcTimeoutMs !== void 0 ? { rpcTimeoutMs: hooks.duckdbRpcTimeoutMs } : {}
|
|
431
|
-
})
|
|
432
|
-
});
|
|
433
|
-
} };
|
|
434
|
-
}
|
|
435
|
-
let sharedRuntime;
|
|
436
|
-
function getAnalyticsEngine(env, db, hooks = {}) {
|
|
437
|
-
sharedRuntime ??= createAnalyticsEngineRuntime();
|
|
438
|
-
return sharedRuntime.getEngine(env, db, hooks);
|
|
439
|
-
}
|
|
440
|
-
function createInflightDedupe() {
|
|
441
|
-
const inflight = /* @__PURE__ */ new Map();
|
|
442
|
-
return {
|
|
443
|
-
dedupe(key, run) {
|
|
444
|
-
const existing = inflight.get(key);
|
|
445
|
-
if (existing) return existing;
|
|
446
|
-
const promise = run().finally(() => {
|
|
447
|
-
if (inflight.get(key) === promise) inflight.delete(key);
|
|
448
|
-
});
|
|
449
|
-
inflight.set(key, promise);
|
|
450
|
-
return promise;
|
|
451
|
-
},
|
|
452
|
-
has(key) {
|
|
453
|
-
return inflight.has(key);
|
|
454
|
-
},
|
|
455
|
-
clear() {
|
|
456
|
-
inflight.clear();
|
|
457
|
-
}
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
function stableStringify(value) {
|
|
461
|
-
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
462
|
-
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => {
|
|
463
|
-
return `${JSON.stringify(key)}:${stableStringify(value[key])}`;
|
|
464
|
-
}).join(",")}}`;
|
|
465
|
-
return JSON.stringify(value);
|
|
466
|
-
}
|
|
467
|
-
function getHostedR2QueryKey(input) {
|
|
468
|
-
return JSON.stringify([
|
|
469
|
-
input.userId,
|
|
470
|
-
input.siteId,
|
|
471
|
-
stableStringify(input.state),
|
|
472
|
-
input.comparison === void 0 ? null : stableStringify(input.comparison),
|
|
473
|
-
input.comparisonFilter ?? null
|
|
474
|
-
]);
|
|
475
|
-
}
|
|
1
|
+
import { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
|
|
2
|
+
import { getAnalyticsEngine } from "./engine.mjs";
|
|
3
|
+
import { createInflightDedupe, getHostedR2QueryKey } from "./inflight-dedupe.mjs";
|
|
476
4
|
export { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
interface InflightDedupe<T> {
|
|
2
|
+
dedupe: (key: string, run: () => Promise<T>) => Promise<T>;
|
|
3
|
+
has: (key: string) => boolean;
|
|
4
|
+
clear: () => void;
|
|
5
|
+
}
|
|
6
|
+
declare function createInflightDedupe<T>(): InflightDedupe<T>;
|
|
7
|
+
interface HostedR2QueryKeyInput {
|
|
8
|
+
userId: string | number;
|
|
9
|
+
siteId: string;
|
|
10
|
+
state: unknown;
|
|
11
|
+
comparison?: unknown;
|
|
12
|
+
comparisonFilter?: string;
|
|
13
|
+
}
|
|
14
|
+
declare function getHostedR2QueryKey(input: HostedR2QueryKeyInput): string;
|
|
15
|
+
export { HostedR2QueryKeyInput, InflightDedupe, createInflightDedupe, getHostedR2QueryKey };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
function createInflightDedupe() {
|
|
2
|
+
const inflight = /* @__PURE__ */ new Map();
|
|
3
|
+
return {
|
|
4
|
+
dedupe(key, run) {
|
|
5
|
+
const existing = inflight.get(key);
|
|
6
|
+
if (existing) return existing;
|
|
7
|
+
const promise = run().finally(() => {
|
|
8
|
+
if (inflight.get(key) === promise) inflight.delete(key);
|
|
9
|
+
});
|
|
10
|
+
inflight.set(key, promise);
|
|
11
|
+
return promise;
|
|
12
|
+
},
|
|
13
|
+
has(key) {
|
|
14
|
+
return inflight.has(key);
|
|
15
|
+
},
|
|
16
|
+
clear() {
|
|
17
|
+
inflight.clear();
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function stableStringify(value) {
|
|
22
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
23
|
+
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => {
|
|
24
|
+
return `${JSON.stringify(key)}:${stableStringify(value[key])}`;
|
|
25
|
+
}).join(",")}}`;
|
|
26
|
+
return JSON.stringify(value);
|
|
27
|
+
}
|
|
28
|
+
function getHostedR2QueryKey(input) {
|
|
29
|
+
return JSON.stringify([
|
|
30
|
+
input.userId,
|
|
31
|
+
input.siteId,
|
|
32
|
+
stableStringify(input.state),
|
|
33
|
+
input.comparison === void 0 ? null : stableStringify(input.comparison),
|
|
34
|
+
input.comparisonFilter ?? null
|
|
35
|
+
]);
|
|
36
|
+
}
|
|
37
|
+
export { createInflightDedupe, getHostedR2QueryKey };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { ArchetypeQuery } from "@gscdump/contracts/archetypes";
|
|
2
|
+
declare const TABLE_PLACEHOLDER = "{{TABLE}}";
|
|
3
|
+
type ArchetypeFactTable = 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates';
|
|
4
|
+
interface ArchetypeSqlPlan {
|
|
5
|
+
sql: string;
|
|
6
|
+
params: unknown[];
|
|
7
|
+
table: ArchetypeFactTable;
|
|
8
|
+
}
|
|
9
|
+
type PartitionPredicateMode = 'bare' | 'r2-sql-concat';
|
|
10
|
+
type PartitionKeyEncoding = 'int' | 'string';
|
|
11
|
+
interface BuildArchetypeSqlOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Set by the DuckDB file-list executor, which reads raw Iceberg parquet
|
|
14
|
+
* directly via `read_parquet([...])`, bypassing the catalog metadata layer
|
|
15
|
+
* that synthesizes the identity-partition columns. `site_id` / `search_type`
|
|
16
|
+
* are partition identities NOT materialized in the data files (see engine
|
|
17
|
+
* `iceberg/schema.ts`: "carried implicitly in the object-key prefix"), so a
|
|
18
|
+
* `WHERE site_id = ?` predicate fails with `Referenced column "site_id" not
|
|
19
|
+
* found in FROM clause`. Those files are already pruned to (site_id,
|
|
20
|
+
* search_type) at resolution time, so the predicate is redundant — emit only
|
|
21
|
+
* the row-level `date` range (a real stored column; `month(date)` is the
|
|
22
|
+
* partition transform, not `date` itself).
|
|
23
|
+
*
|
|
24
|
+
* The R2 SQL catalog path (default `false`) exposes partition columns as
|
|
25
|
+
* virtual columns and needs the full predicate.
|
|
26
|
+
*/
|
|
27
|
+
partitionPruned?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Int-partition catalogs are the default and use bare equality. Legacy
|
|
30
|
+
* string-partition catalogs can request CONCAT-wrapped predicates directly
|
|
31
|
+
* instead of post-processing generated SQL.
|
|
32
|
+
*/
|
|
33
|
+
partitionPredicateMode?: PartitionPredicateMode;
|
|
34
|
+
/** Preferred input for new callers. Defaults to `'int'`. */
|
|
35
|
+
partitionKeyEncoding?: PartitionKeyEncoding;
|
|
36
|
+
}
|
|
37
|
+
declare function buildArchetypeSql(query: ArchetypeQuery, opts?: BuildArchetypeSqlOptions): ArchetypeSqlPlan;
|
|
38
|
+
export { ArchetypeFactTable, ArchetypeSqlPlan, BuildArchetypeSqlOptions, PartitionKeyEncoding, PartitionPredicateMode, TABLE_PLACEHOLDER, buildArchetypeSql };
|