@gscdump/cloudflare 1.3.2 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,476 +1,4 @@
1
- import { coerceRow, createStorageEngine } from "@gscdump/engine";
2
- import { createD1ManifestStore } from "@gscdump/engine-sqlite";
3
- import { createR2DataSource } from "@gscdump/engine/r2";
4
- import { createHyparquetCodec, decodeParquetToRows } from "@gscdump/engine/hyparquet";
5
- import { SCHEMAS } from "@gscdump/engine/schema";
6
- import { bindLiterals } from "@gscdump/engine/sql";
7
- import { float64, int32, int64, tableFromArrays, tableToIPC, utf8 } from "@uwdata/flechette";
8
- function arrowTypeForColumn(type) {
9
- switch (type) {
10
- case "VARCHAR":
11
- case "DATE": return utf8();
12
- case "BIGINT": return int64();
13
- case "INTEGER": return int32();
14
- case "DOUBLE": return float64();
15
- }
16
- }
17
- function rowsToArrowIPC(rows, schemaColumns) {
18
- const colNames = [];
19
- const seen = /* @__PURE__ */ new Set();
20
- const add = (name) => {
21
- if (!seen.has(name)) {
22
- seen.add(name);
23
- colNames.push(name);
24
- }
25
- };
26
- if (schemaColumns) for (const c of schemaColumns) add(c.name);
27
- for (const row of rows) for (const key in row) add(key);
28
- const schemaType = /* @__PURE__ */ new Map();
29
- if (schemaColumns) for (const c of schemaColumns) schemaType.set(c.name, c.type);
30
- const data = {};
31
- const types = {};
32
- for (const name of colNames) {
33
- const values = rows.map((r) => r[name] ?? null);
34
- data[name] = values;
35
- const known = schemaType.get(name);
36
- if (known) {
37
- types[name] = arrowTypeForColumn(known);
38
- continue;
39
- }
40
- let hasString = false;
41
- let hasValue = false;
42
- for (const v of values) {
43
- if (v === null || v === void 0) continue;
44
- hasValue = true;
45
- if (typeof v === "string") {
46
- hasString = true;
47
- break;
48
- }
49
- }
50
- if (hasString || !hasValue) types[name] = utf8();
51
- }
52
- const ipc = tableToIPC(tableFromArrays(data, { types }), { format: "stream" });
53
- if (!ipc) throw new Error("rowsToArrowIPC: tableToIPC returned null");
54
- return ipc;
55
- }
56
- function resolveSvc(env) {
57
- const svc = env.DUCKDB_SVC;
58
- if (!svc) throw new Error("DUCKDB_SVC service binding is not configured");
59
- return svc;
60
- }
61
- const DUCKDB_RPC_TIMEOUT_MS = 22e3;
62
- const WORKER_R2_MAX_FILES = 96;
63
- const WORKER_R2_MAX_BYTES = 64 * 1024 * 1024;
64
- const WORKER_R2_DECODE_CONCURRENCY = 2;
65
- const WORKER_R2_HEAD_CONCURRENCY = 4;
66
- const IPC_CHUNK_BUDGET = 8 * 1024 * 1024;
67
- const IPC_DIRECT_CALL_BUDGET = 30 * 1024 * 1024;
68
- const IPC_STAGED_TOTAL_BUDGET = 64 * 1024 * 1024;
69
- var DuckDBServiceTimeoutError = class extends Error {
70
- name = "DuckDBServiceTimeoutError";
71
- constructor(timeoutMs) {
72
- super(`DUCKDB_SVC.runSQL exceeded ${timeoutMs}ms deadline`);
73
- }
74
- };
75
- function withDuckDBDeadline(op, timeoutMs, signal) {
76
- if (signal?.aborted) return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
77
- return new Promise((resolve, reject) => {
78
- let settled = false;
79
- let timer;
80
- let onAbort;
81
- const cleanup = () => {
82
- if (timer !== void 0) clearTimeout(timer);
83
- if (onAbort) signal?.removeEventListener("abort", onAbort);
84
- };
85
- const settle = (complete, value) => {
86
- if (settled) return;
87
- settled = true;
88
- cleanup();
89
- complete(value);
90
- };
91
- onAbort = () => settle(reject, signal.reason ?? new DOMException("Aborted", "AbortError"));
92
- timer = setTimeout(() => settle(reject, new DuckDBServiceTimeoutError(timeoutMs)), timeoutMs);
93
- signal?.addEventListener("abort", onAbort, { once: true });
94
- op.then((value) => settle(resolve, value), (error) => settle(reject, error));
95
- });
96
- }
97
- function runSQLWithDeadline(svc, args, timeoutMs, signal) {
98
- const deadlineAt = Date.now() + timeoutMs;
99
- return withDuckDBDeadline(svc.runSQL({
100
- ...args,
101
- deadlineAt
102
- }), timeoutMs, signal);
103
- }
104
- function createDucklingsCodec(_env) {
105
- return createHyparquetCodec();
106
- }
107
- const READ_PARQUET_PLACEHOLDER = /read_parquet\(\{\{(\w+)\}\}(?:\s*,[^)]*)?\)/g;
108
- function tmpTableName(placeholder) {
109
- const uuid = crypto.randomUUID().replace(/-/g, "_");
110
- return `tmp_${placeholder.toLowerCase()}_${uuid}`;
111
- }
112
- async function mapLimit(items, concurrency, fn) {
113
- const limit = Math.max(1, Math.floor(concurrency));
114
- const out = Array.from({ length: items.length });
115
- let next = 0;
116
- let failed = false;
117
- let failure;
118
- async function worker() {
119
- while (!failed && next < items.length) {
120
- const index = next++;
121
- try {
122
- out[index] = await fn(items[index], index);
123
- } catch (error) {
124
- failed = true;
125
- failure = error;
126
- }
127
- }
128
- }
129
- await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
130
- if (failed) throw failure;
131
- return out;
132
- }
133
- function assertWorkerReadBudget(opts) {
134
- const maxFiles = opts.maxFiles ?? WORKER_R2_MAX_FILES;
135
- const maxBytes = opts.maxBytes ?? WORKER_R2_MAX_BYTES;
136
- const entries = Object.entries(opts.fileKeys);
137
- const totalFiles = entries.reduce((acc, [, keys]) => acc + keys.length, 0);
138
- if (totalFiles > maxFiles) throw new Error(`createDucklingsExecutor: planned read spans ${totalFiles} files, exceeding the ${maxFiles} file Worker budget. Narrow the date range or route through a background/windowed query.`);
139
- if (!opts.sizes) return;
140
- let totalBytes = 0;
141
- for (const [, keys] of entries) for (const key of keys) {
142
- const bytes = opts.sizes[key];
143
- if (bytes !== void 0) totalBytes += Math.max(0, bytes);
144
- }
145
- if (totalBytes > maxBytes) throw new Error(`createDucklingsExecutor: planned read spans ${totalBytes} bytes, exceeding the ${maxBytes} byte Worker budget. Narrow the date range or route through a background/windowed query.`);
146
- }
147
- const ROW_CACHE_MAX_BYTES = 16 * 1024 * 1024;
148
- function estimateRowsBytes(rows) {
149
- if (rows.length === 0) return 0;
150
- const cols = Object.keys(rows[0]).length;
151
- return rows.length * cols * 64;
152
- }
153
- function addInferredValue(inference, value) {
154
- if (value === null || value === void 0) return;
155
- inference.hasValue = true;
156
- if (typeof value === "string") {
157
- inference.hasString = true;
158
- return;
159
- }
160
- if (typeof value === "bigint") {
161
- inference.hasBigInt = true;
162
- return;
163
- }
164
- if (typeof value === "number") {
165
- if (!Number.isInteger(value)) inference.hasFloat = true;
166
- if (value > 2147483647 || value < -2147483648) inference.hasBigInt = true;
167
- }
168
- }
169
- function inferredColumnType(inference) {
170
- if (!inference.hasValue || inference.hasString) return "VARCHAR";
171
- if (inference.hasFloat) return "DOUBLE";
172
- return inference.hasBigInt ? "BIGINT" : "INTEGER";
173
- }
174
- function chunkSchemaColumns(rows, schemaColumns) {
175
- const columns = schemaColumns ? [...schemaColumns] : [];
176
- const seen = new Set(columns.map((c) => c.name));
177
- const extraColumns = /* @__PURE__ */ new Map();
178
- for (const row of rows) for (const key in row) {
179
- if (seen.has(key)) continue;
180
- let inference = extraColumns.get(key);
181
- if (!inference) {
182
- inference = {
183
- hasValue: false,
184
- hasString: false,
185
- hasFloat: false,
186
- hasBigInt: false
187
- };
188
- extraColumns.set(key, inference);
189
- }
190
- addInferredValue(inference, row[key]);
191
- }
192
- if (extraColumns.size === 0) return schemaColumns ? columns : void 0;
193
- for (const [name, inference] of extraColumns) columns.push({
194
- name,
195
- type: inferredColumnType(inference),
196
- nullable: true
197
- });
198
- return columns;
199
- }
200
- function rowsToArrowIPCChunks(rows, schemaColumns, opts = {}) {
201
- const maxChunkBytes = Math.max(1, Math.floor(opts.maxChunkBytes ?? IPC_CHUNK_BUDGET));
202
- const placeholder = opts.placeholder ? `{{${opts.placeholder}}}` : "placeholder";
203
- const chunkColumns = chunkSchemaColumns(rows, schemaColumns);
204
- if (rows.length === 0) {
205
- const ipc = rowsToArrowIPC([], chunkColumns);
206
- if (ipc.byteLength > maxChunkBytes) throw new Error(`createDucklingsExecutor: empty ${placeholder} Arrow IPC schema encoded to ${ipc.byteLength} bytes, exceeding the ${maxChunkBytes}-byte service-binding chunk budget.`);
207
- return [{
208
- ipc,
209
- rows: 0
210
- }];
211
- }
212
- const estimatedPerRow = Math.max(1, Math.ceil(estimateRowsBytes(rows) / rows.length));
213
- let targetRows = Math.max(1, Math.min(rows.length, Math.floor(maxChunkBytes * .75 / estimatedPerRow)));
214
- const chunks = [];
215
- let index = 0;
216
- while (index < rows.length) {
217
- let take = Math.min(targetRows, rows.length - index);
218
- while (true) {
219
- const slice = rows.slice(index, index + take);
220
- const ipc = rowsToArrowIPC(slice, chunkColumns);
221
- if (ipc.byteLength <= maxChunkBytes) {
222
- chunks.push({
223
- ipc,
224
- rows: slice.length
225
- });
226
- index += take;
227
- if (ipc.byteLength < maxChunkBytes * .4 && take === targetRows) targetRows = Math.min(rows.length - index || targetRows, targetRows * 2);
228
- break;
229
- }
230
- if (take === 1) throw new Error(`createDucklingsExecutor: one ${placeholder} row encoded to ${ipc.byteLength} bytes of Arrow IPC, exceeding the ${maxChunkBytes}-byte service-binding chunk budget. Narrow the query or route through a background/windowed query.`);
231
- take = Math.max(1, Math.floor(take / 2));
232
- }
233
- }
234
- return chunks;
235
- }
236
- function createDucklingsRowCache(maxBytes = ROW_CACHE_MAX_BYTES) {
237
- let totalBytes = 0;
238
- const entries = /* @__PURE__ */ new Map();
239
- return {
240
- clear() {
241
- entries.clear();
242
- totalBytes = 0;
243
- },
244
- get(key) {
245
- const hit = entries.get(key);
246
- if (!hit) return void 0;
247
- entries.delete(key);
248
- entries.set(key, hit);
249
- return hit.rows;
250
- },
251
- put(key, rows) {
252
- const bytes = estimateRowsBytes(rows);
253
- if (bytes > maxBytes) return;
254
- const existing = entries.get(key);
255
- if (existing) {
256
- entries.delete(key);
257
- totalBytes -= existing.bytes;
258
- }
259
- while (totalBytes + bytes > maxBytes) {
260
- const oldest = entries.keys().next().value;
261
- if (oldest === void 0) break;
262
- const evicted = entries.get(oldest);
263
- entries.delete(oldest);
264
- totalBytes -= evicted.bytes;
265
- }
266
- entries.set(key, {
267
- rows,
268
- bytes
269
- });
270
- totalBytes += bytes;
271
- }
272
- };
273
- }
274
- function createDucklingsExecutor(env, opts = {}) {
275
- const rowCache = opts.rowCache ?? createDucklingsRowCache();
276
- const rpcTimeoutMs = opts.rpcTimeoutMs ?? DUCKDB_RPC_TIMEOUT_MS;
277
- if (!Number.isFinite(rpcTimeoutMs) || rpcTimeoutMs <= 0) throw new TypeError("createDucklingsExecutor: rpcTimeoutMs must be a positive finite number");
278
- return { async execute({ sql, params, fileKeys, placeholderTables, pushdownFilters, dataSource, signal, table }) {
279
- signal?.throwIfAborted();
280
- const svc = resolveSvc(env);
281
- assertWorkerReadBudget({ fileKeys });
282
- const cachedUnfiltered = /* @__PURE__ */ new Map();
283
- const scheduledUnfiltered = /* @__PURE__ */ new Set();
284
- const plannedReadKeys = [];
285
- for (const [placeholder, keys] of Object.entries(fileKeys)) {
286
- const filter = pushdownFilters?.[placeholder];
287
- for (const key of keys) {
288
- if (filter) {
289
- plannedReadKeys.push(key);
290
- continue;
291
- }
292
- const cached = cachedUnfiltered.get(key) ?? rowCache.get(key);
293
- if (cached !== void 0) {
294
- cachedUnfiltered.set(key, cached);
295
- continue;
296
- }
297
- if (!scheduledUnfiltered.has(key)) {
298
- scheduledUnfiltered.add(key);
299
- plannedReadKeys.push(key);
300
- }
301
- }
302
- }
303
- if (dataSource.head && plannedReadKeys.length > 0) {
304
- const uniqueKeys = [...new Set(plannedReadKeys)];
305
- const sizes = {};
306
- await mapLimit(uniqueKeys, WORKER_R2_HEAD_CONCURRENCY, async (key) => {
307
- signal?.throwIfAborted();
308
- sizes[key] = (await dataSource.head(key))?.bytes;
309
- });
310
- assertWorkerReadBudget({
311
- fileKeys: { READS: plannedReadKeys },
312
- sizes
313
- });
314
- }
315
- const tempNames = {};
316
- const tableChunks = {};
317
- let totalIpcBytes = 0;
318
- const maxChunkBytes = opts.ipcChunkBytes ?? IPC_CHUNK_BUDGET;
319
- const maxDirectCallBytes = opts.ipcDirectCallBytes ?? IPC_DIRECT_CALL_BUDGET;
320
- const maxTotalBytes = opts.ipcTotalBytes ?? IPC_STAGED_TOTAL_BUDGET;
321
- const unfilteredLoads = /* @__PURE__ */ new Map();
322
- for (const [key, rows] of cachedUnfiltered) unfilteredLoads.set(key, Promise.resolve(rows));
323
- const loadUnfiltered = (key) => {
324
- let loading = unfilteredLoads.get(key);
325
- if (!loading) {
326
- loading = (async () => {
327
- signal?.throwIfAborted();
328
- const bytes = await dataSource.read(key, void 0, signal);
329
- signal?.throwIfAborted();
330
- const rows = await decodeParquetToRows(bytes);
331
- rowCache.put(key, rows);
332
- return rows;
333
- })();
334
- unfilteredLoads.set(key, loading);
335
- }
336
- return loading;
337
- };
338
- for (const [placeholder, keys] of Object.entries(fileKeys)) {
339
- signal?.throwIfAborted();
340
- const filter = pushdownFilters?.[placeholder];
341
- const perFile = await mapLimit(keys, WORKER_R2_DECODE_CONCURRENCY, async (key) => {
342
- if (!filter) return loadUnfiltered(key);
343
- signal?.throwIfAborted();
344
- const bytes = await dataSource.read(key, void 0, signal);
345
- signal?.throwIfAborted();
346
- return await decodeParquetToRows(bytes, filter ? { filter } : {});
347
- });
348
- const merged = [];
349
- for (const rows of perFile) merged.push(...rows);
350
- const chunks = rowsToArrowIPCChunks(merged, SCHEMAS[placeholderTables?.[placeholder] ?? table]?.columns, {
351
- maxChunkBytes,
352
- placeholder
353
- });
354
- signal?.throwIfAborted();
355
- totalIpcBytes += chunks.reduce((acc, chunk) => acc + chunk.ipc.byteLength, 0);
356
- if (totalIpcBytes > maxTotalBytes) throw new Error(`createDucklingsExecutor: query encoded to ${totalIpcBytes} bytes of Arrow IPC across ${Object.keys(tableChunks).length + 1} placeholders, exceeding the ${maxTotalBytes}-byte service-binding transport budget. Window the query (chunk partitions / narrow the range).`);
357
- const tmp = tmpTableName(placeholder);
358
- tempNames[placeholder] = tmp;
359
- tableChunks[tmp] = chunks;
360
- }
361
- signal?.throwIfAborted();
362
- const finalSql = bindLiterals(sql.replace(READ_PARQUET_PLACEHOLDER, (_, placeholder) => {
363
- const tmp = tempNames[placeholder];
364
- if (!tmp) throw new Error(`createDucklingsExecutor: SQL references {{${placeholder}}} but no fileKeys entry provided`);
365
- return tmp;
366
- }), params);
367
- const canInlineTables = totalIpcBytes <= maxDirectCallBytes && Object.values(tableChunks).every((chunks) => chunks.length === 1);
368
- let result;
369
- if (canInlineTables) {
370
- const tables = {};
371
- for (const [name, chunks] of Object.entries(tableChunks)) tables[name] = { ipc: chunks[0].ipc };
372
- result = await runSQLWithDeadline(svc, {
373
- sql: finalSql,
374
- tables
375
- }, rpcTimeoutMs, signal);
376
- } else {
377
- if (!svc.stageArrowTable || !svc.dropTables) throw new Error("createDucklingsExecutor: DUCKDB_SVC does not support chunked Arrow IPC staging. Deploy the gscdump-duckdb worker with stageArrowTable/dropTables support.");
378
- const staged = /* @__PURE__ */ new Set();
379
- let primaryError;
380
- let cleanupError;
381
- try {
382
- for (const [name, chunks] of Object.entries(tableChunks)) for (const chunk of chunks) {
383
- signal?.throwIfAborted();
384
- staged.add(name);
385
- await withDuckDBDeadline(svc.stageArrowTable({
386
- table: name,
387
- ipc: chunk.ipc
388
- }), rpcTimeoutMs, signal);
389
- }
390
- result = await runSQLWithDeadline(svc, { sql: finalSql }, rpcTimeoutMs, signal);
391
- } catch (error) {
392
- primaryError = error;
393
- } finally {
394
- if (staged.size > 0) try {
395
- await withDuckDBDeadline(svc.dropTables({ tables: [...staged] }), rpcTimeoutMs);
396
- } catch (error) {
397
- cleanupError = error;
398
- }
399
- }
400
- if (primaryError) throw primaryError;
401
- if (cleanupError) throw cleanupError;
402
- result = result;
403
- }
404
- return {
405
- rows: result.rows.map(coerceRow),
406
- sql: result.sql
407
- };
408
- } };
409
- }
410
- function createAnalyticsEngineRuntime() {
411
- const rowCache = createDucklingsRowCache();
412
- return { getEngine(env, db, hooks = {}) {
413
- if (!env.R2_DATA) return null;
414
- const baseDataSource = createR2DataSource({
415
- bucket: env.R2_DATA,
416
- bucketName: env.R2_BUCKET_NAME
417
- });
418
- return createStorageEngine({
419
- dataSource: hooks.onR2Write ? {
420
- ...baseDataSource,
421
- async write(key, bytes) {
422
- hooks.onR2Write(bytes.byteLength);
423
- return baseDataSource.write(key, bytes);
424
- }
425
- } : baseDataSource,
426
- manifestStore: createD1ManifestStore(db),
427
- codec: createDucklingsCodec(env),
428
- executor: createDucklingsExecutor(env, {
429
- rowCache,
430
- ...hooks.duckdbRpcTimeoutMs !== void 0 ? { rpcTimeoutMs: hooks.duckdbRpcTimeoutMs } : {}
431
- })
432
- });
433
- } };
434
- }
435
- let sharedRuntime;
436
- function getAnalyticsEngine(env, db, hooks = {}) {
437
- sharedRuntime ??= createAnalyticsEngineRuntime();
438
- return sharedRuntime.getEngine(env, db, hooks);
439
- }
440
- function createInflightDedupe() {
441
- const inflight = /* @__PURE__ */ new Map();
442
- return {
443
- dedupe(key, run) {
444
- const existing = inflight.get(key);
445
- if (existing) return existing;
446
- const promise = run().finally(() => {
447
- if (inflight.get(key) === promise) inflight.delete(key);
448
- });
449
- inflight.set(key, promise);
450
- return promise;
451
- },
452
- has(key) {
453
- return inflight.has(key);
454
- },
455
- clear() {
456
- inflight.clear();
457
- }
458
- };
459
- }
460
- function stableStringify(value) {
461
- if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
462
- if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => {
463
- return `${JSON.stringify(key)}:${stableStringify(value[key])}`;
464
- }).join(",")}}`;
465
- return JSON.stringify(value);
466
- }
467
- function getHostedR2QueryKey(input) {
468
- return JSON.stringify([
469
- input.userId,
470
- input.siteId,
471
- stableStringify(input.state),
472
- input.comparison === void 0 ? null : stableStringify(input.comparison),
473
- input.comparisonFilter ?? null
474
- ]);
475
- }
1
+ import { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
2
+ import { getAnalyticsEngine } from "./engine.mjs";
3
+ import { createInflightDedupe, getHostedR2QueryKey } from "./inflight-dedupe.mjs";
476
4
  export { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey };
@@ -0,0 +1,15 @@
1
+ interface InflightDedupe<T> {
2
+ dedupe: (key: string, run: () => Promise<T>) => Promise<T>;
3
+ has: (key: string) => boolean;
4
+ clear: () => void;
5
+ }
6
+ declare function createInflightDedupe<T>(): InflightDedupe<T>;
7
+ interface HostedR2QueryKeyInput {
8
+ userId: string | number;
9
+ siteId: string;
10
+ state: unknown;
11
+ comparison?: unknown;
12
+ comparisonFilter?: string;
13
+ }
14
+ declare function getHostedR2QueryKey(input: HostedR2QueryKeyInput): string;
15
+ export { HostedR2QueryKeyInput, InflightDedupe, createInflightDedupe, getHostedR2QueryKey };
@@ -0,0 +1,37 @@
1
+ function createInflightDedupe() {
2
+ const inflight = /* @__PURE__ */ new Map();
3
+ return {
4
+ dedupe(key, run) {
5
+ const existing = inflight.get(key);
6
+ if (existing) return existing;
7
+ const promise = run().finally(() => {
8
+ if (inflight.get(key) === promise) inflight.delete(key);
9
+ });
10
+ inflight.set(key, promise);
11
+ return promise;
12
+ },
13
+ has(key) {
14
+ return inflight.has(key);
15
+ },
16
+ clear() {
17
+ inflight.clear();
18
+ }
19
+ };
20
+ }
21
+ function stableStringify(value) {
22
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
23
+ if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => {
24
+ return `${JSON.stringify(key)}:${stableStringify(value[key])}`;
25
+ }).join(",")}}`;
26
+ return JSON.stringify(value);
27
+ }
28
+ function getHostedR2QueryKey(input) {
29
+ return JSON.stringify([
30
+ input.userId,
31
+ input.siteId,
32
+ stableStringify(input.state),
33
+ input.comparison === void 0 ? null : stableStringify(input.comparison),
34
+ input.comparisonFilter ?? null
35
+ ]);
36
+ }
37
+ export { createInflightDedupe, getHostedR2QueryKey };
@@ -0,0 +1,38 @@
1
+ import { ArchetypeQuery } from "@gscdump/contracts/archetypes";
2
+ declare const TABLE_PLACEHOLDER = "{{TABLE}}";
3
+ type ArchetypeFactTable = 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates';
4
+ interface ArchetypeSqlPlan {
5
+ sql: string;
6
+ params: unknown[];
7
+ table: ArchetypeFactTable;
8
+ }
9
+ type PartitionPredicateMode = 'bare' | 'r2-sql-concat';
10
+ type PartitionKeyEncoding = 'int' | 'string';
11
+ interface BuildArchetypeSqlOptions {
12
+ /**
13
+ * Set by the DuckDB file-list executor, which reads raw Iceberg parquet
14
+ * directly via `read_parquet([...])`, bypassing the catalog metadata layer
15
+ * that synthesizes the identity-partition columns. `site_id` / `search_type`
16
+ * are partition identities NOT materialized in the data files (see engine
17
+ * `iceberg/schema.ts`: "carried implicitly in the object-key prefix"), so a
18
+ * `WHERE site_id = ?` predicate fails with `Referenced column "site_id" not
19
+ * found in FROM clause`. Those files are already pruned to (site_id,
20
+ * search_type) at resolution time, so the predicate is redundant — emit only
21
+ * the row-level `date` range (a real stored column; `month(date)` is the
22
+ * partition transform, not `date` itself).
23
+ *
24
+ * The R2 SQL catalog path (default `false`) exposes partition columns as
25
+ * virtual columns and needs the full predicate.
26
+ */
27
+ partitionPruned?: boolean;
28
+ /**
29
+ * Int-partition catalogs are the default and use bare equality. Legacy
30
+ * string-partition catalogs can request CONCAT-wrapped predicates directly
31
+ * instead of post-processing generated SQL.
32
+ */
33
+ partitionPredicateMode?: PartitionPredicateMode;
34
+ /** Preferred input for new callers. Defaults to `'int'`. */
35
+ partitionKeyEncoding?: PartitionKeyEncoding;
36
+ }
37
+ declare function buildArchetypeSql(query: ArchetypeQuery, opts?: BuildArchetypeSqlOptions): ArchetypeSqlPlan;
38
+ export { ArchetypeFactTable, ArchetypeSqlPlan, BuildArchetypeSqlOptions, PartitionKeyEncoding, PartitionPredicateMode, TABLE_PLACEHOLDER, buildArchetypeSql };