@gscdump/engine-duckdb-wasm 0.25.14 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.mjs +184 -45
- package/package.json +4 -4
package/dist/index.d.mts
CHANGED
|
@@ -4,6 +4,7 @@ import { AnyRelations, EmptyRelations, ExtractTablesWithRelations, Schema as Sch
|
|
|
4
4
|
import { DrizzleSchema as Schema, countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
|
|
5
5
|
import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
|
|
6
6
|
import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
|
|
7
|
+
import { Result } from "gscdump/result";
|
|
7
8
|
import { ComparisonMode, ResolveWindowOptions, ResolvedWindow, WindowPreset, resolveWindow } from "@gscdump/engine/period";
|
|
8
9
|
import { ArchetypeQuery } from "@gscdump/sdk";
|
|
9
10
|
import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles, DuckDBConfig } from "@duckdb/duckdb-wasm";
|
|
@@ -217,6 +218,14 @@ interface DuckDBWasmBootResult {
|
|
|
217
218
|
conn: AsyncDuckDBConnection;
|
|
218
219
|
}
|
|
219
220
|
interface BootDuckDBWasmOptions {
|
|
221
|
+
/**
|
|
222
|
+
* DuckDB-WASM logger. Defaults to a `ConsoleLogger` thresholded at
|
|
223
|
+
* `LogLevel.WARNING`, so real warnings/errors still surface but the per-query
|
|
224
|
+
* INFO events (START/OK/RUN) — which DuckDB's default `ConsoleLogger()` emits
|
|
225
|
+
* as raw objects, flooding the host console with dozens of lines per render —
|
|
226
|
+
* are dropped. Pass `new ConsoleLogger(LogLevel.DEBUG)` to see everything, or
|
|
227
|
+
* `new VoidLogger()` to silence it entirely.
|
|
228
|
+
*/
|
|
220
229
|
logger?: unknown;
|
|
221
230
|
/**
|
|
222
231
|
* Override the jsDelivr-hosted bundle map. Required in environments where
|
|
@@ -354,12 +363,40 @@ interface BrowserAnalysisRuntime {
|
|
|
354
363
|
declare class BrowserAttachBudgetExceededError extends Error {
|
|
355
364
|
name: string;
|
|
356
365
|
}
|
|
366
|
+
interface BrowserAttachError {
|
|
367
|
+
kind: 'browser-attach-budget-exceeded';
|
|
368
|
+
message: string;
|
|
369
|
+
/** Which budget tripped: the file count, the hinted byte plan, or the running byte plan. */
|
|
370
|
+
budget: 'maxFiles' | 'maxBytes';
|
|
371
|
+
}
|
|
372
|
+
declare const browserAttachErrors: {
|
|
373
|
+
readonly maxFilesExceeded: (files: number, maxFiles: number) => BrowserAttachError;
|
|
374
|
+
readonly hintedBytesExceeded: (hintedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
375
|
+
readonly plannedBytesExceeded: (plannedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
376
|
+
};
|
|
377
|
+
declare function isBrowserAttachError(value: unknown): value is BrowserAttachError;
|
|
357
378
|
declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
|
|
358
379
|
declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
|
|
380
|
+
/**
|
|
381
|
+
* Errors-as-values core for {@link attachParquetUrlTables}: returns a typed
|
|
382
|
+
* {@link BrowserAttachError} when the requested file set blows the local file-
|
|
383
|
+
* count / byte budget, so callers can branch (route the affected tables
|
|
384
|
+
* server-side) instead of catching an untyped throw. WASM/DuckDB/HTTP IO
|
|
385
|
+
* failures stay defects and propagate. `attachParquetUrlTables` is the thin
|
|
386
|
+
* throwing wrapper preserving the historical `BrowserAttachBudgetExceededError`.
|
|
387
|
+
*/
|
|
388
|
+
declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOptions): Promise<Result<AttachedTablesHandle, BrowserAttachError>>;
|
|
389
|
+
/**
|
|
390
|
+
* Attach browser parquet URL tables, throwing
|
|
391
|
+
* {@link BrowserAttachBudgetExceededError} when the requested set blows the
|
|
392
|
+
* file-count / byte budget. Thin throwing wrapper over
|
|
393
|
+
* {@link attachParquetUrlTablesResult}; existing call sites and their
|
|
394
|
+
* `instanceof BrowserAttachBudgetExceededError` checks keep holding.
|
|
395
|
+
*/
|
|
359
396
|
declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
|
|
360
397
|
declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, options?: {
|
|
361
398
|
schema?: string;
|
|
362
399
|
version?: number | string;
|
|
363
400
|
attachedTables?: readonly string[];
|
|
364
401
|
}): BrowserAnalysisRuntime;
|
|
365
|
-
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
|
402
|
+
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
package/dist/index.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { runAnalyzerFromSource } from "@gscdump/engine/analyzer";
|
|
|
9
9
|
import { pgResolverAdapter } from "@gscdump/engine/resolver";
|
|
10
10
|
import { createAttachedTableSource } from "@gscdump/engine/source";
|
|
11
11
|
import { sqlEscape } from "@gscdump/engine/sql";
|
|
12
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
12
13
|
import { resolveWindow } from "@gscdump/engine/period";
|
|
13
14
|
const METRIC_SQL = {
|
|
14
15
|
clicks: "SUM(clicks)",
|
|
@@ -77,6 +78,55 @@ function rangePredicate(q) {
|
|
|
77
78
|
]
|
|
78
79
|
};
|
|
79
80
|
}
|
|
81
|
+
function compareRangePredicate(q) {
|
|
82
|
+
if (!q.compareRange) return null;
|
|
83
|
+
return {
|
|
84
|
+
sql: "date BETWEEN ? AND ? AND search_type = ?",
|
|
85
|
+
params: [
|
|
86
|
+
q.compareRange.start,
|
|
87
|
+
q.compareRange.end,
|
|
88
|
+
q.searchType
|
|
89
|
+
]
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
const STD_METRICS = [
|
|
93
|
+
"clicks",
|
|
94
|
+
"impressions",
|
|
95
|
+
"ctr",
|
|
96
|
+
"position"
|
|
97
|
+
];
|
|
98
|
+
function coalesceMetric(metric, src, alias) {
|
|
99
|
+
const ref = `${src}.${metric}`;
|
|
100
|
+
return metric === "clicks" || metric === "impressions" ? `CAST(COALESCE(${ref}, 0) AS DOUBLE) AS ${alias}` : `COALESCE(${ref}, 0) AS ${alias}`;
|
|
101
|
+
}
|
|
102
|
+
function prevAlias(metric) {
|
|
103
|
+
return `prev${metric.charAt(0).toUpperCase()}${metric.slice(1)}`;
|
|
104
|
+
}
|
|
105
|
+
function moverClause(movers) {
|
|
106
|
+
const curClicks = "COALESCE(c.clicks, 0)";
|
|
107
|
+
const prevClicks = "COALESCE(p.clicks, 0)";
|
|
108
|
+
const curImpr = "COALESCE(c.impressions, 0)";
|
|
109
|
+
const prevImpr = "COALESCE(p.impressions, 0)";
|
|
110
|
+
switch (movers) {
|
|
111
|
+
case "improving": return {
|
|
112
|
+
where: `${curClicks} > ${prevClicks}`,
|
|
113
|
+
order: `(${curClicks} - ${prevClicks}) DESC`
|
|
114
|
+
};
|
|
115
|
+
case "declining": return {
|
|
116
|
+
where: `${curClicks} < ${prevClicks}`,
|
|
117
|
+
order: `(${curClicks} - ${prevClicks}) ASC`
|
|
118
|
+
};
|
|
119
|
+
case "new": return {
|
|
120
|
+
where: `${prevImpr} = 0 AND ${curImpr} > 0`,
|
|
121
|
+
order: `${curClicks} DESC, ${curImpr} DESC`
|
|
122
|
+
};
|
|
123
|
+
case "lost": return {
|
|
124
|
+
where: `${curImpr} = 0 AND ${prevImpr} > 0`,
|
|
125
|
+
order: `${prevImpr} DESC`
|
|
126
|
+
};
|
|
127
|
+
default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
80
130
|
function facetPredicate(query) {
|
|
81
131
|
const facets = query.facets;
|
|
82
132
|
if (!facets?.length) return {
|
|
@@ -144,9 +194,34 @@ function compileArchetypeSql(query) {
|
|
|
144
194
|
case "top-n-breakdown": {
|
|
145
195
|
const table = tableForDimensions([query.dimension]);
|
|
146
196
|
const where = rangePredicate(query);
|
|
197
|
+
const cmp = compareRangePredicate(query);
|
|
147
198
|
const dir = query.orderBy.dir === "asc" ? "ASC" : "DESC";
|
|
199
|
+
const metricList = query.metrics.includes(query.orderBy.metric) ? query.metrics : [...query.metrics, query.orderBy.metric];
|
|
200
|
+
const variantSel = query.dimension === "queryCanonical" ? ", COUNT(DISTINCT query) AS variantCount" : "";
|
|
148
201
|
if (query.dimension === "device") {
|
|
149
|
-
|
|
202
|
+
if (cmp) {
|
|
203
|
+
const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
|
|
204
|
+
const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
|
|
205
|
+
let sql = `WITH cur AS (${deviceUnpivotSql(metricList, where.sql, false)}), prev AS (${deviceUnpivotSql(STD_METRICS, cmp.sql, false)}) SELECT COALESCE(c.device, p.device) AS device, ${curCols}, ${prevCols} FROM cur c FULL OUTER JOIN prev p ON c.device = p.device ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
|
|
206
|
+
const params = [
|
|
207
|
+
...where.params,
|
|
208
|
+
...where.params,
|
|
209
|
+
...where.params,
|
|
210
|
+
...cmp.params,
|
|
211
|
+
...cmp.params,
|
|
212
|
+
...cmp.params,
|
|
213
|
+
query.limit
|
|
214
|
+
];
|
|
215
|
+
if (query.offset && query.offset > 0) {
|
|
216
|
+
sql += " OFFSET ?";
|
|
217
|
+
params.push(query.offset);
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
table,
|
|
221
|
+
sql,
|
|
222
|
+
params
|
|
223
|
+
};
|
|
224
|
+
}
|
|
150
225
|
let sql = `SELECT device, ${metricList.join(", ")} FROM (${deviceUnpivotSql(metricList, where.sql, false)}) ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
|
|
151
226
|
const params = [
|
|
152
227
|
...where.params,
|
|
@@ -167,7 +242,32 @@ function compileArchetypeSql(query) {
|
|
|
167
242
|
const col = DIM_COLUMN[query.dimension];
|
|
168
243
|
const facet = facetPredicate(query);
|
|
169
244
|
const totalCol = query.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
|
|
170
|
-
|
|
245
|
+
if (cmp) {
|
|
246
|
+
const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
|
|
247
|
+
const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
|
|
248
|
+
const variantOut = query.dimension === "queryCanonical" ? ", c.variantCount AS variantCount" : "";
|
|
249
|
+
const mover = query.movers ? moverClause(query.movers) : null;
|
|
250
|
+
const moverWhere = mover ? `WHERE ${mover.where} ` : "";
|
|
251
|
+
const orderSql = mover ? mover.order : `${query.orderBy.metric} ${dir}`;
|
|
252
|
+
let sql = `WITH cur AS (SELECT ${col} AS k, ${metricSelectList(metricList)}${variantSel} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col}), prev AS (SELECT ${col} AS k, ${metricSelectList(STD_METRICS)} FROM ${table} WHERE ${cmp.sql}${facet.sql} GROUP BY ${col}) SELECT COALESCE(c.k, p.k) AS ${query.dimension}, ${curCols}, ${prevCols}${variantOut}${totalCol} FROM cur c FULL OUTER JOIN prev p ON c.k = p.k ${moverWhere}ORDER BY ${orderSql} LIMIT ?`;
|
|
253
|
+
const params = [
|
|
254
|
+
...where.params,
|
|
255
|
+
...facet.params,
|
|
256
|
+
...cmp.params,
|
|
257
|
+
...facet.params,
|
|
258
|
+
query.limit
|
|
259
|
+
];
|
|
260
|
+
if (query.offset && query.offset > 0) {
|
|
261
|
+
sql += " OFFSET ?";
|
|
262
|
+
params.push(query.offset);
|
|
263
|
+
}
|
|
264
|
+
return {
|
|
265
|
+
table,
|
|
266
|
+
sql,
|
|
267
|
+
params
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
let sql = `SELECT ${col} AS ${query.dimension}, ${metricSelectList(query.metrics)}${variantSel}${totalCol} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col} ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
|
|
171
271
|
const params = [
|
|
172
272
|
...where.params,
|
|
173
273
|
...facet.params,
|
|
@@ -278,7 +378,7 @@ async function createClient(db, conn) {
|
|
|
278
378
|
try {
|
|
279
379
|
return arrowToRows(await stmt.query(...params));
|
|
280
380
|
} finally {
|
|
281
|
-
stmt.close();
|
|
381
|
+
await stmt.close();
|
|
282
382
|
}
|
|
283
383
|
},
|
|
284
384
|
async close() {
|
|
@@ -583,7 +683,7 @@ function readParquetViewSql$1(schema, table, files) {
|
|
|
583
683
|
function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
|
|
584
684
|
const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
|
|
585
685
|
if (lakeFiles.length === 0) return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlay}`;
|
|
586
|
-
return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(lakeFiles)} UNION ALL BY NAME ${overlay} WHERE
|
|
686
|
+
return `CREATE OR REPLACE VIEW ${schema}.${table} AS WITH lake AS MATERIALIZED (${lakeSelect(lakeFiles)}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlay}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
|
|
587
687
|
}
|
|
588
688
|
async function runWithConcurrency$1(items, concurrency, fn) {
|
|
589
689
|
let next = 0;
|
|
@@ -781,6 +881,37 @@ let nextAttachId = 0;
|
|
|
781
881
|
var BrowserAttachBudgetExceededError = class extends Error {
|
|
782
882
|
name = "BrowserAttachBudgetExceededError";
|
|
783
883
|
};
|
|
884
|
+
const browserAttachErrors = {
|
|
885
|
+
maxFilesExceeded(files, maxFiles) {
|
|
886
|
+
return {
|
|
887
|
+
kind: "browser-attach-budget-exceeded",
|
|
888
|
+
budget: "maxFiles",
|
|
889
|
+
message: `browser parquet attach requires ${files} files, above maxFiles=${maxFiles}`
|
|
890
|
+
};
|
|
891
|
+
},
|
|
892
|
+
hintedBytesExceeded(hintedBytes, maxBytes) {
|
|
893
|
+
return {
|
|
894
|
+
kind: "browser-attach-budget-exceeded",
|
|
895
|
+
budget: "maxBytes",
|
|
896
|
+
message: `browser parquet attach requires ${hintedBytes} hinted bytes, above maxBytes=${maxBytes}`
|
|
897
|
+
};
|
|
898
|
+
},
|
|
899
|
+
plannedBytesExceeded(plannedBytes, maxBytes) {
|
|
900
|
+
return {
|
|
901
|
+
kind: "browser-attach-budget-exceeded",
|
|
902
|
+
budget: "maxBytes",
|
|
903
|
+
message: `browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${maxBytes}`
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
function isBrowserAttachError(value) {
|
|
908
|
+
return typeof value === "object" && value !== null && value.kind === "browser-attach-budget-exceeded" && typeof value.message === "string";
|
|
909
|
+
}
|
|
910
|
+
function browserAttachErrorToException(error) {
|
|
911
|
+
const exception = new BrowserAttachBudgetExceededError(error.message);
|
|
912
|
+
exception.browserAttachError = error;
|
|
913
|
+
return exception;
|
|
914
|
+
}
|
|
784
915
|
function fileName(table, index, provided) {
|
|
785
916
|
return provided ?? `${table}_${index}.parquet`;
|
|
786
917
|
}
|
|
@@ -909,11 +1040,11 @@ async function dropAttachedResources(db, conn, schema, tables, files) {
|
|
|
909
1040
|
if (files.length > 0) await db.dropFiles([...files]);
|
|
910
1041
|
}
|
|
911
1042
|
async function bootDuckDBWasm(options = {}) {
|
|
912
|
-
const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger } = await import("@duckdb/duckdb-wasm");
|
|
1043
|
+
const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger, LogLevel } = await import("@duckdb/duckdb-wasm");
|
|
913
1044
|
const bundle = await selectBundle(options.bundles ?? getJsDelivrBundles());
|
|
914
1045
|
const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }));
|
|
915
1046
|
const worker = new Worker(workerUrl);
|
|
916
|
-
const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(), worker);
|
|
1047
|
+
const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(LogLevel.WARNING), worker);
|
|
917
1048
|
try {
|
|
918
1049
|
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
919
1050
|
await db.open(rangeOnlyConfig(options.config));
|
|
@@ -941,7 +1072,7 @@ async function attachParquetTables(options) {
|
|
|
941
1072
|
await conn.query(readParquetViewSql(schema, table.table, names));
|
|
942
1073
|
}
|
|
943
1074
|
}
|
|
944
|
-
async function
|
|
1075
|
+
async function attachParquetUrlTablesResult(options) {
|
|
945
1076
|
const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached, attachMode = "http", trustSizeHint = true } = options;
|
|
946
1077
|
const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
|
|
947
1078
|
const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
|
|
@@ -958,52 +1089,57 @@ async function attachParquetUrlTables(options) {
|
|
|
958
1089
|
index: i
|
|
959
1090
|
});
|
|
960
1091
|
}
|
|
961
|
-
if (flat.length > fileBudget)
|
|
1092
|
+
if (flat.length > fileBudget) return err(browserAttachErrors.maxFilesExceeded(flat.length, fileBudget));
|
|
962
1093
|
const hintedBytes = flat.reduce((acc, item) => {
|
|
963
1094
|
const hint = sizeHintFromUrl(item.url);
|
|
964
1095
|
return hint === null ? acc : acc + hint;
|
|
965
1096
|
}, 0);
|
|
966
|
-
if (hintedBytes > byteBudget)
|
|
1097
|
+
if (hintedBytes > byteBudget) return err(browserAttachErrors.hintedBytesExceeded(hintedBytes, byteBudget));
|
|
967
1098
|
const tableFailures = /* @__PURE__ */ new Map();
|
|
968
1099
|
const budgetController = new AbortController();
|
|
969
1100
|
const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
|
|
970
1101
|
let plannedBytes = 0;
|
|
971
1102
|
const total = flat.length;
|
|
972
1103
|
const prepared = [];
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
try {
|
|
977
|
-
let bytes = null;
|
|
978
|
-
let body = null;
|
|
979
|
-
if (attachMode === "buffer") {
|
|
980
|
-
const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
|
|
981
|
-
if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
|
|
982
|
-
const buf = new Uint8Array(await res.arrayBuffer());
|
|
983
|
-
body = buf;
|
|
984
|
-
bytes = buf.byteLength;
|
|
985
|
-
} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
|
|
986
|
-
else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
|
|
987
|
-
plannedBytes += bytes;
|
|
988
|
-
if (plannedBytes > byteBudget) {
|
|
989
|
-
const err = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
|
|
990
|
-
budgetController.abort(err);
|
|
991
|
-
throw err;
|
|
992
|
-
}
|
|
1104
|
+
try {
|
|
1105
|
+
await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
|
|
1106
|
+
if (tableFailures.has(table)) return;
|
|
993
1107
|
effectiveSignal?.throwIfAborted();
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1108
|
+
try {
|
|
1109
|
+
let bytes = null;
|
|
1110
|
+
let body = null;
|
|
1111
|
+
if (attachMode === "buffer") {
|
|
1112
|
+
const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
|
|
1113
|
+
if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
|
|
1114
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
1115
|
+
body = buf;
|
|
1116
|
+
bytes = buf.byteLength;
|
|
1117
|
+
} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
|
|
1118
|
+
else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
|
|
1119
|
+
plannedBytes += bytes;
|
|
1120
|
+
if (plannedBytes > byteBudget) {
|
|
1121
|
+
const budgetErr = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
|
|
1122
|
+
budgetController.abort(budgetErr);
|
|
1123
|
+
throw budgetErr;
|
|
1124
|
+
}
|
|
1125
|
+
effectiveSignal?.throwIfAborted();
|
|
1126
|
+
prepared.push({
|
|
1127
|
+
table,
|
|
1128
|
+
url,
|
|
1129
|
+
index,
|
|
1130
|
+
name: attachFileName(attachId, table, index),
|
|
1131
|
+
bytes,
|
|
1132
|
+
body
|
|
1133
|
+
});
|
|
1134
|
+
} catch (err) {
|
|
1135
|
+
if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
|
|
1136
|
+
tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
|
|
1137
|
+
}
|
|
1138
|
+
});
|
|
1139
|
+
} catch (downloadErr) {
|
|
1140
|
+
if (downloadErr instanceof BrowserAttachBudgetExceededError && !signal?.aborted) return err(browserAttachErrors.plannedBytesExceeded(plannedBytes, byteBudget));
|
|
1141
|
+
throw downloadErr;
|
|
1142
|
+
}
|
|
1007
1143
|
const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
|
|
1008
1144
|
const attached = [];
|
|
1009
1145
|
const registeredFiles = [];
|
|
@@ -1036,7 +1172,7 @@ async function attachParquetUrlTables(options) {
|
|
|
1036
1172
|
}
|
|
1037
1173
|
if (tableFailures.size > 0) for (const [table, err] of tableFailures) console.warn(`[gscdump/engine-duckdb-wasm] dropped table "${table}" — ${err.message}`);
|
|
1038
1174
|
let detached = false;
|
|
1039
|
-
return {
|
|
1175
|
+
return ok({
|
|
1040
1176
|
version,
|
|
1041
1177
|
tables: attached,
|
|
1042
1178
|
schema,
|
|
@@ -1045,7 +1181,10 @@ async function attachParquetUrlTables(options) {
|
|
|
1045
1181
|
detached = true;
|
|
1046
1182
|
await dropAttachedResources(db, conn, schema, attached, registeredFiles);
|
|
1047
1183
|
}
|
|
1048
|
-
};
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
async function attachParquetUrlTables(options) {
|
|
1187
|
+
return unwrapResult(await attachParquetUrlTablesResult(options), browserAttachErrorToException);
|
|
1049
1188
|
}
|
|
1050
1189
|
function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
1051
1190
|
const { db, conn } = boot;
|
|
@@ -1147,4 +1286,4 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
|
1147
1286
|
}
|
|
1148
1287
|
};
|
|
1149
1288
|
}
|
|
1150
|
-
export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
|
1289
|
+
export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine-duckdb-wasm",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.26.1",
|
|
5
5
|
"description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -45,9 +45,9 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"drizzle-orm": "1.0.0-rc.3",
|
|
48
|
-
"@gscdump/engine": "0.
|
|
49
|
-
"@gscdump/sdk": "0.
|
|
50
|
-
"gscdump": "0.
|
|
48
|
+
"@gscdump/engine": "0.26.1",
|
|
49
|
+
"@gscdump/sdk": "0.26.1",
|
|
50
|
+
"gscdump": "0.26.1"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@duckdb/duckdb-wasm": "^1.32.0",
|