@gscdump/engine-duckdb-wasm 0.25.13 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.mjs +134 -71
- package/package.json +7 -7
package/dist/index.d.mts
CHANGED
|
@@ -4,6 +4,7 @@ import { AnyRelations, EmptyRelations, ExtractTablesWithRelations, Schema as Sch
|
|
|
4
4
|
import { DrizzleSchema as Schema, countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
|
|
5
5
|
import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
|
|
6
6
|
import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
|
|
7
|
+
import { Result } from "gscdump/result";
|
|
7
8
|
import { ComparisonMode, ResolveWindowOptions, ResolvedWindow, WindowPreset, resolveWindow } from "@gscdump/engine/period";
|
|
8
9
|
import { ArchetypeQuery } from "@gscdump/sdk";
|
|
9
10
|
import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles, DuckDBConfig } from "@duckdb/duckdb-wasm";
|
|
@@ -217,6 +218,14 @@ interface DuckDBWasmBootResult {
|
|
|
217
218
|
conn: AsyncDuckDBConnection;
|
|
218
219
|
}
|
|
219
220
|
interface BootDuckDBWasmOptions {
|
|
221
|
+
/**
|
|
222
|
+
* DuckDB-WASM logger. Defaults to a `ConsoleLogger` thresholded at
|
|
223
|
+
* `LogLevel.WARNING`, so real warnings/errors still surface but the per-query
|
|
224
|
+
* INFO events (START/OK/RUN) — which DuckDB's default `ConsoleLogger()` emits
|
|
225
|
+
* as raw objects, flooding the host console with dozens of lines per render —
|
|
226
|
+
* are dropped. Pass `new ConsoleLogger(LogLevel.DEBUG)` to see everything, or
|
|
227
|
+
* `new VoidLogger()` to silence it entirely.
|
|
228
|
+
*/
|
|
220
229
|
logger?: unknown;
|
|
221
230
|
/**
|
|
222
231
|
* Override the jsDelivr-hosted bundle map. Required in environments where
|
|
@@ -354,12 +363,40 @@ interface BrowserAnalysisRuntime {
|
|
|
354
363
|
declare class BrowserAttachBudgetExceededError extends Error {
|
|
355
364
|
name: string;
|
|
356
365
|
}
|
|
366
|
+
interface BrowserAttachError {
|
|
367
|
+
kind: 'browser-attach-budget-exceeded';
|
|
368
|
+
message: string;
|
|
369
|
+
/** Which budget tripped: the file count, the hinted byte plan, or the running byte plan. */
|
|
370
|
+
budget: 'maxFiles' | 'maxBytes';
|
|
371
|
+
}
|
|
372
|
+
declare const browserAttachErrors: {
|
|
373
|
+
readonly maxFilesExceeded: (files: number, maxFiles: number) => BrowserAttachError;
|
|
374
|
+
readonly hintedBytesExceeded: (hintedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
375
|
+
readonly plannedBytesExceeded: (plannedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
376
|
+
};
|
|
377
|
+
declare function isBrowserAttachError(value: unknown): value is BrowserAttachError;
|
|
357
378
|
declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
|
|
358
379
|
declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
|
|
380
|
+
/**
|
|
381
|
+
* Errors-as-values core for {@link attachParquetUrlTables}: returns a typed
|
|
382
|
+
* {@link BrowserAttachError} when the requested file set blows the local file-
|
|
383
|
+
* count / byte budget, so callers can branch (route the affected tables
|
|
384
|
+
* server-side) instead of catching an untyped throw. WASM/DuckDB/HTTP IO
|
|
385
|
+
* failures stay defects and propagate. `attachParquetUrlTables` is the thin
|
|
386
|
+
* throwing wrapper preserving the historical `BrowserAttachBudgetExceededError`.
|
|
387
|
+
*/
|
|
388
|
+
declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOptions): Promise<Result<AttachedTablesHandle, BrowserAttachError>>;
|
|
389
|
+
/**
|
|
390
|
+
* Attach browser parquet URL tables, throwing
|
|
391
|
+
* {@link BrowserAttachBudgetExceededError} when the requested set blows the
|
|
392
|
+
* file-count / byte budget. Thin throwing wrapper over
|
|
393
|
+
* {@link attachParquetUrlTablesResult}; existing call sites and their
|
|
394
|
+
* `instanceof BrowserAttachBudgetExceededError` checks keep holding.
|
|
395
|
+
*/
|
|
359
396
|
declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
|
|
360
397
|
declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, options?: {
|
|
361
398
|
schema?: string;
|
|
362
399
|
version?: number | string;
|
|
363
400
|
attachedTables?: readonly string[];
|
|
364
401
|
}): BrowserAnalysisRuntime;
|
|
365
|
-
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
|
402
|
+
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
package/dist/index.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { runAnalyzerFromSource } from "@gscdump/engine/analyzer";
|
|
|
9
9
|
import { pgResolverAdapter } from "@gscdump/engine/resolver";
|
|
10
10
|
import { createAttachedTableSource } from "@gscdump/engine/source";
|
|
11
11
|
import { sqlEscape } from "@gscdump/engine/sql";
|
|
12
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
12
13
|
import { resolveWindow } from "@gscdump/engine/period";
|
|
13
14
|
const METRIC_SQL = {
|
|
14
15
|
clicks: "SUM(clicks)",
|
|
@@ -166,7 +167,8 @@ function compileArchetypeSql(query) {
|
|
|
166
167
|
}
|
|
167
168
|
const col = DIM_COLUMN[query.dimension];
|
|
168
169
|
const facet = facetPredicate(query);
|
|
169
|
-
|
|
170
|
+
const totalCol = query.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
|
|
171
|
+
let sql = `SELECT ${col} AS ${query.dimension}, ${metricSelectList(query.metrics)}${totalCol} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col} ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
|
|
170
172
|
const params = [
|
|
171
173
|
...where.params,
|
|
172
174
|
...facet.params,
|
|
@@ -277,7 +279,7 @@ async function createClient(db, conn) {
|
|
|
277
279
|
try {
|
|
278
280
|
return arrowToRows(await stmt.query(...params));
|
|
279
281
|
} finally {
|
|
280
|
-
stmt.close();
|
|
282
|
+
await stmt.close();
|
|
281
283
|
}
|
|
282
284
|
},
|
|
283
285
|
async close() {
|
|
@@ -489,12 +491,27 @@ function isOpfsAccessHandleConflict(err) {
|
|
|
489
491
|
const msg = err instanceof Error ? err.message : String(err);
|
|
490
492
|
return /createSyncAccessHandle|Access Handle/i.test(msg);
|
|
491
493
|
}
|
|
492
|
-
function opfsFileName(table,
|
|
493
|
-
|
|
494
|
-
|
|
494
|
+
function opfsFileName(table, hashSlug, index) {
|
|
495
|
+
return hashSlug ? `${OPFS_PREFIX}${table}_${hashSlug}.parquet` : `${OPFS_PREFIX}${table}_${index}.parquet`;
|
|
496
|
+
}
|
|
497
|
+
function escapeRegExp(s) {
|
|
498
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
499
|
+
}
|
|
500
|
+
function tableEntryMatcher(table) {
|
|
501
|
+
return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?:[0-9a-f]{16}|\\d+(?:_[0-9a-f]{16})?)\\.parquet$`);
|
|
495
502
|
}
|
|
496
|
-
function
|
|
497
|
-
|
|
503
|
+
async function sweepStaleEntries(root, registry, expectedByTable) {
|
|
504
|
+
const dir = root;
|
|
505
|
+
if (!dir.keys) return;
|
|
506
|
+
const matchers = [...expectedByTable].map(([table, expected]) => ({
|
|
507
|
+
expected,
|
|
508
|
+
re: tableEntryMatcher(table)
|
|
509
|
+
}));
|
|
510
|
+
for await (const name of dir.keys()) {
|
|
511
|
+
const m = matchers.find((m) => m.re.test(name));
|
|
512
|
+
if (!m || m.expected.has(name) || registry.refs(name) > 0) continue;
|
|
513
|
+
await root.removeEntry(name).catch(() => {});
|
|
514
|
+
}
|
|
498
515
|
}
|
|
499
516
|
async function requestPersistentStorage() {
|
|
500
517
|
const storage = globalThis.navigator?.storage;
|
|
@@ -523,7 +540,7 @@ async function getOpfsRoot() {
|
|
|
523
540
|
if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
|
|
524
541
|
return dir.call(globalThis.navigator.storage);
|
|
525
542
|
}
|
|
526
|
-
async function materialiseFile(root, name,
|
|
543
|
+
async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
|
|
527
544
|
signal?.throwIfAborted();
|
|
528
545
|
let handle;
|
|
529
546
|
try {
|
|
@@ -533,12 +550,6 @@ async function materialiseFile(root, name, staleSweepPrefix, file, fetchImpl, fe
|
|
|
533
550
|
outcome: "cache-hit"
|
|
534
551
|
};
|
|
535
552
|
} catch {}
|
|
536
|
-
const dir = root;
|
|
537
|
-
if (dir.keys) for await (const existing of dir.keys()) {
|
|
538
|
-
if (existing === name || !existing.startsWith(staleSweepPrefix) || !existing.endsWith(".parquet")) continue;
|
|
539
|
-
const next = existing.charAt(staleSweepPrefix.length);
|
|
540
|
-
if (next === "." || next === "_") await root.removeEntry(existing).catch(() => {});
|
|
541
|
-
}
|
|
542
553
|
signal?.throwIfAborted();
|
|
543
554
|
const resp = await fetchImpl(file.url, {
|
|
544
555
|
...fetchInit,
|
|
@@ -573,7 +584,7 @@ function readParquetViewSql$1(schema, table, files) {
|
|
|
573
584
|
function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
|
|
574
585
|
const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
|
|
575
586
|
if (lakeFiles.length === 0) return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlay}`;
|
|
576
|
-
return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(lakeFiles)} UNION ALL BY NAME ${overlay} WHERE
|
|
587
|
+
return `CREATE OR REPLACE VIEW ${schema}.${table} AS WITH lake AS MATERIALIZED (${lakeSelect(lakeFiles)}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlay}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
|
|
577
588
|
}
|
|
578
589
|
async function runWithConcurrency$1(items, concurrency, fn) {
|
|
579
590
|
let next = 0;
|
|
@@ -596,24 +607,39 @@ async function attachOpfsParquetTables(options) {
|
|
|
596
607
|
await requestPersistentStorage();
|
|
597
608
|
const root = await getOpfsRoot();
|
|
598
609
|
const flat = [];
|
|
610
|
+
const expectedByTable = /* @__PURE__ */ new Map();
|
|
599
611
|
for (const t of tables) {
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
file
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
612
|
+
const expected = /* @__PURE__ */ new Set();
|
|
613
|
+
for (let i = 0; i < t.files.length; i++) {
|
|
614
|
+
const file = t.files[i];
|
|
615
|
+
const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
|
|
616
|
+
const name = opfsFileName(t.table, slug, i);
|
|
617
|
+
flat.push({
|
|
618
|
+
table: t.table,
|
|
619
|
+
file,
|
|
620
|
+
name
|
|
621
|
+
});
|
|
622
|
+
expected.add(name);
|
|
623
|
+
}
|
|
624
|
+
if (t.overlay) {
|
|
625
|
+
const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
|
|
626
|
+
const name = opfsFileName(t.table, slug, t.files.length);
|
|
627
|
+
flat.push({
|
|
628
|
+
table: t.table,
|
|
629
|
+
file: t.overlay,
|
|
630
|
+
name,
|
|
631
|
+
overlay: true
|
|
632
|
+
});
|
|
633
|
+
expected.add(name);
|
|
634
|
+
}
|
|
635
|
+
expectedByTable.set(t.table, expected);
|
|
611
636
|
}
|
|
612
637
|
const total = flat.length;
|
|
613
638
|
const overlayNames = /* @__PURE__ */ new Map();
|
|
614
639
|
const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
|
|
615
640
|
const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
|
|
616
641
|
const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
|
|
642
|
+
await sweepStaleEntries(root, registry, expectedByTable).catch(() => {});
|
|
617
643
|
const tableFiles = /* @__PURE__ */ new Map();
|
|
618
644
|
const degraded = /* @__PURE__ */ new Set();
|
|
619
645
|
const acquiredNames = /* @__PURE__ */ new Set();
|
|
@@ -626,12 +652,10 @@ async function attachOpfsParquetTables(options) {
|
|
|
626
652
|
const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
|
|
627
653
|
if (degraded.has(item.table)) return;
|
|
628
654
|
signal?.throwIfAborted();
|
|
629
|
-
const
|
|
630
|
-
const name = opfsFileName(item.table, item.fileIndex, hashSlug);
|
|
631
|
-
const sweepPrefix = opfsFileNamePrefix(item.table, item.fileIndex);
|
|
655
|
+
const name = item.name;
|
|
632
656
|
let result;
|
|
633
657
|
try {
|
|
634
|
-
result = await materialiseFile(root, name,
|
|
658
|
+
result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
|
|
635
659
|
} catch (err) {
|
|
636
660
|
if (isAbortError$1(err)) throw err;
|
|
637
661
|
if (isQuotaError(err)) {
|
|
@@ -758,6 +782,37 @@ let nextAttachId = 0;
|
|
|
758
782
|
var BrowserAttachBudgetExceededError = class extends Error {
|
|
759
783
|
name = "BrowserAttachBudgetExceededError";
|
|
760
784
|
};
|
|
785
|
+
const browserAttachErrors = {
|
|
786
|
+
maxFilesExceeded(files, maxFiles) {
|
|
787
|
+
return {
|
|
788
|
+
kind: "browser-attach-budget-exceeded",
|
|
789
|
+
budget: "maxFiles",
|
|
790
|
+
message: `browser parquet attach requires ${files} files, above maxFiles=${maxFiles}`
|
|
791
|
+
};
|
|
792
|
+
},
|
|
793
|
+
hintedBytesExceeded(hintedBytes, maxBytes) {
|
|
794
|
+
return {
|
|
795
|
+
kind: "browser-attach-budget-exceeded",
|
|
796
|
+
budget: "maxBytes",
|
|
797
|
+
message: `browser parquet attach requires ${hintedBytes} hinted bytes, above maxBytes=${maxBytes}`
|
|
798
|
+
};
|
|
799
|
+
},
|
|
800
|
+
plannedBytesExceeded(plannedBytes, maxBytes) {
|
|
801
|
+
return {
|
|
802
|
+
kind: "browser-attach-budget-exceeded",
|
|
803
|
+
budget: "maxBytes",
|
|
804
|
+
message: `browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${maxBytes}`
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
};
|
|
808
|
+
function isBrowserAttachError(value) {
|
|
809
|
+
return typeof value === "object" && value !== null && value.kind === "browser-attach-budget-exceeded" && typeof value.message === "string";
|
|
810
|
+
}
|
|
811
|
+
function browserAttachErrorToException(error) {
|
|
812
|
+
const exception = new BrowserAttachBudgetExceededError(error.message);
|
|
813
|
+
exception.browserAttachError = error;
|
|
814
|
+
return exception;
|
|
815
|
+
}
|
|
761
816
|
function fileName(table, index, provided) {
|
|
762
817
|
return provided ?? `${table}_${index}.parquet`;
|
|
763
818
|
}
|
|
@@ -886,11 +941,11 @@ async function dropAttachedResources(db, conn, schema, tables, files) {
|
|
|
886
941
|
if (files.length > 0) await db.dropFiles([...files]);
|
|
887
942
|
}
|
|
888
943
|
async function bootDuckDBWasm(options = {}) {
|
|
889
|
-
const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger } = await import("@duckdb/duckdb-wasm");
|
|
944
|
+
const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger, LogLevel } = await import("@duckdb/duckdb-wasm");
|
|
890
945
|
const bundle = await selectBundle(options.bundles ?? getJsDelivrBundles());
|
|
891
946
|
const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }));
|
|
892
947
|
const worker = new Worker(workerUrl);
|
|
893
|
-
const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(), worker);
|
|
948
|
+
const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(LogLevel.WARNING), worker);
|
|
894
949
|
try {
|
|
895
950
|
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
896
951
|
await db.open(rangeOnlyConfig(options.config));
|
|
@@ -918,7 +973,7 @@ async function attachParquetTables(options) {
|
|
|
918
973
|
await conn.query(readParquetViewSql(schema, table.table, names));
|
|
919
974
|
}
|
|
920
975
|
}
|
|
921
|
-
async function
|
|
976
|
+
async function attachParquetUrlTablesResult(options) {
|
|
922
977
|
const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached, attachMode = "http", trustSizeHint = true } = options;
|
|
923
978
|
const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
|
|
924
979
|
const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
|
|
@@ -935,52 +990,57 @@ async function attachParquetUrlTables(options) {
|
|
|
935
990
|
index: i
|
|
936
991
|
});
|
|
937
992
|
}
|
|
938
|
-
if (flat.length > fileBudget)
|
|
993
|
+
if (flat.length > fileBudget) return err(browserAttachErrors.maxFilesExceeded(flat.length, fileBudget));
|
|
939
994
|
const hintedBytes = flat.reduce((acc, item) => {
|
|
940
995
|
const hint = sizeHintFromUrl(item.url);
|
|
941
996
|
return hint === null ? acc : acc + hint;
|
|
942
997
|
}, 0);
|
|
943
|
-
if (hintedBytes > byteBudget)
|
|
998
|
+
if (hintedBytes > byteBudget) return err(browserAttachErrors.hintedBytesExceeded(hintedBytes, byteBudget));
|
|
944
999
|
const tableFailures = /* @__PURE__ */ new Map();
|
|
945
1000
|
const budgetController = new AbortController();
|
|
946
1001
|
const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
|
|
947
1002
|
let plannedBytes = 0;
|
|
948
1003
|
const total = flat.length;
|
|
949
1004
|
const prepared = [];
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
try {
|
|
954
|
-
let bytes = null;
|
|
955
|
-
let body = null;
|
|
956
|
-
if (attachMode === "buffer") {
|
|
957
|
-
const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
|
|
958
|
-
if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
|
|
959
|
-
const buf = new Uint8Array(await res.arrayBuffer());
|
|
960
|
-
body = buf;
|
|
961
|
-
bytes = buf.byteLength;
|
|
962
|
-
} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
|
|
963
|
-
else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
|
|
964
|
-
plannedBytes += bytes;
|
|
965
|
-
if (plannedBytes > byteBudget) {
|
|
966
|
-
const err = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
|
|
967
|
-
budgetController.abort(err);
|
|
968
|
-
throw err;
|
|
969
|
-
}
|
|
1005
|
+
try {
|
|
1006
|
+
await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
|
|
1007
|
+
if (tableFailures.has(table)) return;
|
|
970
1008
|
effectiveSignal?.throwIfAborted();
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
1009
|
+
try {
|
|
1010
|
+
let bytes = null;
|
|
1011
|
+
let body = null;
|
|
1012
|
+
if (attachMode === "buffer") {
|
|
1013
|
+
const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
|
|
1014
|
+
if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
|
|
1015
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
1016
|
+
body = buf;
|
|
1017
|
+
bytes = buf.byteLength;
|
|
1018
|
+
} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
|
|
1019
|
+
else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
|
|
1020
|
+
plannedBytes += bytes;
|
|
1021
|
+
if (plannedBytes > byteBudget) {
|
|
1022
|
+
const budgetErr = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
|
|
1023
|
+
budgetController.abort(budgetErr);
|
|
1024
|
+
throw budgetErr;
|
|
1025
|
+
}
|
|
1026
|
+
effectiveSignal?.throwIfAborted();
|
|
1027
|
+
prepared.push({
|
|
1028
|
+
table,
|
|
1029
|
+
url,
|
|
1030
|
+
index,
|
|
1031
|
+
name: attachFileName(attachId, table, index),
|
|
1032
|
+
bytes,
|
|
1033
|
+
body
|
|
1034
|
+
});
|
|
1035
|
+
} catch (err) {
|
|
1036
|
+
if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
|
|
1037
|
+
tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
|
|
1038
|
+
}
|
|
1039
|
+
});
|
|
1040
|
+
} catch (downloadErr) {
|
|
1041
|
+
if (downloadErr instanceof BrowserAttachBudgetExceededError && !signal?.aborted) return err(browserAttachErrors.plannedBytesExceeded(plannedBytes, byteBudget));
|
|
1042
|
+
throw downloadErr;
|
|
1043
|
+
}
|
|
984
1044
|
const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
|
|
985
1045
|
const attached = [];
|
|
986
1046
|
const registeredFiles = [];
|
|
@@ -1013,7 +1073,7 @@ async function attachParquetUrlTables(options) {
|
|
|
1013
1073
|
}
|
|
1014
1074
|
if (tableFailures.size > 0) for (const [table, err] of tableFailures) console.warn(`[gscdump/engine-duckdb-wasm] dropped table "${table}" — ${err.message}`);
|
|
1015
1075
|
let detached = false;
|
|
1016
|
-
return {
|
|
1076
|
+
return ok({
|
|
1017
1077
|
version,
|
|
1018
1078
|
tables: attached,
|
|
1019
1079
|
schema,
|
|
@@ -1022,7 +1082,10 @@ async function attachParquetUrlTables(options) {
|
|
|
1022
1082
|
detached = true;
|
|
1023
1083
|
await dropAttachedResources(db, conn, schema, attached, registeredFiles);
|
|
1024
1084
|
}
|
|
1025
|
-
};
|
|
1085
|
+
});
|
|
1086
|
+
}
|
|
1087
|
+
async function attachParquetUrlTables(options) {
|
|
1088
|
+
return unwrapResult(await attachParquetUrlTablesResult(options), browserAttachErrorToException);
|
|
1026
1089
|
}
|
|
1027
1090
|
function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
1028
1091
|
const { db, conn } = boot;
|
|
@@ -1124,4 +1187,4 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
|
1124
1187
|
}
|
|
1125
1188
|
};
|
|
1126
1189
|
}
|
|
1127
|
-
export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
|
1190
|
+
export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine-duckdb-wasm",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.26.0",
|
|
5
5
|
"description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -45,16 +45,16 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"drizzle-orm": "1.0.0-rc.3",
|
|
48
|
-
"@gscdump/engine": "0.
|
|
49
|
-
"gscdump": "0.
|
|
50
|
-
"
|
|
48
|
+
"@gscdump/engine": "0.26.0",
|
|
49
|
+
"@gscdump/sdk": "0.26.0",
|
|
50
|
+
"gscdump": "0.26.0"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@duckdb/duckdb-wasm": "^1.32.0",
|
|
54
|
-
"@vitest/browser": "^4.1.
|
|
55
|
-
"@vitest/browser-playwright": "^4.1.
|
|
54
|
+
"@vitest/browser": "^4.1.8",
|
|
55
|
+
"@vitest/browser-playwright": "^4.1.8",
|
|
56
56
|
"playwright": "^1.60.0",
|
|
57
|
-
"vitest": "^4.1.
|
|
57
|
+
"vitest": "^4.1.8"
|
|
58
58
|
},
|
|
59
59
|
"scripts": {
|
|
60
60
|
"build": "obuild",
|