@gscdump/engine-duckdb-wasm 0.32.10 → 0.32.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +24 -1
- package/dist/index.mjs +226 -80
- package/package.json +4 -4
package/dist/index.d.mts
CHANGED
|
@@ -136,6 +136,12 @@ interface AttachOpfsTablesOptions {
|
|
|
136
136
|
version?: string;
|
|
137
137
|
/** Ticks once per file as it lands in OPFS + registers. UI progress. */
|
|
138
138
|
onFileProgress?: (info: OpfsFileProgress) => void;
|
|
139
|
+
/**
|
|
140
|
+
* Low-volume diagnostics for the OPFS attach waterfall. This is intentionally
|
|
141
|
+
* phase-level rather than a logger so hosts can join it with their own route
|
|
142
|
+
* timings without parsing console text.
|
|
143
|
+
*/
|
|
144
|
+
onTiming?: (info: OpfsAttachTiming) => void;
|
|
139
145
|
/**
|
|
140
146
|
* Serializer for the operations that mutate the DuckDB-WASM virtual
|
|
141
147
|
* filesystem / catalog (`registerFileHandle`, `dropFile`, `CREATE`/`DROP
|
|
@@ -178,6 +184,23 @@ interface OpfsFileProgress {
|
|
|
178
184
|
bytes: number;
|
|
179
185
|
/** `'cache-hit'` — already in OPFS, verified; `'downloaded'` — fetched. */
|
|
180
186
|
outcome: 'cache-hit' | 'downloaded';
|
|
187
|
+
/** Time spent probing/downloading/writing this file before DuckDB registration. */
|
|
188
|
+
materialiseMs?: number;
|
|
189
|
+
/** Time spent inside DuckDB's file-registration mutation lock for this file. */
|
|
190
|
+
registerMs?: number;
|
|
191
|
+
/** End-to-end file phase time, materialise + register + callback overhead. */
|
|
192
|
+
totalMs?: number;
|
|
193
|
+
}
|
|
194
|
+
type OpfsAttachTimingStage = 'persist' | 'root' | 'plan' | 'duckdb-import' | 'sweep' | 'materialise' | 'register' | 'downloads' | 'view' | 'recovery' | 'total';
|
|
195
|
+
interface OpfsAttachTiming {
|
|
196
|
+
stage: OpfsAttachTimingStage;
|
|
197
|
+
durationMs: number;
|
|
198
|
+
table?: string;
|
|
199
|
+
file?: string;
|
|
200
|
+
files?: number;
|
|
201
|
+
tables?: number;
|
|
202
|
+
bytes?: number;
|
|
203
|
+
outcome?: 'cache-hit' | 'downloaded';
|
|
181
204
|
}
|
|
182
205
|
/** Handle returned from {@link attachOpfsParquetTables}. */
|
|
183
206
|
interface OpfsAttachedHandle {
|
|
@@ -484,4 +507,4 @@ declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, option
|
|
|
484
507
|
version?: number | string;
|
|
485
508
|
attachedTables?: readonly string[];
|
|
486
509
|
}): BrowserAnalysisRuntime;
|
|
487
|
-
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
|
510
|
+
export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachTiming, type OpfsAttachTimingStage, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
|
package/dist/index.mjs
CHANGED
|
@@ -525,13 +525,26 @@ function createOpfsHandleRegistry(backend) {
|
|
|
525
525
|
}
|
|
526
526
|
}
|
|
527
527
|
const viewRefs = /* @__PURE__ */ new Map();
|
|
528
|
-
function acquireView(key) {
|
|
529
|
-
|
|
528
|
+
function acquireView(key, signature) {
|
|
529
|
+
const existing = viewRefs.get(key);
|
|
530
|
+
if (existing) {
|
|
531
|
+
if (existing.signature !== signature) throw new Error(`view ${key} is already attached with a different signature`);
|
|
532
|
+
existing.refs++;
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
viewRefs.set(key, {
|
|
536
|
+
refs: 1,
|
|
537
|
+
signature
|
|
538
|
+
});
|
|
530
539
|
}
|
|
531
540
|
async function releaseView(key, drop) {
|
|
532
|
-
const
|
|
541
|
+
const existing = viewRefs.get(key);
|
|
542
|
+
const next = (existing?.refs ?? 0) - 1;
|
|
533
543
|
if (next > 0) {
|
|
534
|
-
viewRefs.set(key,
|
|
544
|
+
viewRefs.set(key, {
|
|
545
|
+
refs: next,
|
|
546
|
+
signature: existing?.signature
|
|
547
|
+
});
|
|
535
548
|
return;
|
|
536
549
|
}
|
|
537
550
|
viewRefs.delete(key);
|
|
@@ -544,7 +557,8 @@ function createOpfsHandleRegistry(backend) {
|
|
|
544
557
|
refs: (name) => entries.get(name)?.refs ?? 0,
|
|
545
558
|
acquireView,
|
|
546
559
|
releaseView,
|
|
547
|
-
viewRefs: (key) => viewRefs.get(key) ?? 0
|
|
560
|
+
viewRefs: (key) => viewRefs.get(key)?.refs ?? 0,
|
|
561
|
+
viewSignature: (key) => viewRefs.get(key)?.signature
|
|
548
562
|
};
|
|
549
563
|
}
|
|
550
564
|
function overlayViewBody(args) {
|
|
@@ -567,6 +581,29 @@ var OpfsQuotaExceededError = class extends Error {
|
|
|
567
581
|
const DEFAULT_CONCURRENCY = 2;
|
|
568
582
|
const OPFS_PREFIX = "gscdump-snapshot__";
|
|
569
583
|
const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
|
|
584
|
+
const MAX_CONTENT_HASH_SLUG_CACHE = 4096;
|
|
585
|
+
const contentHashSlugCache = /* @__PURE__ */ new Map();
|
|
586
|
+
const IDENT_RE = /^[A-Z_][\w$]*$/i;
|
|
587
|
+
function nowMs() {
|
|
588
|
+
return globalThis.performance?.now?.() ?? Date.now();
|
|
589
|
+
}
|
|
590
|
+
function emitTiming(onTiming, info) {
|
|
591
|
+
try {
|
|
592
|
+
onTiming?.(info);
|
|
593
|
+
} catch {}
|
|
594
|
+
}
|
|
595
|
+
async function timed(onTiming, stage, meta, fn) {
|
|
596
|
+
const started = nowMs();
|
|
597
|
+
try {
|
|
598
|
+
return await fn();
|
|
599
|
+
} finally {
|
|
600
|
+
emitTiming(onTiming, {
|
|
601
|
+
...meta,
|
|
602
|
+
stage,
|
|
603
|
+
durationMs: nowMs() - started
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
}
|
|
570
607
|
const dbRegistries = /* @__PURE__ */ new WeakMap();
|
|
571
608
|
function getOpfsRegistry(db, protocol) {
|
|
572
609
|
let registry = dbRegistries.get(db);
|
|
@@ -581,6 +618,23 @@ function getOpfsRegistry(db, protocol) {
|
|
|
581
618
|
}
|
|
582
619
|
return registry;
|
|
583
620
|
}
|
|
621
|
+
const dbBufferRegistries = /* @__PURE__ */ new WeakMap();
|
|
622
|
+
function getBufferRegistry(db) {
|
|
623
|
+
let registry = dbBufferRegistries.get(db);
|
|
624
|
+
if (!registry) {
|
|
625
|
+
registry = createOpfsHandleRegistry({
|
|
626
|
+
register: (name, handle) => db.registerFileBuffer(name, handle),
|
|
627
|
+
drop: async (name) => {
|
|
628
|
+
await db.dropFile(name);
|
|
629
|
+
}
|
|
630
|
+
});
|
|
631
|
+
dbBufferRegistries.set(db, registry);
|
|
632
|
+
}
|
|
633
|
+
return registry;
|
|
634
|
+
}
|
|
635
|
+
function assertSqlIdentifier(kind, value) {
|
|
636
|
+
if (!IDENT_RE.test(value)) throw new TypeError(`[engine-duckdb-wasm/opfs] invalid ${kind} identifier ${JSON.stringify(value)}`);
|
|
637
|
+
}
|
|
584
638
|
function isQuotaError(err) {
|
|
585
639
|
if (typeof err !== "object" || err === null) return false;
|
|
586
640
|
return err.name === "QuotaExceededError" || err.code === 22;
|
|
@@ -636,6 +690,20 @@ async function estimateOpfsStorage() {
|
|
|
636
690
|
} : {};
|
|
637
691
|
}
|
|
638
692
|
async function contentHashSlug(contentHash) {
|
|
693
|
+
const cached = contentHashSlugCache.get(contentHash);
|
|
694
|
+
if (cached) return cached;
|
|
695
|
+
if (contentHashSlugCache.size >= MAX_CONTENT_HASH_SLUG_CACHE) {
|
|
696
|
+
const oldest = contentHashSlugCache.keys().next().value;
|
|
697
|
+
if (oldest !== void 0) contentHashSlugCache.delete(oldest);
|
|
698
|
+
}
|
|
699
|
+
const promise = deriveContentHashSlug(contentHash).catch((err) => {
|
|
700
|
+
contentHashSlugCache.delete(contentHash);
|
|
701
|
+
throw err;
|
|
702
|
+
});
|
|
703
|
+
contentHashSlugCache.set(contentHash, promise);
|
|
704
|
+
return promise;
|
|
705
|
+
}
|
|
706
|
+
async function deriveContentHashSlug(contentHash) {
|
|
639
707
|
const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contentHash));
|
|
640
708
|
const bytes = new Uint8Array(digest);
|
|
641
709
|
let hex = "";
|
|
@@ -707,6 +775,28 @@ function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
|
|
|
707
775
|
materializeLake: true
|
|
708
776
|
})}`;
|
|
709
777
|
}
|
|
778
|
+
function viewKey(schema, table) {
|
|
779
|
+
return `${schema}.${table}`;
|
|
780
|
+
}
|
|
781
|
+
function viewSignature(lakeFiles, overlayFile) {
|
|
782
|
+
return JSON.stringify({
|
|
783
|
+
lake: [...lakeFiles].sort(),
|
|
784
|
+
overlay: overlayFile ?? null
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
async function ensureView(registry, conn, schema, table, signature, sql) {
|
|
788
|
+
const key = viewKey(schema, table);
|
|
789
|
+
if (registry.viewSignature(key) === signature) {
|
|
790
|
+
registry.acquireView(key, signature);
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
if (registry.viewRefs(key) > 0) throw new Error(`[engine-duckdb-wasm/opfs] view ${key} is already attached with a different fileset`);
|
|
794
|
+
await conn.query(sql);
|
|
795
|
+
registry.acquireView(key, signature);
|
|
796
|
+
}
|
|
797
|
+
async function recoveredBufferName(table, file, index) {
|
|
798
|
+
return `${RECOVER_BUFFER_PREFIX}${table}_${file.contentHash ? await contentHashSlug(file.contentHash) : String(index)}.parquet`;
|
|
799
|
+
}
|
|
710
800
|
async function runWithConcurrency$1(items, concurrency, fn) {
|
|
711
801
|
let next = 0;
|
|
712
802
|
let failed;
|
|
@@ -721,46 +811,56 @@ async function runWithConcurrency$1(items, concurrency, fn) {
|
|
|
721
811
|
}
|
|
722
812
|
}
|
|
723
813
|
}
|
|
724
|
-
await Promise.
|
|
814
|
+
const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((r) => r.status === "rejected");
|
|
815
|
+
if (rejected) throw rejected.reason;
|
|
725
816
|
}
|
|
726
817
|
async function attachOpfsParquetTables(options) {
|
|
727
|
-
const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, withDb = (fn) => fn(), recoverContention = true } = options;
|
|
728
|
-
|
|
729
|
-
const
|
|
818
|
+
const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, onTiming, withDb = (fn) => fn(), recoverContention = true } = options;
|
|
819
|
+
assertSqlIdentifier("schema", schema);
|
|
820
|
+
for (const table of tables) assertSqlIdentifier("table", table.table);
|
|
821
|
+
const totalStarted = nowMs();
|
|
822
|
+
await timed(onTiming, "persist", {}, requestPersistentStorage);
|
|
823
|
+
const root = await timed(onTiming, "root", {}, getOpfsRoot);
|
|
730
824
|
const flat = [];
|
|
731
825
|
const expectedByTable = /* @__PURE__ */ new Map();
|
|
732
|
-
|
|
733
|
-
const
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
826
|
+
await timed(onTiming, "plan", { tables: tables.length }, async () => {
|
|
827
|
+
for (const t of tables) {
|
|
828
|
+
const expected = /* @__PURE__ */ new Set();
|
|
829
|
+
for (let i = 0; i < t.files.length; i++) {
|
|
830
|
+
const file = t.files[i];
|
|
831
|
+
const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
|
|
832
|
+
const name = opfsFileName(t.table, slug, i);
|
|
833
|
+
flat.push({
|
|
834
|
+
table: t.table,
|
|
835
|
+
file,
|
|
836
|
+
name
|
|
837
|
+
});
|
|
838
|
+
expected.add(name);
|
|
839
|
+
}
|
|
840
|
+
if (t.overlay) {
|
|
841
|
+
const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
|
|
842
|
+
const name = opfsFileName(t.table, slug, t.files.length);
|
|
843
|
+
flat.push({
|
|
844
|
+
table: t.table,
|
|
845
|
+
file: t.overlay,
|
|
846
|
+
name,
|
|
847
|
+
overlay: true
|
|
848
|
+
});
|
|
849
|
+
expected.add(name);
|
|
850
|
+
}
|
|
851
|
+
expectedByTable.set(t.table, expected);
|
|
755
852
|
}
|
|
756
|
-
|
|
757
|
-
}
|
|
853
|
+
});
|
|
758
854
|
const total = flat.length;
|
|
759
855
|
const overlayNames = /* @__PURE__ */ new Map();
|
|
760
856
|
const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
|
|
761
|
-
const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
|
|
857
|
+
const { DuckDBDataProtocol } = await timed(onTiming, "duckdb-import", {}, () => import("@duckdb/duckdb-wasm"));
|
|
762
858
|
const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
|
|
763
|
-
|
|
859
|
+
const bufferRegistry = getBufferRegistry(db);
|
|
860
|
+
await timed(onTiming, "sweep", {
|
|
861
|
+
files: total,
|
|
862
|
+
tables: tables.length
|
|
863
|
+
}, () => sweepStaleEntries(root, registry, expectedByTable)).catch(() => {});
|
|
764
864
|
const tableFiles = /* @__PURE__ */ new Map();
|
|
765
865
|
const degraded = /* @__PURE__ */ new Set();
|
|
766
866
|
const degradeReason = /* @__PURE__ */ new Map();
|
|
@@ -776,10 +876,23 @@ async function attachOpfsParquetTables(options) {
|
|
|
776
876
|
const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
|
|
777
877
|
if (degraded.has(item.table)) return;
|
|
778
878
|
signal?.throwIfAborted();
|
|
879
|
+
const fileStarted = nowMs();
|
|
779
880
|
const name = item.name;
|
|
780
881
|
let result;
|
|
882
|
+
let materialiseMs = 0;
|
|
883
|
+
let registerMs = 0;
|
|
781
884
|
try {
|
|
885
|
+
const materialiseStarted = nowMs();
|
|
782
886
|
result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
|
|
887
|
+
materialiseMs = nowMs() - materialiseStarted;
|
|
888
|
+
emitTiming(onTiming, {
|
|
889
|
+
stage: "materialise",
|
|
890
|
+
durationMs: materialiseMs,
|
|
891
|
+
table: item.table,
|
|
892
|
+
file: name,
|
|
893
|
+
bytes: item.file.bytes,
|
|
894
|
+
outcome: result.outcome
|
|
895
|
+
});
|
|
783
896
|
} catch (err) {
|
|
784
897
|
if (isAbortError$1(err)) throw err;
|
|
785
898
|
if (isQuotaError(err)) {
|
|
@@ -795,7 +908,17 @@ async function attachOpfsParquetTables(options) {
|
|
|
795
908
|
throw err;
|
|
796
909
|
}
|
|
797
910
|
try {
|
|
911
|
+
const registerStarted = nowMs();
|
|
798
912
|
await withDb(() => registry.acquire(name, () => result.handle));
|
|
913
|
+
registerMs = nowMs() - registerStarted;
|
|
914
|
+
emitTiming(onTiming, {
|
|
915
|
+
stage: "register",
|
|
916
|
+
durationMs: registerMs,
|
|
917
|
+
table: item.table,
|
|
918
|
+
file: name,
|
|
919
|
+
bytes: item.file.bytes,
|
|
920
|
+
outcome: result.outcome
|
|
921
|
+
});
|
|
799
922
|
acquiredNames.add(name);
|
|
800
923
|
} catch (err) {
|
|
801
924
|
if (isAbortError$1(err)) throw err;
|
|
@@ -820,11 +943,17 @@ async function attachOpfsParquetTables(options) {
|
|
|
820
943
|
index,
|
|
821
944
|
total,
|
|
822
945
|
bytes: item.file.bytes,
|
|
823
|
-
outcome: result.outcome
|
|
946
|
+
outcome: result.outcome,
|
|
947
|
+
materialiseMs,
|
|
948
|
+
registerMs,
|
|
949
|
+
totalMs: nowMs() - fileStarted
|
|
824
950
|
});
|
|
825
951
|
});
|
|
826
952
|
try {
|
|
827
|
-
await
|
|
953
|
+
await timed(onTiming, "downloads", {
|
|
954
|
+
files: total,
|
|
955
|
+
tables: tables.length
|
|
956
|
+
}, runDownloads);
|
|
828
957
|
} catch (err) {
|
|
829
958
|
await withDb(() => registry.release([...acquiredNames])).catch(() => {});
|
|
830
959
|
throw err;
|
|
@@ -844,17 +973,21 @@ async function attachOpfsParquetTables(options) {
|
|
|
844
973
|
continue;
|
|
845
974
|
}
|
|
846
975
|
const overlayName = overlayNames.get(t.table);
|
|
847
|
-
const lakeNames = overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name);
|
|
976
|
+
const lakeNames = (overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name)).sort();
|
|
977
|
+
const signature = viewSignature(lakeNames, overlayName);
|
|
848
978
|
try {
|
|
849
979
|
signal?.throwIfAborted();
|
|
850
|
-
await
|
|
980
|
+
await timed(onTiming, "view", {
|
|
981
|
+
table: t.table,
|
|
982
|
+
files: files.length
|
|
983
|
+
}, () => withDb(() => ensureView(registry, conn, schema, t.table, signature, overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
|
|
851
984
|
} catch (err) {
|
|
852
985
|
if (isAbortError$1(err)) {
|
|
853
986
|
await withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])).catch(() => {});
|
|
854
987
|
throw err;
|
|
855
988
|
}
|
|
856
989
|
if (isOpfsAccessHandleConflict(err)) {
|
|
857
|
-
if (registry.viewRefs(
|
|
990
|
+
if (registry.viewRefs(viewKey(schema, t.table)) === 0) await withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)).catch(() => {});
|
|
858
991
|
degraded.add(t.table);
|
|
859
992
|
degradeReason.set(t.table, "contention");
|
|
860
993
|
await releaseTable(t.table);
|
|
@@ -864,57 +997,70 @@ async function attachOpfsParquetTables(options) {
|
|
|
864
997
|
throw err;
|
|
865
998
|
}
|
|
866
999
|
attached.push(t.table);
|
|
867
|
-
registry.acquireView(`${schema}.${t.table}`);
|
|
868
1000
|
for (const f of files) registeredNames.push(f.name);
|
|
869
1001
|
}
|
|
870
1002
|
const recoverTableToBuffers = async (t) => {
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
const
|
|
876
|
-
|
|
877
|
-
|
|
1003
|
+
return timed(onTiming, "recovery", {
|
|
1004
|
+
table: t.table,
|
|
1005
|
+
files: t.files.length + (t.overlay ? 1 : 0)
|
|
1006
|
+
}, async () => {
|
|
1007
|
+
const readOne = async (file, index) => {
|
|
1008
|
+
const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
|
|
1009
|
+
if (cached) return cached;
|
|
1010
|
+
signal?.throwIfAborted();
|
|
1011
|
+
const resp = await fetchImpl(file.url, {
|
|
1012
|
+
...fetchInit,
|
|
1013
|
+
signal
|
|
1014
|
+
});
|
|
1015
|
+
if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
|
|
1016
|
+
return new Uint8Array(await resp.arrayBuffer());
|
|
1017
|
+
};
|
|
1018
|
+
const lakeNames = [];
|
|
1019
|
+
for (let i = 0; i < t.files.length; i++) {
|
|
1020
|
+
const file = t.files[i];
|
|
1021
|
+
const name = await recoveredBufferName(t.table, file, i);
|
|
1022
|
+
const buf = await readOne(file, i);
|
|
1023
|
+
await withDb(() => bufferRegistry.acquire(name, () => buf));
|
|
1024
|
+
bufferFiles.push(name);
|
|
1025
|
+
lakeNames.push(name);
|
|
1026
|
+
bytesAttached += file.bytes;
|
|
1027
|
+
}
|
|
1028
|
+
let overlayName;
|
|
1029
|
+
if (t.overlay) {
|
|
1030
|
+
overlayName = await recoveredBufferName(t.table, t.overlay, t.files.length);
|
|
1031
|
+
const buf = await readOne(t.overlay, t.files.length);
|
|
1032
|
+
await withDb(() => bufferRegistry.acquire(overlayName, () => buf));
|
|
1033
|
+
bufferFiles.push(overlayName);
|
|
1034
|
+
bytesAttached += t.overlay.bytes;
|
|
1035
|
+
}
|
|
1036
|
+
lakeNames.sort();
|
|
1037
|
+
const body = overlayViewBody({
|
|
1038
|
+
lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
|
|
1039
|
+
overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
|
|
1040
|
+
materializeLake: false
|
|
878
1041
|
});
|
|
879
|
-
if (!
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
for (let i = 0; i < t.files.length; i++) {
|
|
884
|
-
const buf = await readOne(t.files[i], i);
|
|
885
|
-
const name = `${RECOVER_BUFFER_PREFIX}${t.table}_${i}.parquet`;
|
|
886
|
-
await withDb(() => db.registerFileBuffer(name, buf));
|
|
887
|
-
bufferFiles.push(name);
|
|
888
|
-
lakeNames.push(name);
|
|
889
|
-
bytesAttached += t.files[i].bytes;
|
|
890
|
-
}
|
|
891
|
-
let overlayName;
|
|
892
|
-
if (t.overlay) {
|
|
893
|
-
const buf = await readOne(t.overlay, t.files.length);
|
|
894
|
-
overlayName = `${RECOVER_BUFFER_PREFIX}${t.table}_overlay.parquet`;
|
|
895
|
-
await withDb(() => db.registerFileBuffer(overlayName, buf));
|
|
896
|
-
bufferFiles.push(overlayName);
|
|
897
|
-
bytesAttached += t.overlay.bytes;
|
|
898
|
-
}
|
|
899
|
-
const body = overlayViewBody({
|
|
900
|
-
lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
|
|
901
|
-
overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
|
|
902
|
-
materializeLake: false
|
|
1042
|
+
if (!body) return false;
|
|
1043
|
+
await withDb(() => ensureView(registry, conn, schema, t.table, viewSignature(lakeNames, overlayName), `CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
|
|
1044
|
+
bufferViews.push(t.table);
|
|
1045
|
+
return true;
|
|
903
1046
|
});
|
|
904
|
-
if (!body) return false;
|
|
905
|
-
await withDb(() => conn.query(`CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
|
|
906
|
-
bufferViews.push(t.table);
|
|
907
|
-
return true;
|
|
908
1047
|
};
|
|
909
1048
|
if (recoverContention && !signal?.aborted) for (const t of tables) {
|
|
910
1049
|
if (!degraded.has(t.table) || degradeReason.get(t.table) !== "contention") continue;
|
|
911
1050
|
const before = bufferFiles.length;
|
|
912
1051
|
if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
|
|
913
1052
|
else {
|
|
914
|
-
|
|
1053
|
+
await withDb(() => bufferRegistry.release(bufferFiles.slice(before))).catch(() => {});
|
|
915
1054
|
bufferFiles.length = before;
|
|
916
1055
|
}
|
|
917
1056
|
}
|
|
1057
|
+
emitTiming(onTiming, {
|
|
1058
|
+
stage: "total",
|
|
1059
|
+
durationMs: nowMs() - totalStarted,
|
|
1060
|
+
files: total,
|
|
1061
|
+
tables: tables.length,
|
|
1062
|
+
bytes: bytesAttached
|
|
1063
|
+
});
|
|
918
1064
|
let detached = false;
|
|
919
1065
|
return {
|
|
920
1066
|
version,
|
|
@@ -926,8 +1072,8 @@ async function attachOpfsParquetTables(options) {
|
|
|
926
1072
|
if (detached) return;
|
|
927
1073
|
detached = true;
|
|
928
1074
|
await detachOpfs(registry, conn, schema, attached, registeredNames);
|
|
929
|
-
for (const v of bufferViews) await conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).
|
|
930
|
-
|
|
1075
|
+
for (const v of bufferViews) await registry.releaseView(viewKey(schema, v), () => conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).then(() => {}));
|
|
1076
|
+
await bufferRegistry.release(bufferFiles);
|
|
931
1077
|
}
|
|
932
1078
|
};
|
|
933
1079
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine-duckdb-wasm",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.32.
|
|
4
|
+
"version": "0.32.12",
|
|
5
5
|
"description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -45,9 +45,9 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"drizzle-orm": "1.0.0-rc.3",
|
|
48
|
-
"@gscdump/
|
|
49
|
-
"@gscdump/
|
|
50
|
-
"gscdump": "0.32.
|
|
48
|
+
"@gscdump/engine": "0.32.12",
|
|
49
|
+
"@gscdump/contracts": "0.32.12",
|
|
50
|
+
"gscdump": "0.32.12"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@duckdb/duckdb-wasm": "^1.32.0",
|