@gscdump/engine-duckdb-wasm 0.37.2 → 0.37.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs +60 -20
- package/package.json +4 -4
package/dist/index.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { arrowToRows } from "@gscdump/engine/arrow";
|
|
|
2
2
|
import { DefaultLogger, NoopLogger, entityKind, sql } from "drizzle-orm";
|
|
3
3
|
import { PgAsyncDatabase, PgAsyncPreparedQuery, PgAsyncSession, PgDialect } from "drizzle-orm/pg-core";
|
|
4
4
|
import { buildRelations } from "drizzle-orm/relations";
|
|
5
|
-
import { toIsoDate } from "gscdump";
|
|
5
|
+
import { toIsoDate } from "gscdump/dates";
|
|
6
6
|
import { countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
|
|
7
7
|
import { createScopedHelpers } from "@gscdump/engine/scope";
|
|
8
8
|
import { runAnalyzerFromSource } from "@gscdump/engine/analyzer";
|
|
@@ -505,6 +505,9 @@ async function strikingMomentum(runner, opts = {}) {
|
|
|
505
505
|
function createOpfsHandleRegistry(backend) {
|
|
506
506
|
const entries = /* @__PURE__ */ new Map();
|
|
507
507
|
const pending = /* @__PURE__ */ new Map();
|
|
508
|
+
function reportCleanupFailure(operation, error) {
|
|
509
|
+
console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
|
|
510
|
+
}
|
|
508
511
|
async function acquire(name, openHandle) {
|
|
509
512
|
const existing = entries.get(name);
|
|
510
513
|
if (existing) {
|
|
@@ -513,7 +516,7 @@ function createOpfsHandleRegistry(backend) {
|
|
|
513
516
|
}
|
|
514
517
|
const inflight = pending.get(name);
|
|
515
518
|
if (inflight) {
|
|
516
|
-
await inflight.
|
|
519
|
+
await inflight.then(() => void 0, () => void 0);
|
|
517
520
|
const settled = entries.get(name);
|
|
518
521
|
if (settled) {
|
|
519
522
|
settled.refs++;
|
|
@@ -543,7 +546,9 @@ function createOpfsHandleRegistry(backend) {
|
|
|
543
546
|
entry.refs--;
|
|
544
547
|
if (entry.refs > 0) continue;
|
|
545
548
|
entries.delete(name);
|
|
546
|
-
await backend.drop(name).catch(() => {
|
|
549
|
+
await backend.drop(name).catch((error) => {
|
|
550
|
+
reportCleanupFailure(`dropping OPFS handle ${name}`, error);
|
|
551
|
+
});
|
|
547
552
|
}
|
|
548
553
|
}
|
|
549
554
|
const viewRefs = /* @__PURE__ */ new Map();
|
|
@@ -570,7 +575,9 @@ function createOpfsHandleRegistry(backend) {
|
|
|
570
575
|
return;
|
|
571
576
|
}
|
|
572
577
|
viewRefs.delete(key);
|
|
573
|
-
await drop().catch(() => {
|
|
578
|
+
await drop().catch((error) => {
|
|
579
|
+
reportCleanupFailure(`dropping view ${key}`, error);
|
|
580
|
+
});
|
|
574
581
|
}
|
|
575
582
|
return {
|
|
576
583
|
acquire,
|
|
@@ -609,10 +616,25 @@ const IDENT_RE = /^[A-Z_][\w$]*$/i;
|
|
|
609
616
|
function nowMs() {
|
|
610
617
|
return globalThis.performance?.now?.() ?? Date.now();
|
|
611
618
|
}
|
|
619
|
+
function isNotFoundError(error) {
|
|
620
|
+
return typeof error === "object" && error !== null && error.name === "NotFoundError";
|
|
621
|
+
}
|
|
622
|
+
function reportBestEffortFailure(operation, error) {
|
|
623
|
+
console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
|
|
624
|
+
}
|
|
625
|
+
async function attemptCleanup(operation, cleanup) {
|
|
626
|
+
try {
|
|
627
|
+
await cleanup();
|
|
628
|
+
} catch (error) {
|
|
629
|
+
reportBestEffortFailure(operation, error);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
612
632
|
function emitTiming(onTiming, info) {
|
|
613
633
|
try {
|
|
614
634
|
onTiming?.(info);
|
|
615
|
-
} catch {
|
|
635
|
+
} catch (error) {
|
|
636
|
+
reportBestEffortFailure("timing callback", error);
|
|
637
|
+
}
|
|
616
638
|
}
|
|
617
639
|
async function timed(onTiming, stage, meta, fn) {
|
|
618
640
|
const started = nowMs();
|
|
@@ -694,7 +716,11 @@ async function sweepStaleEntries(root, registry, expectedByTable) {
|
|
|
694
716
|
for await (const name of dir.keys()) {
|
|
695
717
|
const m = matchers.find((m) => m.re.test(name));
|
|
696
718
|
if (!m || m.expected.has(name) || registry.refs(name) > 0) continue;
|
|
697
|
-
|
|
719
|
+
try {
|
|
720
|
+
await root.removeEntry(name);
|
|
721
|
+
} catch (error) {
|
|
722
|
+
if (!isNotFoundError(error)) reportBestEffortFailure(`removing stale OPFS entry ${name}`, error);
|
|
723
|
+
}
|
|
698
724
|
}
|
|
699
725
|
}
|
|
700
726
|
async function requestPersistentStorage() {
|
|
@@ -745,7 +771,8 @@ async function readOpfsSnapshotFile(table, contentHash, index, expectedBytes) {
|
|
|
745
771
|
const file = await (await root.getFileHandle(name)).getFile();
|
|
746
772
|
if (file.size !== expectedBytes) return null;
|
|
747
773
|
return new Uint8Array(await file.arrayBuffer());
|
|
748
|
-
} catch {
|
|
774
|
+
} catch (error) {
|
|
775
|
+
if (!isNotFoundError(error)) reportBestEffortFailure(`reading OPFS snapshot ${table}`, error);
|
|
749
776
|
return null;
|
|
750
777
|
}
|
|
751
778
|
}
|
|
@@ -758,7 +785,9 @@ async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
|
|
|
758
785
|
handle,
|
|
759
786
|
outcome: "cache-hit"
|
|
760
787
|
};
|
|
761
|
-
} catch {
|
|
788
|
+
} catch (error) {
|
|
789
|
+
if (!isNotFoundError(error)) throw error;
|
|
790
|
+
}
|
|
762
791
|
signal?.throwIfAborted();
|
|
763
792
|
const deadline = AbortSignal.timeout(DOWNLOAD_DEADLINE_MS);
|
|
764
793
|
const fetchSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
|
|
@@ -775,8 +804,8 @@ async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
|
|
|
775
804
|
await writable.write(buf);
|
|
776
805
|
await writable.close();
|
|
777
806
|
} catch (err) {
|
|
778
|
-
|
|
779
|
-
await
|
|
807
|
+
if (writable.abort) await attemptCleanup(`aborting partial OPFS write ${name}`, () => writable.abort());
|
|
808
|
+
await attemptCleanup(`removing partial OPFS file ${name}`, () => root.removeEntry(name));
|
|
780
809
|
throw err;
|
|
781
810
|
}
|
|
782
811
|
return {
|
|
@@ -886,7 +915,9 @@ async function attachOpfsParquetTables(options) {
|
|
|
886
915
|
await timed(onTiming, "sweep", {
|
|
887
916
|
files: total,
|
|
888
917
|
tables: tables.length
|
|
889
|
-
}, () => sweepStaleEntries(root, registry, expectedByTable)).catch(() => {
|
|
918
|
+
}, () => sweepStaleEntries(root, registry, expectedByTable)).catch((error) => {
|
|
919
|
+
reportBestEffortFailure("stale OPFS cache sweep", error);
|
|
920
|
+
});
|
|
890
921
|
const tableFiles = /* @__PURE__ */ new Map();
|
|
891
922
|
const degraded = /* @__PURE__ */ new Set();
|
|
892
923
|
const degradeReason = /* @__PURE__ */ new Map();
|
|
@@ -981,7 +1012,7 @@ async function attachOpfsParquetTables(options) {
|
|
|
981
1012
|
tables: tables.length
|
|
982
1013
|
}, runDownloads);
|
|
983
1014
|
} catch (err) {
|
|
984
|
-
await withDb(() => registry.release([...acquiredNames]))
|
|
1015
|
+
await attemptCleanup("releasing handles after download failure", () => withDb(() => registry.release([...acquiredNames])));
|
|
985
1016
|
throw err;
|
|
986
1017
|
}
|
|
987
1018
|
const attached = [];
|
|
@@ -1009,17 +1040,17 @@ async function attachOpfsParquetTables(options) {
|
|
|
1009
1040
|
}, () => withDb(() => ensureView(registry, conn, schema, t.table, signature, overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
|
|
1010
1041
|
} catch (err) {
|
|
1011
1042
|
if (isAbortError$1(err)) {
|
|
1012
|
-
await withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames]))
|
|
1043
|
+
await attemptCleanup("detaching OPFS resources after abort", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
|
|
1013
1044
|
throw err;
|
|
1014
1045
|
}
|
|
1015
1046
|
if (isOpfsAccessHandleConflict(err)) {
|
|
1016
|
-
if (registry.viewRefs(viewKey(schema, t.table)) === 0) await withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`))
|
|
1047
|
+
if (registry.viewRefs(viewKey(schema, t.table)) === 0) await attemptCleanup(`dropping incomplete view ${schema}.${t.table}`, () => withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)));
|
|
1017
1048
|
degraded.add(t.table);
|
|
1018
1049
|
degradeReason.set(t.table, "contention");
|
|
1019
1050
|
await releaseTable(t.table);
|
|
1020
1051
|
continue;
|
|
1021
1052
|
}
|
|
1022
|
-
await withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames]))
|
|
1053
|
+
await attemptCleanup("detaching OPFS resources after view failure", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
|
|
1023
1054
|
throw err;
|
|
1024
1055
|
}
|
|
1025
1056
|
attached.push(t.table);
|
|
@@ -1076,7 +1107,7 @@ async function attachOpfsParquetTables(options) {
|
|
|
1076
1107
|
const before = bufferFiles.length;
|
|
1077
1108
|
if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
|
|
1078
1109
|
else {
|
|
1079
|
-
await withDb(() => bufferRegistry.release(bufferFiles.slice(before)))
|
|
1110
|
+
await attemptCleanup(`releasing recovery buffers for ${t.table}`, () => withDb(() => bufferRegistry.release(bufferFiles.slice(before))));
|
|
1080
1111
|
bufferFiles.length = before;
|
|
1081
1112
|
}
|
|
1082
1113
|
}
|
|
@@ -1109,14 +1140,21 @@ async function detachOpfs(registry, conn, schema, tables, files) {
|
|
|
1109
1140
|
await registry.release(files);
|
|
1110
1141
|
}
|
|
1111
1142
|
async function clearOpfsSnapshotCache() {
|
|
1112
|
-
const root = await getOpfsRoot().catch(() =>
|
|
1143
|
+
const root = await getOpfsRoot().catch((error) => {
|
|
1144
|
+
reportBestEffortFailure("opening OPFS cache for clearing", error);
|
|
1145
|
+
return null;
|
|
1146
|
+
});
|
|
1113
1147
|
if (!root) return;
|
|
1114
1148
|
const removable = [];
|
|
1115
1149
|
const dir = root;
|
|
1116
1150
|
if (dir.keys) {
|
|
1117
1151
|
for await (const name of dir.keys()) if (name.startsWith(OPFS_PREFIX)) removable.push(name);
|
|
1118
1152
|
}
|
|
1119
|
-
for (const name of removable)
|
|
1153
|
+
for (const name of removable) try {
|
|
1154
|
+
await root.removeEntry(name);
|
|
1155
|
+
} catch (error) {
|
|
1156
|
+
if (!isNotFoundError(error)) reportBestEffortFailure(`clearing OPFS cache entry ${name}`, error);
|
|
1157
|
+
}
|
|
1120
1158
|
}
|
|
1121
1159
|
async function createInsightRunner(opts) {
|
|
1122
1160
|
const client = await createClient(opts.db, opts.conn);
|
|
@@ -1476,14 +1514,16 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
|
1476
1514
|
}
|
|
1477
1515
|
function runExclusive(signal, work) {
|
|
1478
1516
|
const next = chain.then(work, work);
|
|
1479
|
-
chain = next.
|
|
1517
|
+
chain = next.then(() => void 0, () => void 0);
|
|
1480
1518
|
return raceSignal(next, signal);
|
|
1481
1519
|
}
|
|
1482
1520
|
async function cancelOnAbort(signal, work) {
|
|
1483
1521
|
if (!signal) return work;
|
|
1484
1522
|
if (signal.aborted) throw abortError(signal);
|
|
1485
1523
|
const onAbort = () => {
|
|
1486
|
-
conn.cancelSent().catch(() => {
|
|
1524
|
+
conn.cancelSent().catch((error) => {
|
|
1525
|
+
console.warn("[gscdump/engine-duckdb-wasm] failed to cancel aborted query", error);
|
|
1526
|
+
});
|
|
1487
1527
|
};
|
|
1488
1528
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
1489
1529
|
try {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine-duckdb-wasm",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.37.
|
|
4
|
+
"version": "0.37.4",
|
|
5
5
|
"description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -45,9 +45,9 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"drizzle-orm": "1.0.0-rc.3",
|
|
48
|
-
"@gscdump/
|
|
49
|
-
"gscdump": "0.37.
|
|
50
|
-
"@gscdump/
|
|
48
|
+
"@gscdump/engine": "0.37.4",
|
|
49
|
+
"gscdump": "0.37.4",
|
|
50
|
+
"@gscdump/contracts": "0.37.4"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@duckdb/duckdb-wasm": "^1.32.0",
|