@gscdump/engine-duckdb-wasm 3.5.0 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -4,11 +4,7 @@
4
4
  [![npm downloads](https://img.shields.io/npm/dm/@gscdump/engine-duckdb-wasm?color=yellow)](https://npm.chart.dev/@gscdump/engine-duckdb-wasm)
5
5
  [![license](https://img.shields.io/github/license/harlan-zw/gscdump?color=yellow)](https://github.com/harlan-zw/gscdump/blob/main/LICENSE)
6
6
 
7
- > DuckDB-WASM engine adapter for `@gscdump/analysis` typed browser analytics against parquet via R2.
8
-
9
- In-browser DuckDB-WASM connection wrapped as a `SqlQuerySource`. Ships a vendored, stripped-down drizzle-orm DuckDB-WASM adapter (~240 LoC, adapted from `@proj-airi/drizzle-duckdb-wasm`, MIT). Transactions throw — analytics workload is read-only.
10
-
11
- Bundle: **10.3 kB / 2.72 kB gzipped**. `@duckdb/duckdb-wasm` is an optional peer dep.
7
+ Run DuckDB queries and Analyzers over Parquet files in the browser.
12
8
 
13
9
  ## Install
14
10
 
@@ -16,7 +12,7 @@ Bundle: **10.3 kB / 2.72 kB gzipped**. `@duckdb/duckdb-wasm` is an optional peer
16
12
  npm install @gscdump/engine-duckdb-wasm @duckdb/duckdb-wasm
17
13
  ```
18
14
 
19
- ## Usage
15
+ ## Query Parquet URLs
20
16
 
21
17
  ```ts
22
18
  import {
@@ -26,59 +22,66 @@ import {
26
22
  } from '@gscdump/engine-duckdb-wasm'
27
23
 
28
24
  const { db, conn } = await bootDuckDBWasm()
29
- await attachParquetUrlTables(conn, { tables: [{ name: 'queries', url: '/r2/queries.parquet' }] })
25
+ await attachParquetUrlTables({
26
+ db,
27
+ conn,
28
+ tables: [{ table: 'queries', urls: ['https://example.com/data/queries.parquet'] }],
29
+ maxFiles: 10,
30
+ maxBytes: 50_000_000,
31
+ })
30
32
 
31
33
  const runner = await createInsightRunner({ db, conn })
32
34
  const client = await runner.client
33
35
  const rows = await client.query('SELECT query, clicks, impressions FROM queries LIMIT 50')
36
+ console.log(rows)
37
+ await runner.close()
38
+ await db.terminate()
34
39
  ```
35
40
 
36
- ## Browser Parquet Attachment Strategy
41
+ Replace the URL with a Parquet file containing the expected table columns, including `date`.
42
+ The endpoint must support browser access and byte-range reads.
37
43
 
38
- `attachParquetUrlTables()` uses URL registration with DuckDB-WASM's HTTP file
39
- reader rather than downloading parquet objects into JS memory. Each exact-object
40
- URL is preflighted with `HEAD`; if the endpoint does not support `HEAD`, the
41
- runtime performs a one-byte `Range: bytes=0-0` probe. Attachment fails closed
42
- unless the response proves `Content-Length` / `Content-Range` and byte-range
43
- support.
44
+ ## Attachment and caching
44
45
 
45
- The runtime keeps local guards independent of the server response:
46
- `maxFiles`, `maxBytes`, `fetchConcurrency`, and `AbortSignal` are enforced
47
- before registering files. `bootDuckDBWasm()` opens DuckDB with full HTTP reads
48
- disabled, so a server that cannot satisfy range reads routes to server-side
49
- fallback instead of causing broad browser object downloads.
46
+ `attachParquetUrlTables` registers URLs with DuckDB's HTTP reader.
47
+ It supports `maxFiles`, `maxBytes`, `fetchConcurrency`, and `signal` limits.
50
48
 
51
- OPFS (`attachOpfsParquetTables` et al., in `src/opfs.ts`) is the primary
52
- write-through parquet cache: attached snapshot files persist keyed by
53
- manifest `contentHash`/object key, gated by the same file/byte budgets before
54
- anything is materialized locally.
49
+ Preflight uses `HEAD`, with a one-byte range probe when needed.
50
+ Trusted URL size hints can skip preflight by default; set `trustSizeHint: false` to require endpoint checks.
51
+ The default HTTP mode disables full-file fallback reads.
52
+ Handle attachment failures in your application if you want to offer server queries instead.
55
53
 
56
- ### Attaching parquet as analyzer sources
54
+ `fetchInit` applies to preflight requests only.
55
+ DuckDB's own range reads need URLs that work without custom request headers.
57
56
 
58
- Per [ADR-0001](../../docs/adr/0001-browser-engine-uses-attached-tables.md), the
59
- browser engine dispatches over attached parquet tables (`createAttachedTableSource`
60
- from `@gscdump/engine/source`, wired up inside `createBrowserAnalysisRuntime`),
61
- not a canonical-schema `SqlQuerySource`. An earlier `createEngine()` wrapping the
62
- canonical-schema path had zero callers and was deleted in 2026-05 — don't
63
- reintroduce it; extend `createAttachedTableSource` or the attach helpers below
64
- instead.
57
+ Use `attachOpfsParquetTables` to cache snapshot files in the browser's Origin Private File System.
58
+ Its file and byte limits apply before download.
59
+ Use `estimateOpfsStorage`, `requestPersistentStorage`, and `clearOpfsSnapshotCache` to manage that cache.
65
60
 
66
- ## Exports
61
+ ## Analyzers
67
62
 
68
- - `createInsightRunner({ db, conn })` drizzle-orm handle for typed `.select()` / window functions, with `sql\`...\`` raw escape hatch.
69
- - `bootDuckDBWasm()` / `attachParquetTables()` / `attachParquetUrlTables()` / `attachSingleTable()` / `createBrowserAnalysisRuntime()` / `createDuckDBBundlesFromBase()` / `listAttachedTables()` — browser runtime primitives.
70
- - `attachOpfsParquetTables()` / `readOpfsSnapshotFile()` / `estimateOpfsStorage()` / `requestPersistentStorage()` / `clearOpfsSnapshotCache()` — OPFS-backed parquet cache.
71
- - `scopeFor(table, { siteId, window })` / `mergeScope()` — tenant scope predicates.
72
- - `pages` / `queries` / `page_queries` / `countries` / `dates` / `hourly_pages` / `schema` — drizzle schema mirroring `gscdump/analytics` `SCHEMAS`. Drift fails loudly at module load.
73
- - `compileArchetypeSql()` / `tableForArchetype()` — archetype query compilation.
74
- - `createClient` / `drizzle` / `DuckDBWasmDatabase` — vendored drizzle-orm DuckDB-WASM adapter.
75
- - `resolveWindow` (re-exported from `@gscdump/engine/period`).
63
+ `createBrowserAnalysisRuntime` runs Analyzers against attached tables.
64
+ See [`@gscdump/analysis`](../analysis/README.md#duckdb-and-browser-use) and
65
+ [ADR-0001](../../docs/adr/0001-browser-engine-uses-attached-tables.md) for the Source contract.
76
66
 
77
- ## Related
67
+ The package includes a vendored Drizzle adapter for typed queries.
68
+ Transactions are unsupported.
69
+
70
+ ## Exports
78
71
 
79
- - [`@gscdump/engine`](../engine) Storage contracts + dialect-neutral resolver.
80
- - [`@gscdump/analysis`](../analysis) Portable analyzer registry and browser dispatch.
81
- - [`@gscdump/engine-sqlite`](../engine-sqlite) SQLite / D1 counterpart.
72
+ | Group | Exports |
73
+ | --- | --- |
74
+ | Runtime | `bootDuckDBWasm`, `createInsightRunner`, `createBrowserAnalysisRuntime` |
75
+ | Attachment | `attachParquetTables`, `attachParquetUrlTables`, `attachParquetUrlTablesResult` |
76
+ | OPFS | `attachOpfsParquetTables`, `readOpfsSnapshotFile`, storage and cache helpers |
77
+ | Schema | `pages`, `queries`, `page_queries`, `countries`, `dates`, `hourly_pages`, `schema` |
78
+ | Queries | `compileArchetypeSql`, `tableForArchetype`, `scopeFor`, `mergeScope` |
79
+ | Drizzle | `createClient`, `drizzle`, `DuckDBWasmDatabase` |
80
+ | Dates | `resolveWindow`, re-exported from `@gscdump/engine/period` |
81
+
82
+ Schema exports follow `@gscdump/engine/schema`.
83
+ Parquet snapshots are Site-specific; browser `scopeFor` currently adds no `siteId` predicate.
84
+ Keep authorization and file selection in your application.
82
85
 
83
86
  ## License
84
87
 
@@ -1,6 +1,6 @@
1
1
  import { ArchetypeQuery } from "@gscdump/contracts/archetypes";
2
2
  /** A compiled, parameterised statement. */
3
- interface CompiledArchetypeSql {
3
+ export interface CompiledArchetypeSql {
4
4
  sql: string;
5
5
  params: unknown[];
6
6
  /** The fact-table view the query reads. */
@@ -9,7 +9,6 @@ interface CompiledArchetypeSql {
9
9
  /**
10
10
  * Compile one archetype query to DuckDB SQL. Throws for `aux-cloud-only`.
11
11
  */
12
- declare function compileArchetypeSql(query: ArchetypeQuery): CompiledArchetypeSql;
12
+ export declare function compileArchetypeSql(query: ArchetypeQuery): CompiledArchetypeSql;
13
13
  /** The fact-table view an archetype reads — drives per-table browser routing. */
14
- declare function tableForArchetype(query: ArchetypeQuery): string | null;
15
- export { CompiledArchetypeSql, compileArchetypeSql, tableForArchetype };
14
+ export declare function tableForArchetype(query: ArchetypeQuery): string | null;
@@ -1,9 +1,8 @@
1
1
  import { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm";
2
- interface DuckDBWasmClient {
2
+ export interface DuckDBWasmClient {
3
3
  db: AsyncDuckDB;
4
4
  conn: AsyncDuckDBConnection;
5
5
  query: (sql: string, params?: unknown[]) => Promise<Record<string, unknown>[]>;
6
6
  close: () => Promise<void>;
7
7
  }
8
- declare function createClient(db: AsyncDuckDB, conn: AsyncDuckDBConnection): Promise<DuckDBWasmClient>;
9
- export { DuckDBWasmClient, createClient };
8
+ export declare function createClient(db: AsyncDuckDB, conn: AsyncDuckDBConnection): Promise<DuckDBWasmClient>;
@@ -4,11 +4,10 @@ import { DrizzleConfig, entityKind } from "drizzle-orm";
4
4
  import { PgAsyncDatabase } from "drizzle-orm/pg-core";
5
5
  import { AnyRelations, EmptyRelations, ExtractTablesWithRelations, Schema } from "drizzle-orm/relations";
6
6
  type SchemaRelations<TSchema extends Schema> = ExtractTablesWithRelations<Record<string, never>, TSchema>;
7
- declare class DuckDBWasmDatabase<TRelations extends AnyRelations = EmptyRelations> extends PgAsyncDatabase<DuckDBWasmQueryResultHKT, TRelations> {
7
+ export declare class DuckDBWasmDatabase<TRelations extends AnyRelations = EmptyRelations> extends PgAsyncDatabase<DuckDBWasmQueryResultHKT, TRelations> {
8
8
  static readonly [entityKind]: string;
9
9
  }
10
- interface DuckDBWasmDrizzleDatabase<TSchema extends Schema = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>> extends DuckDBWasmDatabase<TRelations> {
10
+ export interface DuckDBWasmDrizzleDatabase<TSchema extends Schema = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>> extends DuckDBWasmDatabase<TRelations> {
11
11
  $client: Promise<DuckDBWasmClient>;
12
12
  }
13
- declare function drizzle<TSchema extends Schema = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>>(client: Promise<DuckDBWasmClient> | DuckDBWasmClient, config?: DrizzleConfig<TSchema, TRelations>): DuckDBWasmDrizzleDatabase<TSchema, TRelations>;
14
- export { DuckDBWasmDatabase, DuckDBWasmDrizzleDatabase, drizzle };
13
+ export declare function drizzle<TSchema extends Schema = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>>(client: Promise<DuckDBWasmClient> | DuckDBWasmClient, config?: DrizzleConfig<TSchema, TRelations>): DuckDBWasmDrizzleDatabase<TSchema, TRelations>;
@@ -1,8 +1,7 @@
1
1
  import "./client.mjs";
2
2
  import { Assume, entityKind } from "drizzle-orm";
3
3
  import { PgAsyncPreparedQuery, PgAsyncSession, PgDialect, PgQueryResultHKT } from "drizzle-orm/pg-core";
4
- type Row = Record<string, unknown>;
5
- interface DuckDBWasmQueryResultHKT extends PgQueryResultHKT {
4
+ export type Row = Record<string, unknown>;
5
+ export interface DuckDBWasmQueryResultHKT extends PgQueryResultHKT {
6
6
  type: Assume<this['row'], Row>[];
7
- }
8
- export { DuckDBWasmQueryResultHKT, Row };
7
+ }
package/dist/opfs.d.mts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm";
2
2
  /** A parquet data file to materialise into OPFS. */
3
- interface OpfsParquetFile {
3
+ export interface OpfsParquetFile {
4
4
  /** Same-origin URL carrying a signed size hint + short-lived access token. */
5
5
  url: string;
6
6
  /** Expected byte size — drives progress + a cheap pre-verify shortcut. */
@@ -17,7 +17,7 @@ interface OpfsParquetFile {
17
17
  rowCount?: number;
18
18
  }
19
19
  /** One logical table and the parquet files that compose it. */
20
- interface OpfsParquetTable {
20
+ export interface OpfsParquetTable {
21
21
  /** Iceberg table name — becomes the DuckDB view name. */
22
22
  table: string;
23
23
  files: OpfsParquetFile[];
@@ -33,7 +33,7 @@ interface OpfsParquetTable {
33
33
  */
34
34
  overlay?: OpfsParquetFile;
35
35
  }
36
- interface AttachOpfsTablesOptions {
36
+ export interface AttachOpfsTablesOptions {
37
37
  db: AsyncDuckDB;
38
38
  conn: AsyncDuckDBConnection;
39
39
  tables: OpfsParquetTable[];
@@ -87,7 +87,7 @@ interface AttachOpfsTablesOptions {
87
87
  */
88
88
  recoverContention?: boolean;
89
89
  }
90
- interface OpfsFileProgress {
90
+ export interface OpfsFileProgress {
91
91
  table: string;
92
92
  /** OPFS file name. */
93
93
  file: string;
@@ -106,8 +106,8 @@ interface OpfsFileProgress {
106
106
  /** End-to-end file phase time, materialise + register + callback overhead. */
107
107
  totalMs?: number;
108
108
  }
109
- type OpfsAttachTimingStage = 'persist' | 'root' | 'plan' | 'duckdb-import' | 'sweep' | 'materialise' | 'register' | 'downloads' | 'view' | 'recovery' | 'total';
110
- interface OpfsAttachTiming {
109
+ export type OpfsAttachTimingStage = 'persist' | 'root' | 'plan' | 'duckdb-import' | 'sweep' | 'materialise' | 'register' | 'downloads' | 'view' | 'recovery' | 'total';
110
+ export interface OpfsAttachTiming {
111
111
  stage: OpfsAttachTimingStage;
112
112
  durationMs: number;
113
113
  table?: string;
@@ -118,7 +118,7 @@ interface OpfsAttachTiming {
118
118
  outcome?: 'cache-hit' | 'downloaded';
119
119
  }
120
120
  /** Handle returned from {@link attachOpfsParquetTables}. */
121
- interface OpfsAttachedHandle {
121
+ export interface OpfsAttachedHandle {
122
122
  version: string | undefined;
123
123
  /** Tables that successfully attached (a per-table failure drops only that table). */
124
124
  tables: string[];
@@ -144,7 +144,7 @@ interface OpfsAttachedHandle {
144
144
  * Raised when OPFS cannot hold the file set. Carries the partial state so the
145
145
  * caller can degrade — attach what fit, route the rest server-side.
146
146
  */
147
- declare class OpfsQuotaExceededError extends Error {
147
+ export declare class OpfsQuotaExceededError extends Error {
148
148
  name: string;
149
149
  /** Tables that did not fit. */
150
150
  readonly degradedTables: string[];
@@ -155,9 +155,9 @@ declare class OpfsQuotaExceededError extends Error {
155
155
  * cache under pressure. Returns the granted state — `false` is normal for an
156
156
  * un-engaged origin and is NOT an error; it just means eviction is possible.
157
157
  */
158
- declare function requestPersistentStorage(): Promise<boolean>;
158
+ export declare function requestPersistentStorage(): Promise<boolean>;
159
159
  /** Best-effort `{ usageBytes, quotaBytes }` from the Storage API. */
160
- declare function estimateOpfsStorage(): Promise<{
160
+ export declare function estimateOpfsStorage(): Promise<{
161
161
  usageBytes?: number;
162
162
  quotaBytes?: number;
163
163
  }>;
@@ -174,7 +174,7 @@ declare function estimateOpfsStorage(): Promise<{
174
174
  * logic to drift out of lock-step. `index` is only the disambiguator for the
175
175
  * degraded no-`contentHash` fallback (mirrors `attachOpfsParquetTables`).
176
176
  */
177
- declare function readOpfsSnapshotFile(table: string, contentHash: string | undefined, index: number, expectedBytes: number): Promise<Uint8Array | null>;
177
+ export declare function readOpfsSnapshotFile(table: string, contentHash: string | undefined, index: number, expectedBytes: number): Promise<Uint8Array | null>;
178
178
  /**
179
179
  * Download every parquet file in `tables` into OPFS, content-hash verify, and
180
180
  * attach them as DuckDB-WASM views. Attach-once: call this once per
@@ -185,11 +185,10 @@ declare function readOpfsSnapshotFile(table: string, contentHash: string | undef
185
185
  * NOT throw — that table is recorded in `degradedTables` and skipped; the
186
186
  * caller routes it to the server tail. The remaining tables still attach.
187
187
  */
188
- declare function attachOpfsParquetTables(options: AttachOpfsTablesOptions): Promise<OpfsAttachedHandle>;
188
+ export declare function attachOpfsParquetTables(options: AttachOpfsTablesOptions): Promise<OpfsAttachedHandle>;
189
189
  /**
190
190
  * Delete every OPFS entry this module created. Used to reclaim space after a
191
191
  * quota error, or to force a clean re-download. Best-effort — missing entries
192
192
  * are ignored.
193
193
  */
194
- declare function clearOpfsSnapshotCache(): Promise<void>;
195
- export { AttachOpfsTablesOptions, OpfsAttachTiming, OpfsAttachTimingStage, OpfsAttachedHandle, OpfsFileProgress, OpfsParquetFile, OpfsParquetTable, OpfsQuotaExceededError, attachOpfsParquetTables, clearOpfsSnapshotCache, estimateOpfsStorage, readOpfsSnapshotFile, requestPersistentStorage };
194
+ export declare function clearOpfsSnapshotCache(): Promise<void>;
package/dist/opfs.mjs CHANGED
@@ -104,15 +104,15 @@ function opfsFileName(table, hashSlug, index) {
104
104
  function escapeRegExp(s) {
105
105
  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
106
106
  }
107
- function tableEntryMatcher(table) {
108
- return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?:[0-9a-f]{16}|\\d+(?:_[0-9a-f]{16})?)\\.parquet$`);
107
+ function legacyEntryMatcher(table) {
108
+ return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?![0-9a-f]{16}\\.parquet$)\\d+(?:_[0-9a-f]{16})?\\.parquet$`);
109
109
  }
110
- async function sweepStaleEntries(root, registry, expectedByTable) {
110
+ async function sweepLegacyEntries(root, registry, expectedByTable) {
111
111
  const dir = root;
112
112
  if (!dir.keys) return;
113
113
  const matchers = [...expectedByTable].map(([table, expected]) => ({
114
114
  expected,
115
- re: tableEntryMatcher(table)
115
+ re: legacyEntryMatcher(table)
116
116
  }));
117
117
  for await (const name of dir.keys()) {
118
118
  const m = matchers.find((m) => m.re.test(name));
@@ -120,7 +120,7 @@ async function sweepStaleEntries(root, registry, expectedByTable) {
120
120
  try {
121
121
  await root.removeEntry(name);
122
122
  } catch (error) {
123
- if (!isNotFoundError(error)) reportBestEffortFailure(`removing stale OPFS entry ${name}`, error);
123
+ if (!isNotFoundError(error)) reportBestEffortFailure(`removing legacy OPFS entry ${name}`, error);
124
124
  }
125
125
  }
126
126
  }
@@ -177,18 +177,47 @@ async function readOpfsSnapshotFile(table, contentHash, index, expectedBytes) {
177
177
  return null;
178
178
  }
179
179
  }
180
- async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
180
+ const fileWriteTails = /* @__PURE__ */ new Map();
181
+ async function withFileWrite(name, signal, write) {
181
182
  signal?.throwIfAborted();
182
- let handle;
183
+ const previous = fileWriteTails.get(name) ?? Promise.resolve();
184
+ const released = Promise.withResolvers();
185
+ const tail = previous.then(() => released.promise);
186
+ fileWriteTails.set(name, tail);
187
+ tail.then(() => {
188
+ if (fileWriteTails.get(name) === tail) fileWriteTails.delete(name);
189
+ });
190
+ let onAbort;
191
+ try {
192
+ if (signal) await new Promise((resolve, reject) => {
193
+ onAbort = () => reject(signal.reason);
194
+ signal.addEventListener("abort", onAbort, { once: true });
195
+ previous.then(resolve);
196
+ });
197
+ else await previous;
198
+ signal?.throwIfAborted();
199
+ return await write();
200
+ } finally {
201
+ if (signal && onAbort) signal.removeEventListener("abort", onAbort);
202
+ released.resolve();
203
+ }
204
+ }
205
+ async function cachedFile(root, name, expectedBytes) {
183
206
  try {
184
- handle = await root.getFileHandle(name);
185
- if ((await handle.getFile()).size === file.bytes) return {
186
- handle,
187
- outcome: "cache-hit"
188
- };
207
+ const handle = await root.getFileHandle(name);
208
+ return (await handle.getFile()).size === expectedBytes ? handle : void 0;
189
209
  } catch (error) {
190
210
  if (!isNotFoundError(error)) throw error;
211
+ return;
191
212
  }
213
+ }
214
+ async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
215
+ signal?.throwIfAborted();
216
+ const cached = await cachedFile(root, name, file.bytes);
217
+ if (cached) return {
218
+ handle: cached,
219
+ outcome: "cache-hit"
220
+ };
192
221
  signal?.throwIfAborted();
193
222
  const deadline = AbortSignal.timeout(DOWNLOAD_DEADLINE_MS);
194
223
  const fetchSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
@@ -197,45 +226,56 @@ async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
197
226
  signal: fetchSignal
198
227
  });
199
228
  if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} failed: ${resp.status}`);
200
- handle = await root.getFileHandle(name, { create: true });
201
- let writable;
202
- try {
203
- writable = await handle.createWritable();
204
- } catch (err) {
205
- await resp.body?.cancel().catch(() => void 0);
206
- throw err;
207
- }
208
- let reader;
209
- let bytesWritten = 0;
210
229
  try {
211
- if (resp.body) {
212
- reader = resp.body.getReader();
213
- while (true) {
214
- const { done, value } = await reader.read();
215
- if (done) break;
216
- bytesWritten += value.byteLength;
217
- if (bytesWritten > file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got more than ${file.bytes}`);
218
- await writable.write(value);
230
+ return await withFileWrite(name, signal, async () => {
231
+ const cached = await cachedFile(root, name, file.bytes);
232
+ if (cached) return {
233
+ handle: cached,
234
+ outcome: "cache-hit"
235
+ };
236
+ const handle = await root.getFileHandle(name, { create: true });
237
+ let writable;
238
+ try {
239
+ writable = await handle.createWritable();
240
+ } catch (err) {
241
+ await resp.body?.cancel().catch(() => void 0);
242
+ throw err;
219
243
  }
220
- } else {
221
- const buf = await resp.arrayBuffer();
222
- bytesWritten = buf.byteLength;
223
- await writable.write(buf);
224
- }
225
- if (bytesWritten !== file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got ${bytesWritten}`);
226
- await writable.close();
227
- } catch (err) {
228
- await reader?.cancel().catch(() => void 0);
229
- if (writable.abort) await attemptCleanup(`aborting partial OPFS write ${name}`, () => writable.abort());
230
- await attemptCleanup(`removing partial OPFS file ${name}`, () => root.removeEntry(name));
231
- throw err;
244
+ let reader;
245
+ let bytesWritten = 0;
246
+ try {
247
+ if (resp.body) {
248
+ reader = resp.body.getReader();
249
+ while (true) {
250
+ const { done, value } = await reader.read();
251
+ if (done) break;
252
+ bytesWritten += value.byteLength;
253
+ if (bytesWritten > file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got more than ${file.bytes}`);
254
+ await writable.write(value);
255
+ }
256
+ } else {
257
+ const buf = await resp.arrayBuffer();
258
+ bytesWritten = buf.byteLength;
259
+ await writable.write(buf);
260
+ }
261
+ if (bytesWritten !== file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got ${bytesWritten}`);
262
+ await writable.close();
263
+ } catch (err) {
264
+ await reader?.cancel().catch(() => void 0);
265
+ if (writable.abort) await attemptCleanup(`aborting partial OPFS write ${name}`, () => writable.abort());
266
+ await attemptCleanup(`removing partial OPFS file ${name}`, () => root.removeEntry(name));
267
+ throw err;
268
+ } finally {
269
+ reader?.releaseLock();
270
+ }
271
+ return {
272
+ handle,
273
+ outcome: "downloaded"
274
+ };
275
+ });
232
276
  } finally {
233
- reader?.releaseLock();
277
+ if (resp.body && !resp.bodyUsed) await attemptCleanup(`cancelling unused OPFS download ${name}`, () => resp.body.cancel());
234
278
  }
235
- return {
236
- handle,
237
- outcome: "downloaded"
238
- };
239
279
  }
240
280
  function quoteList(files) {
241
281
  return files.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ");
@@ -340,8 +380,8 @@ async function attachOpfsParquetTables(options) {
340
380
  await timed(onTiming, "sweep", {
341
381
  files: total,
342
382
  tables: tables.length
343
- }, () => sweepStaleEntries(root, registry, expectedByTable)).catch((error) => {
344
- reportBestEffortFailure("stale OPFS cache sweep", error);
383
+ }, () => sweepLegacyEntries(root, registry, expectedByTable)).catch((error) => {
384
+ reportBestEffortFailure("legacy OPFS cache sweep", error);
345
385
  });
346
386
  const tableFiles = /* @__PURE__ */ new Map();
347
387
  const degraded = /* @__PURE__ */ new Set();
@@ -25,9 +25,8 @@
25
25
  * plain CTE inlines and re-scans the parquet, the dominant cost in DuckDB-WASM.
26
26
  * Set false to keep the streaming re-scan shape (lower peak memory, two scans).
27
27
  */
28
- declare function overlayViewBody(args: {
28
+ export declare function overlayViewBody(args: {
29
29
  lakeSelect: string | null;
30
30
  overlaySelect: string | null;
31
31
  materializeLake?: boolean;
32
- }): string | null;
33
- export { overlayViewBody };
32
+ }): string | null;
package/dist/runner.d.mts CHANGED
@@ -4,17 +4,17 @@ import "./drizzle-adapter/index.mjs";
4
4
  import { Schema } from "./schema.mjs";
5
5
  import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
6
6
  import { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm";
7
- interface InsightRunnerOptions {
7
+ export interface InsightRunnerOptions {
8
8
  db: AsyncDuckDB;
9
9
  conn: AsyncDuckDBConnection;
10
10
  logger?: boolean;
11
11
  }
12
- interface InsightRunner {
12
+ export interface InsightRunner {
13
13
  db: DuckDBWasmDrizzleDatabase<Schema>;
14
14
  client: Promise<DuckDBWasmClient>;
15
15
  close: () => Promise<void>;
16
16
  }
17
- declare function createInsightRunner(opts: InsightRunnerOptions): Promise<InsightRunner>;
18
- declare const scopeFor: (table: "pages" | "queries" | "countries" | "page_queries" | "dates" | "search_appearance" | "search_appearance_pages" | "search_appearance_queries" | "search_appearance_page_queries" | "hourly_pages", opts: ScopedRunnerOptions) => TableScope;
19
- declare const mergeScope: typeof import("@gscdump/engine/scope").mergeScope;
20
- export { InsightRunner, InsightRunnerOptions, type ScopedRunnerOptions, type TableScope, createInsightRunner, mergeScope, scopeFor };
17
+ export declare function createInsightRunner(opts: InsightRunnerOptions): Promise<InsightRunner>;
18
+ export declare const scopeFor: (table: "pages" | "queries" | "countries" | "page_queries" | "dates" | "search_appearance" | "search_appearance_pages" | "search_appearance_queries" | "search_appearance_page_queries" | "hourly_pages", opts: ScopedRunnerOptions) => TableScope;
19
+ export declare const mergeScope: typeof import("@gscdump/engine/scope").mergeScope;
20
+ export type { ScopedRunnerOptions, TableScope };
@@ -2,20 +2,20 @@ import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
2
2
  import { Result } from "gscdump/result";
3
3
  import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles, DuckDBConfig } from "@duckdb/duckdb-wasm";
4
4
  import { AnalysisParams } from "@gscdump/engine/analysis-types";
5
- interface QueryResult {
5
+ export interface QueryResult {
6
6
  rows: Record<string, unknown>[];
7
7
  queryMs: number;
8
8
  }
9
- interface AnalyzeResult {
9
+ export interface AnalyzeResult {
10
10
  results: Record<string, unknown>[];
11
11
  meta: Record<string, unknown>;
12
12
  queryMs: number;
13
13
  }
14
- interface DuckDBWasmBootResult {
14
+ export interface DuckDBWasmBootResult {
15
15
  db: AsyncDuckDB;
16
16
  conn: AsyncDuckDBConnection;
17
17
  }
18
- interface BootDuckDBWasmOptions {
18
+ export interface BootDuckDBWasmOptions {
19
19
  /**
20
20
  * DuckDB-WASM logger. Defaults to a `ConsoleLogger` thresholded at
21
21
  * `LogLevel.WARNING`, so real warnings/errors still surface but the per-query
@@ -48,25 +48,25 @@ interface BootDuckDBWasmOptions {
48
48
  */
49
49
  memoryLimit?: string;
50
50
  }
51
- interface BrowserParquetFile {
51
+ export interface BrowserParquetFile {
52
52
  bytes: Uint8Array;
53
53
  name?: string;
54
54
  }
55
- interface BrowserParquetTable {
55
+ export interface BrowserParquetTable {
56
56
  table: string;
57
57
  files: BrowserParquetFile[];
58
58
  }
59
- interface BrowserParquetUrlTable {
59
+ export interface BrowserParquetUrlTable {
60
60
  table: string;
61
61
  urls: string[];
62
62
  }
63
- interface AttachParquetTablesOptions {
63
+ export interface AttachParquetTablesOptions {
64
64
  db: AsyncDuckDB;
65
65
  conn: AsyncDuckDBConnection;
66
66
  tables: BrowserParquetTable[];
67
67
  schema?: string;
68
68
  }
69
- interface AttachParquetUrlTablesOptions {
69
+ export interface AttachParquetUrlTablesOptions {
70
70
  db: AsyncDuckDB;
71
71
  conn: AsyncDuckDBConnection;
72
72
  tables: BrowserParquetUrlTable[];
@@ -136,13 +136,13 @@ interface AttachParquetUrlTablesOptions {
136
136
  * the created views (for lazy re-attach on a new manifest version) or cheap-
137
137
  * check the embedded version against a fresh probe.
138
138
  */
139
- interface AttachedTablesHandle {
139
+ export interface AttachedTablesHandle {
140
140
  version: number | string | undefined;
141
141
  tables: string[];
142
142
  schema: string;
143
143
  detach: () => Promise<void>;
144
144
  }
145
- interface BrowserAnalysisRuntime {
145
+ export interface BrowserAnalysisRuntime {
146
146
  db: AsyncDuckDB;
147
147
  conn: AsyncDuckDBConnection;
148
148
  query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<QueryResult>;
@@ -168,23 +168,23 @@ interface BrowserAnalysisRuntime {
168
168
  setAttachedTables: (tables: readonly string[]) => void;
169
169
  close: () => Promise<void>;
170
170
  }
171
- declare class BrowserAttachBudgetExceededError extends Error {
171
+ export declare class BrowserAttachBudgetExceededError extends Error {
172
172
  name: string;
173
173
  }
174
- interface BrowserAttachError {
174
+ export interface BrowserAttachError {
175
175
  kind: 'browser-attach-budget-exceeded';
176
176
  message: string;
177
177
  /** Which budget tripped: the file count, the hinted byte plan, or the running byte plan. */
178
178
  budget: 'maxFiles' | 'maxBytes';
179
179
  }
180
- declare const browserAttachErrors: {
180
+ export declare const browserAttachErrors: {
181
181
  readonly maxFilesExceeded: (files: number, maxFiles: number) => BrowserAttachError;
182
182
  readonly hintedBytesExceeded: (hintedBytes: number, maxBytes: number) => BrowserAttachError;
183
183
  readonly plannedBytesExceeded: (plannedBytes: number, maxBytes: number) => BrowserAttachError;
184
184
  };
185
- declare function isBrowserAttachError(value: unknown): value is BrowserAttachError;
186
- declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
187
- declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
185
+ export declare function isBrowserAttachError(value: unknown): value is BrowserAttachError;
186
+ export declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
187
+ export declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
188
188
  /**
189
189
  * Errors-as-values core for {@link attachParquetUrlTables}: returns a typed
190
190
  * {@link BrowserAttachError} when the requested file set blows the local file-
@@ -193,7 +193,7 @@ declare function attachParquetTables(options: AttachParquetTablesOptions): Promi
193
193
  * failures stay defects and propagate. `attachParquetUrlTables` is the thin
194
194
  * throwing wrapper preserving the historical `BrowserAttachBudgetExceededError`.
195
195
  */
196
- declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOptions): Promise<Result<AttachedTablesHandle, BrowserAttachError>>;
196
+ export declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOptions): Promise<Result<AttachedTablesHandle, BrowserAttachError>>;
197
197
  /**
198
198
  * Attach browser parquet URL tables, throwing
199
199
  * {@link BrowserAttachBudgetExceededError} when the requested set blows the
@@ -201,10 +201,9 @@ declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOpt
201
201
  * {@link attachParquetUrlTablesResult}; existing call sites and their
202
202
  * `instanceof BrowserAttachBudgetExceededError` checks keep holding.
203
203
  */
204
- declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
205
- declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, options?: {
204
+ export declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
205
+ export declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, options?: {
206
206
  schema?: string;
207
207
  version?: number | string;
208
208
  attachedTables?: readonly string[];
209
- }): BrowserAnalysisRuntime;
210
- export { AnalyzeResult, AttachParquetTablesOptions, AttachParquetUrlTablesOptions, AttachedTablesHandle, BootDuckDBWasmOptions, BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, BrowserAttachError, BrowserParquetFile, BrowserParquetTable, BrowserParquetUrlTable, DuckDBWasmBootResult, QueryResult, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, createBrowserAnalysisRuntime, isBrowserAttachError };
209
+ }): BrowserAnalysisRuntime;
package/dist/runtime.mjs CHANGED
@@ -423,8 +423,11 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
423
423
  attachedTables = next;
424
424
  },
425
425
  async close() {
426
- await conn.close();
427
- await db.terminate();
426
+ try {
427
+ await conn.close();
428
+ } finally {
429
+ await db.terminate();
430
+ }
428
431
  }
429
432
  };
430
433
  }
@@ -1,4 +1,4 @@
1
- type SharedAsyncResourceState = {
1
+ export type SharedAsyncResourceState = {
2
2
  _tag: 'idle' | 'loading' | 'ready';
3
3
  consecutiveFailures: number;
4
4
  lastFailureAt: null;
@@ -8,21 +8,20 @@ type SharedAsyncResourceState = {
8
8
  lastFailureAt: number;
9
9
  error: unknown;
10
10
  };
11
- interface SharedAsyncResourceOptions<T> {
11
+ export interface SharedAsyncResourceOptions<T> {
12
12
  load: () => Promise<T>;
13
13
  retryCooldownMs?: number;
14
14
  maxConsecutiveFailures?: number;
15
15
  now?: () => number;
16
16
  onStateChange?: (state: SharedAsyncResourceState) => void;
17
17
  }
18
- interface SharedAsyncResource<T> {
18
+ export interface SharedAsyncResource<T> {
19
19
  get: () => Promise<T>;
20
20
  state: () => SharedAsyncResourceState;
21
21
  reset: () => void;
22
22
  }
23
- declare function createSharedAsyncResource<T>(options: SharedAsyncResourceOptions<T>): SharedAsyncResource<T>;
24
- interface ObjectAsyncLock<T extends object> {
23
+ export declare function createSharedAsyncResource<T>(options: SharedAsyncResourceOptions<T>): SharedAsyncResource<T>;
24
+ export interface ObjectAsyncLock<T extends object> {
25
25
  run: <R>(key: T, task: () => Promise<R>) => Promise<R>;
26
26
  }
27
- declare function createObjectAsyncLock<T extends object>(onRejected?: (error: unknown) => void): ObjectAsyncLock<T>;
28
- export { ObjectAsyncLock, SharedAsyncResource, SharedAsyncResourceOptions, SharedAsyncResourceState, createObjectAsyncLock, createSharedAsyncResource };
27
+ export declare function createObjectAsyncLock<T extends object>(onRejected?: (error: unknown) => void): ObjectAsyncLock<T>;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/engine-duckdb-wasm",
3
3
  "type": "module",
4
- "version": "3.5.0",
4
+ "version": "3.6.0",
5
5
  "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -44,17 +44,17 @@
44
44
  }
45
45
  },
46
46
  "dependencies": {
47
- "@gscdump/contracts": "^3.5.0",
48
- "@gscdump/engine": "^3.5.0",
47
+ "@gscdump/contracts": "^3.6.0",
48
+ "@gscdump/engine": "^3.6.0",
49
49
  "drizzle-orm": "1.0.0-rc.4",
50
- "gscdump": "^3.5.0"
50
+ "gscdump": "^3.6.0"
51
51
  },
52
52
  "devDependencies": {
53
53
  "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
54
- "@vitest/browser": "^4.1.11",
55
- "@vitest/browser-playwright": "^4.1.11",
56
- "playwright": "^1.62.1",
57
- "vitest": "^4.1.11"
54
+ "@vitest/browser": "^5.0.0",
55
+ "@vitest/browser-playwright": "^5.0.0",
56
+ "playwright": "^1.63.0",
57
+ "vitest": "^5.0.0"
58
58
  },
59
59
  "scripts": {
60
60
  "build": "obuild",