@gscdump/engine-duckdb-wasm 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/archetype-sql.d.mts +15 -0
- package/dist/archetype-sql.mjs +373 -0
- package/dist/drizzle-adapter/client.d.mts +9 -0
- package/dist/drizzle-adapter/client.mjs +20 -0
- package/dist/drizzle-adapter/driver.d.mts +14 -0
- package/dist/drizzle-adapter/driver.mjs +21 -0
- package/dist/drizzle-adapter/index.d.mts +3 -0
- package/dist/drizzle-adapter/index.mjs +3 -0
- package/dist/drizzle-adapter/session.d.mts +8 -0
- package/dist/drizzle-adapter/session.mjs +28 -0
- package/dist/index.d.mts +9 -493
- package/dist/index.mjs +9 -1555
- package/dist/opfs-registry.mjs +89 -0
- package/dist/opfs.d.mts +195 -0
- package/dist/opfs.mjs +584 -0
- package/dist/overlay-view.d.mts +33 -0
- package/dist/overlay-view.mjs +10 -0
- package/dist/runner.d.mts +28 -0
- package/dist/runner.mjs +19 -0
- package/dist/runtime.d.mts +210 -0
- package/dist/runtime.mjs +429 -0
- package/dist/schema.d.mts +2 -0
- package/dist/schema.mjs +2 -0
- package/package.json +4 -4
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
|
|
2
|
+
import { Result } from "gscdump/result";
|
|
3
|
+
import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles, DuckDBConfig } from "@duckdb/duckdb-wasm";
|
|
4
|
+
import { AnalysisParams } from "@gscdump/engine/analysis-types";
|
|
5
|
+
interface QueryResult {
|
|
6
|
+
rows: Record<string, unknown>[];
|
|
7
|
+
queryMs: number;
|
|
8
|
+
}
|
|
9
|
+
interface AnalyzeResult {
|
|
10
|
+
results: Record<string, unknown>[];
|
|
11
|
+
meta: Record<string, unknown>;
|
|
12
|
+
queryMs: number;
|
|
13
|
+
}
|
|
14
|
+
interface DuckDBWasmBootResult {
|
|
15
|
+
db: AsyncDuckDB;
|
|
16
|
+
conn: AsyncDuckDBConnection;
|
|
17
|
+
}
|
|
18
|
+
interface BootDuckDBWasmOptions {
|
|
19
|
+
/**
|
|
20
|
+
* DuckDB-WASM logger. Defaults to a `ConsoleLogger` thresholded at
|
|
21
|
+
* `LogLevel.WARNING`, so real warnings/errors still surface but the per-query
|
|
22
|
+
* INFO events (START/OK/RUN) — which DuckDB's default `ConsoleLogger()` emits
|
|
23
|
+
* as raw objects, flooding the host console with dozens of lines per render —
|
|
24
|
+
* are dropped. Pass `new ConsoleLogger(LogLevel.DEBUG)` to see everything, or
|
|
25
|
+
* `new VoidLogger()` to silence it entirely.
|
|
26
|
+
*/
|
|
27
|
+
logger?: unknown;
|
|
28
|
+
/**
|
|
29
|
+
* Override the jsDelivr-hosted bundle map. Required in environments where
|
|
30
|
+
* the default CDN is unreachable or where hosts must serve the WASM +
|
|
31
|
+
* worker assets themselves (e.g. Cloudflare Workers' 25 MB per-asset cap).
|
|
32
|
+
*/
|
|
33
|
+
bundles?: DuckDBBundles;
|
|
34
|
+
/**
|
|
35
|
+
* Extra DuckDB open config. The browser runtime always forces HTTP files
|
|
36
|
+
* into range-only mode (reliable HEAD probes, no full HTTP fallback) so a
|
|
37
|
+
* server that cannot answer bounded reads fails closed.
|
|
38
|
+
*/
|
|
39
|
+
config?: DuckDBConfig;
|
|
40
|
+
/**
|
|
41
|
+
* Cap DuckDB's memory with `SET memory_limit=<value>` right after open (e.g.
|
|
42
|
+
* `'2GB'`, `'512MB'`). A runaway query then errors with a clean
|
|
43
|
+
* out-of-memory rather than growing the WASM heap until the tab crashes —
|
|
44
|
+
* DuckDB has no statement-level timeout, so abandoned queries are otherwise
|
|
45
|
+
* only stopped by the caller's `AbortSignal` (which the runtime forwards to
|
|
46
|
+
* `conn.cancelSent()`). Opt-in: omit to keep DuckDB-WASM's default sizing.
|
|
47
|
+
* Accepts `<number>` with an optional `B`/`KB`/`MB`/`GB`/`TB` suffix.
|
|
48
|
+
*/
|
|
49
|
+
memoryLimit?: string;
|
|
50
|
+
}
|
|
51
|
+
interface BrowserParquetFile {
|
|
52
|
+
bytes: Uint8Array;
|
|
53
|
+
name?: string;
|
|
54
|
+
}
|
|
55
|
+
interface BrowserParquetTable {
|
|
56
|
+
table: string;
|
|
57
|
+
files: BrowserParquetFile[];
|
|
58
|
+
}
|
|
59
|
+
interface BrowserParquetUrlTable {
|
|
60
|
+
table: string;
|
|
61
|
+
urls: string[];
|
|
62
|
+
}
|
|
63
|
+
interface AttachParquetTablesOptions {
|
|
64
|
+
db: AsyncDuckDB;
|
|
65
|
+
conn: AsyncDuckDBConnection;
|
|
66
|
+
tables: BrowserParquetTable[];
|
|
67
|
+
schema?: string;
|
|
68
|
+
}
|
|
69
|
+
interface AttachParquetUrlTablesOptions {
|
|
70
|
+
db: AsyncDuckDB;
|
|
71
|
+
conn: AsyncDuckDBConnection;
|
|
72
|
+
tables: BrowserParquetUrlTable[];
|
|
73
|
+
fetch?: typeof fetch;
|
|
74
|
+
schema?: string;
|
|
75
|
+
/**
|
|
76
|
+
* Request init used only for runtime-owned HEAD / one-byte Range preflights.
|
|
77
|
+
* DuckDB-WASM's internal HTTP reader cannot receive custom fetch headers;
|
|
78
|
+
* URL reads must therefore be authorized by the URL itself.
|
|
79
|
+
*/
|
|
80
|
+
fetchInit?: RequestInit;
|
|
81
|
+
/**
|
|
82
|
+
* Caps simultaneous URL preflights. Browser source endpoints should already
|
|
83
|
+
* return small, coverage-planned URL sets; this is the runtime's local
|
|
84
|
+
* guard against accidental unbounded attachment.
|
|
85
|
+
*/
|
|
86
|
+
fetchConcurrency?: number;
|
|
87
|
+
/** Reject before preflight when the URL set exceeds this many parquet files. */
|
|
88
|
+
maxFiles?: number;
|
|
89
|
+
/** Reject before registration when hinted or authoritative bytes exceed budget. */
|
|
90
|
+
maxBytes?: number;
|
|
91
|
+
/** Abort signal passed through to URL preflights and registration. */
|
|
92
|
+
signal?: AbortSignal;
|
|
93
|
+
/**
|
|
94
|
+
* How DuckDB reads the parquet bytes:
|
|
95
|
+
* - `'http'` (default): register the URL as an HTTP file; DuckDB issues
|
|
96
|
+
* its own range reads during query execution. Right for large parquet
|
|
97
|
+
* where only some column chunks are touched per query.
|
|
98
|
+
* - `'buffer'`: fetch the full file once into an `ArrayBuffer` and register
|
|
99
|
+
* it via `registerFileBuffer`. DuckDB makes zero HTTP calls after
|
|
100
|
+
* registration — every query reads from the in-memory copy. Right for
|
|
101
|
+
* tiny files (e.g. `dates` daily-totals parquet, <50 KB per file) where
|
|
102
|
+
* the per-file round-trip overhead dominates the actual bytes moved.
|
|
103
|
+
*/
|
|
104
|
+
attachMode?: 'http' | 'buffer';
|
|
105
|
+
/**
|
|
106
|
+
* When `true` (default), skip per-file HEAD/Range preflight if the URL
|
|
107
|
+
* carries a signed `?s=<bytes>.<sig>` size hint — the size is already
|
|
108
|
+
* known and DuckDB will learn `Accept-Ranges` from its first read.
|
|
109
|
+
* Eliminates one round-trip per file on hosts that mint size hints. Set
|
|
110
|
+
* `false` to force preflight against URLs whose size you don't trust.
|
|
111
|
+
*/
|
|
112
|
+
trustSizeHint?: boolean;
|
|
113
|
+
/**
|
|
114
|
+
* Manifest version the caller associates with this set of URLs. Returned
|
|
115
|
+
* on the resulting handle so callers can compare against a fresh manifest
|
|
116
|
+
* probe without re-attaching. Purely advisory — the runtime never derives
|
|
117
|
+
* behavior from the value itself.
|
|
118
|
+
*/
|
|
119
|
+
version?: number | string;
|
|
120
|
+
/**
|
|
121
|
+
* Called once per parquet file after it's been fetched and registered with
|
|
122
|
+
* DuckDB. For URL-backed HTTP files this means "preflighted and registered"
|
|
123
|
+
* rather than fully downloaded; DuckDB then performs range reads during the
|
|
124
|
+
* query. Fires in bounded-concurrency completion order, which is still not
|
|
125
|
+
* guaranteed to match manifest URL order. Used by UI progress indicators to
|
|
126
|
+
* tick a per-site counter; a no-op default keeps the hot path free.
|
|
127
|
+
*/
|
|
128
|
+
onFileAttached?: (info: {
|
|
129
|
+
table: string;
|
|
130
|
+
index: number;
|
|
131
|
+
total: number;
|
|
132
|
+
}) => void;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Handle returned from {@link attachParquetUrlTables}. Lets callers detach
|
|
136
|
+
* the created views (for lazy re-attach on a new manifest version) or cheap-
|
|
137
|
+
* check the embedded version against a fresh probe.
|
|
138
|
+
*/
|
|
139
|
+
interface AttachedTablesHandle {
|
|
140
|
+
version: number | string | undefined;
|
|
141
|
+
tables: string[];
|
|
142
|
+
schema: string;
|
|
143
|
+
detach: () => Promise<void>;
|
|
144
|
+
}
|
|
145
|
+
interface BrowserAnalysisRuntime {
|
|
146
|
+
db: AsyncDuckDB;
|
|
147
|
+
conn: AsyncDuckDBConnection;
|
|
148
|
+
query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<QueryResult>;
|
|
149
|
+
analyze: (params: AnalysisParams, registry: AnalyzerRegistry, options?: {
|
|
150
|
+
signal?: AbortSignal;
|
|
151
|
+
}) => Promise<AnalyzeResult>;
|
|
152
|
+
/**
|
|
153
|
+
* Returns true when `expected` doesn't match the version the runtime was
|
|
154
|
+
* attached with — cheap check callers can run before each query to decide
|
|
155
|
+
* whether to detach + re-attach against a fresher manifest. Undefined
|
|
156
|
+
* values on either side compare equal so the no-version path is a no-op.
|
|
157
|
+
*/
|
|
158
|
+
isStale: (expected: number | string | undefined) => boolean;
|
|
159
|
+
/** Update the runtime's cached manifest version in-place (e.g. after a re-attach). */
|
|
160
|
+
setVersion: (version: number | string | undefined) => void;
|
|
161
|
+
/**
|
|
162
|
+
* Update the list of attached table names. Lets callers fast-fail in
|
|
163
|
+
* `analyze()` when a SQL plan references a table that wasn't in the manifest
|
|
164
|
+
* for this site (e.g. site has only `queries` parquet, analyzer wants
|
|
165
|
+
* `page_queries`) — surface a clean `AttachedTableMissingError` so the
|
|
166
|
+
* caller can route to cloud fallback without paying the SQL execution cost.
|
|
167
|
+
*/
|
|
168
|
+
setAttachedTables: (tables: readonly string[]) => void;
|
|
169
|
+
close: () => Promise<void>;
|
|
170
|
+
}
|
|
171
|
+
declare class BrowserAttachBudgetExceededError extends Error {
|
|
172
|
+
name: string;
|
|
173
|
+
}
|
|
174
|
+
interface BrowserAttachError {
|
|
175
|
+
kind: 'browser-attach-budget-exceeded';
|
|
176
|
+
message: string;
|
|
177
|
+
/** Which budget tripped: the file count, the hinted byte plan, or the running byte plan. */
|
|
178
|
+
budget: 'maxFiles' | 'maxBytes';
|
|
179
|
+
}
|
|
180
|
+
declare const browserAttachErrors: {
|
|
181
|
+
readonly maxFilesExceeded: (files: number, maxFiles: number) => BrowserAttachError;
|
|
182
|
+
readonly hintedBytesExceeded: (hintedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
183
|
+
readonly plannedBytesExceeded: (plannedBytes: number, maxBytes: number) => BrowserAttachError;
|
|
184
|
+
};
|
|
185
|
+
declare function isBrowserAttachError(value: unknown): value is BrowserAttachError;
|
|
186
|
+
declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
|
|
187
|
+
declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
|
|
188
|
+
/**
|
|
189
|
+
* Errors-as-values core for {@link attachParquetUrlTables}: returns a typed
|
|
190
|
+
* {@link BrowserAttachError} when the requested file set blows the local file-
|
|
191
|
+
* count / byte budget, so callers can branch (route the affected tables
|
|
192
|
+
* server-side) instead of catching an untyped throw. WASM/DuckDB/HTTP IO
|
|
193
|
+
* failures stay defects and propagate. `attachParquetUrlTables` is the thin
|
|
194
|
+
* throwing wrapper preserving the historical `BrowserAttachBudgetExceededError`.
|
|
195
|
+
*/
|
|
196
|
+
declare function attachParquetUrlTablesResult(options: AttachParquetUrlTablesOptions): Promise<Result<AttachedTablesHandle, BrowserAttachError>>;
|
|
197
|
+
/**
|
|
198
|
+
* Attach browser parquet URL tables, throwing
|
|
199
|
+
* {@link BrowserAttachBudgetExceededError} when the requested set blows the
|
|
200
|
+
* file-count / byte budget. Thin throwing wrapper over
|
|
201
|
+
* {@link attachParquetUrlTablesResult}; existing call sites and their
|
|
202
|
+
* `instanceof BrowserAttachBudgetExceededError` checks keep holding.
|
|
203
|
+
*/
|
|
204
|
+
declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
|
|
205
|
+
declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, options?: {
|
|
206
|
+
schema?: string;
|
|
207
|
+
version?: number | string;
|
|
208
|
+
attachedTables?: readonly string[];
|
|
209
|
+
}): BrowserAnalysisRuntime;
|
|
210
|
+
export { AnalyzeResult, AttachParquetTablesOptions, AttachParquetUrlTablesOptions, AttachedTablesHandle, BootDuckDBWasmOptions, BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, BrowserAttachError, BrowserParquetFile, BrowserParquetTable, BrowserParquetUrlTable, DuckDBWasmBootResult, QueryResult, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, createBrowserAnalysisRuntime, isBrowserAttachError };
|
package/dist/runtime.mjs
ADDED
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
import { arrowToRows } from "@gscdump/engine/arrow";
|
|
2
|
+
import { runAnalyzerFromSource } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { pgResolverAdapter } from "@gscdump/engine/resolver";
|
|
4
|
+
import { createAttachedTableSource } from "@gscdump/engine/source";
|
|
5
|
+
import { sqlEscape } from "@gscdump/engine/sql";
|
|
6
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
7
|
+
const DEFAULT_ATTACH_FETCH_CONCURRENCY = 2;
|
|
8
|
+
const DEFAULT_ATTACH_MAX_FILES = 32;
|
|
9
|
+
const DEFAULT_ATTACH_MAX_BYTES = 16 * 1024 * 1024;
|
|
10
|
+
let nextAttachId = 0;
|
|
11
|
+
var BrowserAttachBudgetExceededError = class extends Error {
|
|
12
|
+
name = "BrowserAttachBudgetExceededError";
|
|
13
|
+
};
|
|
14
|
+
const browserAttachErrors = {
|
|
15
|
+
maxFilesExceeded(files, maxFiles) {
|
|
16
|
+
return {
|
|
17
|
+
kind: "browser-attach-budget-exceeded",
|
|
18
|
+
budget: "maxFiles",
|
|
19
|
+
message: `browser parquet attach requires ${files} files, above maxFiles=${maxFiles}`
|
|
20
|
+
};
|
|
21
|
+
},
|
|
22
|
+
hintedBytesExceeded(hintedBytes, maxBytes) {
|
|
23
|
+
return {
|
|
24
|
+
kind: "browser-attach-budget-exceeded",
|
|
25
|
+
budget: "maxBytes",
|
|
26
|
+
message: `browser parquet attach requires ${hintedBytes} hinted bytes, above maxBytes=${maxBytes}`
|
|
27
|
+
};
|
|
28
|
+
},
|
|
29
|
+
plannedBytesExceeded(plannedBytes, maxBytes) {
|
|
30
|
+
return {
|
|
31
|
+
kind: "browser-attach-budget-exceeded",
|
|
32
|
+
budget: "maxBytes",
|
|
33
|
+
message: `browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${maxBytes}`
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
function isBrowserAttachError(value) {
|
|
38
|
+
return typeof value === "object" && value !== null && value.kind === "browser-attach-budget-exceeded" && typeof value.message === "string";
|
|
39
|
+
}
|
|
40
|
+
function browserAttachErrorToException(error) {
|
|
41
|
+
const exception = new BrowserAttachBudgetExceededError(error.message);
|
|
42
|
+
exception.browserAttachError = error;
|
|
43
|
+
return exception;
|
|
44
|
+
}
|
|
45
|
+
function fileName(table, index, provided) {
|
|
46
|
+
return provided ?? `${table}_${index}.parquet`;
|
|
47
|
+
}
|
|
48
|
+
function attachFileName(attachId, table, index) {
|
|
49
|
+
return `__gscdump_attach_${attachId}_${fileName(table, index)}`;
|
|
50
|
+
}
|
|
51
|
+
function readParquetViewSql(schema, table, files) {
|
|
52
|
+
return `CREATE OR REPLACE VIEW ${schema}.${table} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${files.map((name) => `'${sqlEscape(name)}'`).join(", ")}], union_by_name = true)`;
|
|
53
|
+
}
|
|
54
|
+
function positiveInteger(value, fallback, label) {
|
|
55
|
+
const raw = value ?? fallback;
|
|
56
|
+
if (!Number.isFinite(raw) || raw < 1) throw new Error(`${label} must be a positive integer`);
|
|
57
|
+
return Math.floor(raw);
|
|
58
|
+
}
|
|
59
|
+
async function runWithConcurrency(items, concurrency, fn) {
|
|
60
|
+
let next = 0;
|
|
61
|
+
let failed = false;
|
|
62
|
+
async function worker() {
|
|
63
|
+
while (!failed && next < items.length) {
|
|
64
|
+
const index = next++;
|
|
65
|
+
try {
|
|
66
|
+
await fn(items[index], index);
|
|
67
|
+
} catch (err) {
|
|
68
|
+
failed = true;
|
|
69
|
+
throw err;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((result) => result.status === "rejected");
|
|
74
|
+
if (rejected) throw rejected.reason;
|
|
75
|
+
}
|
|
76
|
+
function sizeHintFromUrl(url) {
|
|
77
|
+
try {
|
|
78
|
+
const base = typeof globalThis.location?.href === "string" ? globalThis.location.href : "http://localhost";
|
|
79
|
+
const raw = new URL(url, base).searchParams.get("s")?.split(".")[0];
|
|
80
|
+
if (!raw) return null;
|
|
81
|
+
const size = Number(raw);
|
|
82
|
+
return Number.isFinite(size) && size >= 0 ? size : null;
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function mergeAbortSignals(primary, secondary) {
|
|
88
|
+
if (!primary) return secondary;
|
|
89
|
+
if (!secondary) return primary;
|
|
90
|
+
if (primary.aborted) return primary;
|
|
91
|
+
if (secondary.aborted) return secondary;
|
|
92
|
+
const controller = new AbortController();
|
|
93
|
+
const abort = (signal) => {
|
|
94
|
+
controller.abort(signal.reason);
|
|
95
|
+
};
|
|
96
|
+
primary.addEventListener("abort", () => abort(primary), { once: true });
|
|
97
|
+
secondary.addEventListener("abort", () => abort(secondary), { once: true });
|
|
98
|
+
return controller.signal;
|
|
99
|
+
}
|
|
100
|
+
function fetchInitFor(fetchInit, method, signal, extraHeaders) {
|
|
101
|
+
const { body: _body, method: _method, signal: initSignal, headers: initHeaders, ...rest } = fetchInit ?? {};
|
|
102
|
+
const headers = new Headers(initHeaders);
|
|
103
|
+
for (const [key, value] of Object.entries(extraHeaders ?? {})) headers.set(key, value);
|
|
104
|
+
return {
|
|
105
|
+
...rest,
|
|
106
|
+
method,
|
|
107
|
+
headers,
|
|
108
|
+
signal: mergeAbortSignals(signal, initSignal ?? void 0)
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function isAbortError(err) {
|
|
112
|
+
return typeof err === "object" && err !== null && err.name === "AbortError";
|
|
113
|
+
}
|
|
114
|
+
function parseContentLength(headers) {
|
|
115
|
+
const raw = headers.get("content-length");
|
|
116
|
+
if (!raw) return null;
|
|
117
|
+
const size = Number(raw);
|
|
118
|
+
return Number.isFinite(size) && size >= 0 ? size : null;
|
|
119
|
+
}
|
|
120
|
+
function parseContentRangeSize(headers) {
|
|
121
|
+
const raw = headers.get("content-range");
|
|
122
|
+
if (!raw) return null;
|
|
123
|
+
const match = /^bytes\s+\d+-\d+\/(\d+)$/i.exec(raw);
|
|
124
|
+
if (!match) return null;
|
|
125
|
+
const size = Number(match[1]);
|
|
126
|
+
return Number.isFinite(size) && size >= 0 ? size : null;
|
|
127
|
+
}
|
|
128
|
+
function supportsRangeReads(headers) {
|
|
129
|
+
return headers.get("accept-ranges")?.toLowerCase().split(",").map((v) => v.trim()).includes("bytes") === true;
|
|
130
|
+
}
|
|
131
|
+
async function cancelBody(response) {
|
|
132
|
+
await response.body?.cancel().catch(() => void 0);
|
|
133
|
+
}
|
|
134
|
+
async function preflightHttpUrl(url, fetchImpl, fetchInit, signal) {
|
|
135
|
+
const head = await fetchImpl(url, fetchInitFor(fetchInit, "HEAD", signal));
|
|
136
|
+
if (head.ok) {
|
|
137
|
+
const size = parseContentLength(head.headers);
|
|
138
|
+
if (size === null) throw new Error(`HEAD ${url} missing Content-Length`);
|
|
139
|
+
if (!supportsRangeReads(head.headers)) throw new Error(`HEAD ${url} missing Accept-Ranges: bytes`);
|
|
140
|
+
return size;
|
|
141
|
+
}
|
|
142
|
+
await cancelBody(head);
|
|
143
|
+
if (![
|
|
144
|
+
403,
|
|
145
|
+
405,
|
|
146
|
+
501
|
|
147
|
+
].includes(head.status)) throw new Error(`HEAD ${url} failed: ${head.status}`);
|
|
148
|
+
const probe = await fetchImpl(url, fetchInitFor(fetchInit, "GET", signal, { Range: "bytes=0-0" }));
|
|
149
|
+
try {
|
|
150
|
+
if (probe.status !== 206) throw new Error(`range probe ${url} failed: ${probe.status}`);
|
|
151
|
+
const size = parseContentRangeSize(probe.headers);
|
|
152
|
+
if (size === null) throw new Error(`range probe ${url} missing Content-Range size`);
|
|
153
|
+
return size;
|
|
154
|
+
} finally {
|
|
155
|
+
await cancelBody(probe);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function rangeOnlyConfig(config) {
|
|
159
|
+
return {
|
|
160
|
+
...config ?? {},
|
|
161
|
+
filesystem: {
|
|
162
|
+
reliableHeadRequests: true,
|
|
163
|
+
allowFullHTTPReads: false,
|
|
164
|
+
...config?.filesystem ?? {},
|
|
165
|
+
forceFullHTTPReads: false
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
async function dropAttachedResources(db, conn, schema, tables, files) {
|
|
170
|
+
for (const table of tables) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`);
|
|
171
|
+
if (files.length > 0) await db.dropFiles([...files]);
|
|
172
|
+
}
|
|
173
|
+
function parseMemoryLimit(value) {
|
|
174
|
+
const trimmed = value.trim();
|
|
175
|
+
if (!/^\d+(?:\.\d+)?\s*(?:B|KB|MB|GB|TB)?$/i.test(trimmed)) throw new Error(`invalid memoryLimit '${value}' — expected e.g. '512MB' or '2GB'`);
|
|
176
|
+
return trimmed;
|
|
177
|
+
}
|
|
178
|
+
async function bootDuckDBWasm(options = {}) {
|
|
179
|
+
const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger, LogLevel } = await import("@duckdb/duckdb-wasm");
|
|
180
|
+
const bundle = await selectBundle(options.bundles ?? getJsDelivrBundles());
|
|
181
|
+
const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }));
|
|
182
|
+
const worker = new Worker(workerUrl);
|
|
183
|
+
const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(LogLevel.WARNING), worker);
|
|
184
|
+
try {
|
|
185
|
+
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
186
|
+
await db.open(rangeOnlyConfig(options.config));
|
|
187
|
+
const conn = await db.connect();
|
|
188
|
+
if (options.memoryLimit !== void 0) await conn.query(`SET memory_limit='${parseMemoryLimit(options.memoryLimit)}'`);
|
|
189
|
+
return {
|
|
190
|
+
db,
|
|
191
|
+
conn
|
|
192
|
+
};
|
|
193
|
+
} catch (err) {
|
|
194
|
+
worker.terminate();
|
|
195
|
+
throw err;
|
|
196
|
+
} finally {
|
|
197
|
+
URL.revokeObjectURL(workerUrl);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async function attachParquetTables(options) {
|
|
201
|
+
const { db, conn, tables, schema = "main" } = options;
|
|
202
|
+
for (const table of tables) {
|
|
203
|
+
const names = [];
|
|
204
|
+
for (let i = 0; i < table.files.length; i++) {
|
|
205
|
+
const file = table.files[i];
|
|
206
|
+
const name = fileName(table.table, i, file.name);
|
|
207
|
+
names.push(name);
|
|
208
|
+
await db.registerFileBuffer(name, file.bytes);
|
|
209
|
+
}
|
|
210
|
+
await conn.query(readParquetViewSql(schema, table.table, names));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async function attachParquetUrlTablesResult(options) {
|
|
214
|
+
const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached, attachMode = "http", trustSizeHint = true } = options;
|
|
215
|
+
const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
|
|
216
|
+
const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
|
|
217
|
+
const byteBudget = positiveInteger(maxBytes, DEFAULT_ATTACH_MAX_BYTES, "maxBytes");
|
|
218
|
+
const attachId = nextAttachId++;
|
|
219
|
+
const flat = [];
|
|
220
|
+
const counts = {};
|
|
221
|
+
for (const [table, urls] of tables.map((t) => [t.table, t.urls])) {
|
|
222
|
+
if (urls.length === 0) continue;
|
|
223
|
+
counts[table] = urls.length;
|
|
224
|
+
for (let i = 0; i < urls.length; i++) flat.push({
|
|
225
|
+
table,
|
|
226
|
+
url: urls[i],
|
|
227
|
+
index: i
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (flat.length > fileBudget) return err(browserAttachErrors.maxFilesExceeded(flat.length, fileBudget));
|
|
231
|
+
const hintedBytes = flat.reduce((acc, item) => {
|
|
232
|
+
const hint = sizeHintFromUrl(item.url);
|
|
233
|
+
return hint === null ? acc : acc + hint;
|
|
234
|
+
}, 0);
|
|
235
|
+
if (hintedBytes > byteBudget) return err(browserAttachErrors.hintedBytesExceeded(hintedBytes, byteBudget));
|
|
236
|
+
const tableFailures = /* @__PURE__ */ new Map();
|
|
237
|
+
const budgetController = new AbortController();
|
|
238
|
+
const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
|
|
239
|
+
let plannedBytes = 0;
|
|
240
|
+
const total = flat.length;
|
|
241
|
+
const prepared = [];
|
|
242
|
+
try {
|
|
243
|
+
await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
|
|
244
|
+
if (tableFailures.has(table)) return;
|
|
245
|
+
effectiveSignal?.throwIfAborted();
|
|
246
|
+
try {
|
|
247
|
+
let bytes = null;
|
|
248
|
+
let body = null;
|
|
249
|
+
if (attachMode === "buffer") {
|
|
250
|
+
const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
|
|
251
|
+
if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
|
|
252
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
253
|
+
body = buf;
|
|
254
|
+
bytes = buf.byteLength;
|
|
255
|
+
} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
|
|
256
|
+
else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
|
|
257
|
+
plannedBytes += bytes;
|
|
258
|
+
if (plannedBytes > byteBudget) {
|
|
259
|
+
const budgetErr = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
|
|
260
|
+
budgetController.abort(budgetErr);
|
|
261
|
+
throw budgetErr;
|
|
262
|
+
}
|
|
263
|
+
effectiveSignal?.throwIfAborted();
|
|
264
|
+
prepared.push({
|
|
265
|
+
table,
|
|
266
|
+
url,
|
|
267
|
+
index,
|
|
268
|
+
name: attachFileName(attachId, table, index),
|
|
269
|
+
bytes,
|
|
270
|
+
body
|
|
271
|
+
});
|
|
272
|
+
} catch (err) {
|
|
273
|
+
if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
|
|
274
|
+
tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
} catch (downloadErr) {
|
|
278
|
+
if (downloadErr instanceof BrowserAttachBudgetExceededError && !signal?.aborted) return err(browserAttachErrors.plannedBytesExceeded(plannedBytes, byteBudget));
|
|
279
|
+
throw downloadErr;
|
|
280
|
+
}
|
|
281
|
+
const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
|
|
282
|
+
const attached = [];
|
|
283
|
+
const registeredFiles = [];
|
|
284
|
+
try {
|
|
285
|
+
for (const file of prepared) {
|
|
286
|
+
if (tableFailures.has(file.table)) continue;
|
|
287
|
+
effectiveSignal?.throwIfAborted();
|
|
288
|
+
if (file.body !== null) await db.registerFileBuffer(file.name, file.body);
|
|
289
|
+
else await db.registerFileURL(file.name, file.url, DuckDBDataProtocol.HTTP, false);
|
|
290
|
+
registeredFiles.push(file.name);
|
|
291
|
+
onFileAttached?.({
|
|
292
|
+
table: file.table,
|
|
293
|
+
index: file.index,
|
|
294
|
+
total
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
for (const table of Object.keys(counts)) {
|
|
298
|
+
if (tableFailures.has(table)) continue;
|
|
299
|
+
const files = prepared.filter((file) => file.table === table).sort((a, b) => a.index - b.index);
|
|
300
|
+
if (files.length !== counts[table]) continue;
|
|
301
|
+
effectiveSignal?.throwIfAborted();
|
|
302
|
+
await conn.query(readParquetViewSql(schema, table, files.map((file) => file.name)));
|
|
303
|
+
attached.push(table);
|
|
304
|
+
}
|
|
305
|
+
} catch (err) {
|
|
306
|
+
await dropAttachedResources(db, conn, schema, attached, registeredFiles).catch((cleanupErr) => {
|
|
307
|
+
console.warn("[gscdump/engine-duckdb-wasm] cleanup after failed attach failed", cleanupErr);
|
|
308
|
+
});
|
|
309
|
+
throw err;
|
|
310
|
+
}
|
|
311
|
+
if (tableFailures.size > 0) for (const [table, err] of tableFailures) console.warn(`[gscdump/engine-duckdb-wasm] dropped table "${table}" — ${err.message}`);
|
|
312
|
+
let detached = false;
|
|
313
|
+
return ok({
|
|
314
|
+
version,
|
|
315
|
+
tables: attached,
|
|
316
|
+
schema,
|
|
317
|
+
async detach() {
|
|
318
|
+
if (detached) return;
|
|
319
|
+
detached = true;
|
|
320
|
+
await dropAttachedResources(db, conn, schema, attached, registeredFiles);
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
async function attachParquetUrlTables(options) {
|
|
325
|
+
return unwrapResult(await attachParquetUrlTablesResult(options), browserAttachErrorToException);
|
|
326
|
+
}
|
|
327
|
+
function createBrowserAnalysisRuntime(boot, options = {}) {
|
|
328
|
+
const { db, conn } = boot;
|
|
329
|
+
const schema = options.schema ?? "main";
|
|
330
|
+
let version = options.version;
|
|
331
|
+
let attachedTables = options.attachedTables;
|
|
332
|
+
let chain = Promise.resolve();
|
|
333
|
+
function abortError(signal) {
|
|
334
|
+
return signal.reason ?? new DOMException("aborted", "AbortError");
|
|
335
|
+
}
|
|
336
|
+
function raceSignal(promise, signal) {
|
|
337
|
+
if (!signal) return promise;
|
|
338
|
+
if (signal.aborted) return Promise.reject(abortError(signal));
|
|
339
|
+
return new Promise((resolve, reject) => {
|
|
340
|
+
const onAbort = () => reject(abortError(signal));
|
|
341
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
342
|
+
promise.then((value) => {
|
|
343
|
+
signal.removeEventListener("abort", onAbort);
|
|
344
|
+
resolve(value);
|
|
345
|
+
}, (err) => {
|
|
346
|
+
signal.removeEventListener("abort", onAbort);
|
|
347
|
+
reject(err);
|
|
348
|
+
});
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
function runExclusive(signal, work) {
|
|
352
|
+
const next = chain.then(work, work);
|
|
353
|
+
chain = next.then(() => void 0, () => void 0);
|
|
354
|
+
return raceSignal(next, signal);
|
|
355
|
+
}
|
|
356
|
+
async function cancelOnAbort(signal, work) {
|
|
357
|
+
if (!signal) return work;
|
|
358
|
+
if (signal.aborted) throw abortError(signal);
|
|
359
|
+
const onAbort = () => {
|
|
360
|
+
conn.cancelSent().catch((error) => {
|
|
361
|
+
console.warn("[gscdump/engine-duckdb-wasm] failed to cancel aborted query", error);
|
|
362
|
+
});
|
|
363
|
+
};
|
|
364
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
365
|
+
try {
|
|
366
|
+
return await work;
|
|
367
|
+
} finally {
|
|
368
|
+
signal.removeEventListener("abort", onAbort);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
async function runParameterizedDirect(sql, params, signal) {
|
|
372
|
+
signal?.throwIfAborted();
|
|
373
|
+
return cancelOnAbort(signal, (async () => {
|
|
374
|
+
if (!params || params.length === 0) return conn.query(sql);
|
|
375
|
+
const stmt = await conn.prepare(sql);
|
|
376
|
+
try {
|
|
377
|
+
return await stmt.query(...params);
|
|
378
|
+
} finally {
|
|
379
|
+
await stmt.close();
|
|
380
|
+
}
|
|
381
|
+
})());
|
|
382
|
+
}
|
|
383
|
+
return {
|
|
384
|
+
db,
|
|
385
|
+
conn,
|
|
386
|
+
async query(sql, params, signal) {
|
|
387
|
+
const t0 = performance.now();
|
|
388
|
+
return {
|
|
389
|
+
rows: arrowToRows(await runExclusive(signal, () => runParameterizedDirect(sql, params, signal))),
|
|
390
|
+
queryMs: performance.now() - t0
|
|
391
|
+
};
|
|
392
|
+
},
|
|
393
|
+
async analyze(params, registry, options = {}) {
|
|
394
|
+
const signal = options.signal;
|
|
395
|
+
const run = async () => {
|
|
396
|
+
signal?.throwIfAborted();
|
|
397
|
+
const t0 = performance.now();
|
|
398
|
+
const result = await runAnalyzerFromSource(createAttachedTableSource({ query: async (sql, bindParams, innerSignal) => {
|
|
399
|
+
return arrowToRows(await runParameterizedDirect(sql, bindParams, innerSignal ?? signal));
|
|
400
|
+
} }, {
|
|
401
|
+
schema,
|
|
402
|
+
signal,
|
|
403
|
+
attachedTables,
|
|
404
|
+
adapter: pgResolverAdapter
|
|
405
|
+
}), params, registry);
|
|
406
|
+
return {
|
|
407
|
+
results: result.results,
|
|
408
|
+
meta: result.meta,
|
|
409
|
+
queryMs: performance.now() - t0
|
|
410
|
+
};
|
|
411
|
+
};
|
|
412
|
+
return runExclusive(signal, run);
|
|
413
|
+
},
|
|
414
|
+
isStale(expected) {
|
|
415
|
+
return expected !== version;
|
|
416
|
+
},
|
|
417
|
+
setVersion(next) {
|
|
418
|
+
version = next;
|
|
419
|
+
},
|
|
420
|
+
setAttachedTables(next) {
|
|
421
|
+
attachedTables = next;
|
|
422
|
+
},
|
|
423
|
+
async close() {
|
|
424
|
+
await conn.close();
|
|
425
|
+
await db.terminate();
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
export { BrowserAttachBudgetExceededError, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, createBrowserAnalysisRuntime, isBrowserAttachError };
|
package/dist/schema.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine-duckdb-wasm",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.4.
|
|
4
|
+
"version": "1.4.1",
|
|
5
5
|
"description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -45,9 +45,9 @@
|
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"drizzle-orm": "1.0.0-rc.3",
|
|
48
|
-
"@gscdump/contracts": "^1.4.
|
|
49
|
-
"@gscdump/engine": "^1.4.
|
|
50
|
-
"gscdump": "^1.4.
|
|
48
|
+
"@gscdump/contracts": "^1.4.1",
|
|
49
|
+
"@gscdump/engine": "^1.4.1",
|
|
50
|
+
"gscdump": "^1.4.1"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@duckdb/duckdb-wasm": "1.33.1-dev57.0",
|