duckfn-docs-kit 0.2.1 → 0.3.0

package/src/sql/harness.ts

@@ -0,0 +1,134 @@
+/**
+ * `duckfn-docs-kit/sql/harness` — the browser page the offline SQL verifier
+ * (`sql/browserRunner`) drives with Playwright.
+ *
+ * It is deliberately thin: it reuses the site's own runtime (`sql/runtime`) so
+ * a verified block runs through exactly the code path a reader's **Run** click
+ * does — same `AsyncDuckDB` wiring, same ordered extension preload, same
+ * "multi-statement returns the last result" `execute()`. The only difference is
+ * where the engine comes from: instead of the jsDelivr CDN, the runner serves
+ * `duckdb-*.wasm` and its worker script same-origin and hands those URLs in
+ * through {@link window.DFK_HARNESS_BUNDLE}, which `runtime.init` accepts as a
+ * {@link LocalBundle}.
+ *
+ * The two contracts the page reads are injected by the served `harness.html`:
+ *
+ * - `window.DFK_HARNESS_BUNDLE` — `{mainModule, mainWorker, pthreadWorker?}`,
+ *   the locally-served engine bundle (an absolute same-origin URL each).
+ * - the `<script id="dfk-sql-runtime">` JSON tag — the preload list and
+ *   `allowUnsignedExtensions`, in the very shape the site plugin injects
+ *   (see `sql/runtimeConfig`), so nothing here re-implements extension loading.
+ *
+ * The runner communicates through `window.__dfkReady`, `window.__dfkRun` and
+ * `window.__dfkQuery`, each reached with a Playwright `page.evaluate` (which
+ * awaits the returned promise and marshals the value back to Node). Page reload
+ * per content file is what gives "one instance per page": a fresh document
+ * resets the module-level singleton, so pages never share a DuckDB instance
+ * while the blocks of one page keep sharing its connection (via
+ * `DuckDBRuntime.execute`).
+ */
+import {DuckDBRuntime, type LocalBundle} from './runtime';
+
+declare global {
+  interface Window {
+    /** The locally-served engine bundle, injected by the runner's harness.html. */
+    DFK_HARNESS_BUNDLE?: LocalBundle;
+    /** Resolves once the instance is up and the site preloads have loaded. */
+    __dfkReady?: Promise<void>;
+    /** Runs one block and reports rows/columns, or a captured error message. */
+    __dfkRun?: (sql: string) => Promise<HarnessRunResult>;
+    /**
+     * Runs one statement and returns its actual result — the rows, not just a
+     * count. Used by the vfs browser probe (`browserRunner.query`) to read the
+     * value each statement produced; the docs verifier never needs it.
+     */
+    __dfkQuery?: (sql: string) => Promise<HarnessQueryResult>;
+  }
+}
+
+/** What {@link window.__dfkRun} resolves with; mirrors the runner's `RunResult`. */
+export interface HarnessRunResult {
+  rows: number;
+  columns: number;
+  /** Set when the block failed; `rows`/`columns` are then 0. */
+  error?: string;
+}
+
+/** What {@link window.__dfkQuery} resolves with: the marshalled result rows. */
+export interface HarnessQueryResult {
+  columns: string[];
+  rows: Record<string, unknown>[];
+  error?: string;
+}
+
+async function ready(): Promise<void> {
+  const bundle = window.DFK_HARNESS_BUNDLE;
+  if (!bundle) {
+    throw new Error('sql/harness: window.DFK_HARNESS_BUNDLE is not set');
+  }
+  const runtime = DuckDBRuntime.getInstance();
+  const allowUnsigned = readAllowUnsigned();
+  await runtime.init({bundle, allowUnsignedExtensions: allowUnsigned});
+}
+
+/**
+ * `allowUnsignedExtensions` is fixed at `open()` time, so the harness has to
+ * hand it to `init()` directly rather than let the runtime read it lazily from
+ * the config tag. Reading the same tag here (idempotently) keeps the injected
+ * config the single source of truth — the runtime still reads the preload list
+ * from it inside `#create()`.
+ */
+function readAllowUnsigned(): boolean {
+  const text = document
+    .getElementById('dfk-sql-runtime')
+    ?.textContent?.trim();
+  if (!text) {
+    return false;
+  }
+  try {
+    const parsed = JSON.parse(text) as {allowUnsignedExtensions?: boolean};
+    return parsed.allowUnsignedExtensions === true;
+  } catch {
+    return false;
+  }
+}
+
+async function run(sql: string): Promise<HarnessRunResult> {
+  const result = await DuckDBRuntime.getInstance().execute(sql);
+  if (result.error) {
+    return {rows: 0, columns: 0, error: result.error};
+  }
+  return {rows: result.rows.length, columns: result.columns.length};
+}
+
+async function query(sql: string): Promise<HarnessQueryResult> {
+  const result = await DuckDBRuntime.getInstance().execute(sql);
+  if (result.error) {
+    return {columns: [], rows: [], error: result.error};
+  }
+  // Rebuild each row as a plain, own-enumerable object. `execute()` hands back
+  // Apache-Arrow row objects whose values sit behind getters, so Playwright's
+  // `page.evaluate` serializer would marshal them to `{}`; reading the getters
+  // here, in-page, and copying to a literal survives the hop. BigInt values
+  // (DuckDB integers) become strings for the same reason.
+  const rows = result.rows.map((row) => {
+    const plain: Record<string, unknown> = {};
+    for (const column of result.columns) {
+      const value = row[column];
+      plain[column] =
+        typeof value === 'bigint'
+          ? value.toString()
+          : value === null || value === undefined
+            ? null
+            : typeof value === 'object'
+              ? JSON.stringify(value)
+              : value;
+    }
+    return plain;
+  });
+  return {columns: result.columns, rows};
+}
+
+window.__dfkReady = ready();
+window.__dfkRun = run;
+window.__dfkQuery = query;

package/src/sql/runtime.ts

@@ -65,6 +65,27 @@ export type RuntimeState = 'idle' | 'loading' | 'ready' | 'error';
 export interface RuntimeOptions {
   /** Let `LOAD` accept an extension whose signature does not verify. */
   allowUnsignedExtensions?: boolean;
+  /**
+   * Serve the DuckDB-Wasm engine from explicit same-origin URLs instead of the
+   * jsDelivr CDN. Used by the offline SQL verifier (`sql/browserRunner`): the
+   * harness passes the locally-served `duckdb-*.wasm` / worker script so a CI
+   * run never reaches the network. When set, `#create()` skips
+   * `getJsDelivrBundles()`/`selectBundle()` and the cross-origin blob-worker
+   * wrapper (the worker is same-origin here, so it is constructed directly).
+   */
+  bundle?: LocalBundle;
+}
+
+/**
+ * A locally-served DuckDB-Wasm bundle: absolute same-origin URLs for the
+ * engine wasm and its worker script. `pthreadWorker` is only needed for the
+ * cross-origin-isolated (COI) bundle; the default non-COI `eh`/`mvp` bundles run
+ * single-threaded and leave it unset.
+ */
+export interface LocalBundle {
+  mainModule: string;
+  mainWorker: string;
+  pthreadWorker?: string;
 }
 
 /** Options for {@link DuckDBRuntime.loadExtension}. */
@@ -96,6 +117,8 @@ export class DuckDBRuntime {
   #db: DuckdbWasm.AsyncDuckDB | null = null;
   #conn: DuckdbWasm.AsyncDuckDBConnection | null = null;
   #allowUnsigned = false;
+  /** A locally-served engine bundle (offline verifier); `null` means CDN. */
+  #bundle: LocalBundle | null = null;
   /** The injected site config; read (and validated) once on first init. */
   #site: SiteRuntimeConfig | null = null;
   /** Loaded / in-flight extensions, keyed by repository + name, or by URL. */
@@ -125,6 +148,9 @@ export class DuckDBRuntime {
     if (options.allowUnsignedExtensions) {
       this.#allowUnsigned = true;
     }
+    if (options.bundle) {
+      this.#bundle = options.bundle;
+    }
     if (this.#state === 'ready') {
       return Promise.resolve();
     }
@@ -154,6 +180,24 @@ export class DuckDBRuntime {
 
   async #create(preload: readonly PreloadEntry[]): Promise<void> {
     const duckdb = await import('@duckdb/duckdb-wasm');
+
+    // Offline verifier path: the engine and its worker are served same-origin
+    // by the runner, so the worker is constructed directly (no cross-origin
+    // blob wrapper) and no CDN bundle is selected.
+    if (this.#bundle) {
+      const worker = new Worker(this.#bundle.mainWorker);
+      this.#db = new duckdb.AsyncDuckDB(new duckdb.ConsoleLogger(), worker);
+      await this.#db.instantiate(this.#bundle.mainModule, this.#bundle.pthreadWorker ?? null);
+      await this.#db.open({allowUnsignedExtensions: this.#allowUnsigned});
+      this.#conn = await this.#db.connect();
+      for (const entry of preload) {
+        await this.#loadEntry(entry);
+      }
+      this.#state = 'ready';
+      this.#message = '';
+      return;
+    }
+
     const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
     if (!bundle.mainWorker) {
       throw new Error('The selected DuckDB-Wasm bundle has no worker script');

package/src/sql/verify.ts

@@ -3,10 +3,10 @@
  * publishes, so a broken example is caught by CI instead of by a reader
  * clicking **Run**.
  *
- * It is the same environment the site gives a block: DuckDB-Wasm in a worker,
+ * It is the same environment the site gives a block: DuckDB-Wasm in a browser,
  * the site's extension preloaded, one instance per page and the page's blocks
- * sharing a connection (see `sql/nodeRunner.ts` for why it is this target and
- * not the blocking one).
+ * sharing a connection (see `sql/browserRunner.ts` for why this is a real
+ * browser and not the Node worker the kit used to run).
  *
  * A block may fail *on purpose* — half the guide ends on a statement that
  * demonstrates an error. Such a block says so in its own metadata
@@ -15,12 +15,11 @@
  * on has to be data rather than a string match on a comment. Only blocks that
  * did not behave as declared make the command exit non-zero.
  */
-import {mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync} from 'node:fs';
-import {tmpdir} from 'node:os';
+import {readdirSync, statSync, writeFileSync} from 'node:fs';
 import {join, resolve} from 'node:path';
 
 import {collectRunnableSql, expectsError, type RunnableSqlBlock} from './collect';
-import {WasmSqlRunner, type WasmPlatform} from './nodeRunner';
+import {BrowserSqlRunner, type WasmPlatform} from './browserRunner';
 
 export interface VerifyOptions {
   /** Docs site root; defaults to the working directory. */
@@ -39,11 +38,10 @@ export interface VerifyOptions {
   /** Per-block timeout in milliseconds; a hang is reported instead of blocking CI. */
   timeoutMs?: number;
   /**
-   * Directory the blocks run in. Defaults to a fresh temporary directory that
-   * is removed afterwards: a block may `COPY … TO 'a.csv'`, and on Node
-   * DuckDB's file system is the real one, relative to the working directory.
+   * The browser executable that runs the blocks. Defaults to a detected
+   * Chrome/Edge; the `DFK_BROWSER` environment variable is the same override.
    */
-  workingDir?: string;
+  browser?: string;
   /** Write the full result list here as JSON. */
   reportFile?: string;
 }
@@ -79,8 +77,6 @@ const DEFAULT_TIMEOUT_MS = 30_000;
 
 /** Runs every runnable block of the site and returns the outcome of each. */
 export async function verifySqlDocs(options: VerifyOptions = {}): Promise<VerifyReport> {
-  // Everything is resolved to absolute paths first: the run changes the working
-  // directory (see below).
   const siteDir = resolve(options.siteDir ?? process.cwd());
   const blocks = collectRunnableSql({
     siteDir,
@@ -94,26 +90,10 @@ export async function verifySqlDocs(options: VerifyOptions = {}): Promise<Verify
     : defaultExtension(siteDir);
   const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 
-  // A block may write files — `COPY (SELECT 1) TO 'a.csv'` — and on Node the
-  // file system behind DuckDB is the real one, resolved against the working
-  // directory. Giving the run a scratch directory of its own keeps the docs tree
-  // clean instead of dropping test residue into it.
-  const scratch = options.workingDir
-    ? resolve(options.workingDir)
-    : mkdtempSync(join(tmpdir(), 'duckfn-sql-verify-'));
-  mkdirSync(scratch, {recursive: true});
-  const previousCwd = process.cwd();
-  process.chdir(scratch);
-
-  let results: BlockResult[];
-  try {
-    results = await runPages(blocks, extension, options, timeoutMs);
-  } finally {
-    process.chdir(previousCwd);
-    if (!options.workingDir) {
-      rmSync(scratch, {recursive: true, force: true});
-    }
-  }
+  // No working directory: in a browser DuckDB's file system is the instance's
+  // own memory, so `COPY … TO` / `dfn_file_write_*` never touch the docs tree —
+  // they land in the page and vanish on the next `newPage()`.
+  const results = await runPages(blocks, extension, options, timeoutMs);
 
   const report: VerifyReport = {
     blocks: results,
@@ -135,10 +115,11 @@ async function runPages(
   options: VerifyOptions,
   timeoutMs: number,
 ): Promise<BlockResult[]> {
-  const runner = await WasmSqlRunner.create({
+  const runner = await BrowserSqlRunner.create({
     extension,
     platform: options.platform,
     engine: options.engine,
+    browser: options.browser,
   });
   const results: BlockResult[] = [];
   try {
@@ -158,7 +139,7 @@ async function runPages(
 
 /** Runs one block and judges the result against the block's own declaration. */
 async function runBlock(
-  runner: WasmSqlRunner,
+  runner: BrowserSqlRunner,
   block: RunnableSqlBlock,
   timeoutMs: number,
 ): Promise<BlockResult> {
@@ -278,9 +259,9 @@ export async function cliMain(argv: readonly string[]): Promise<void> {
 
 const USAGE = `Usage: duckfn-sql-verify [options]
 
-Runs every runnable SQL block of a duckfn docs site in DuckDB-Wasm, and checks
-that each one behaves as its own metadata declares ("expect": "error" for a
-block that demonstrates a failure).
+Runs every runnable SQL block of a duckfn docs site in a headless browser
+(DuckDB-Wasm), and checks that each one behaves as its own metadata declares
+("expect": "error" for a block that demonstrates a failure).
 
   --site <dir>          Docs site root (default: the working directory)
   --content <dir>       Content directory, relative to the site root (repeatable;
@@ -291,8 +272,9 @@ block that demonstrates a failure).
   --platform <eh|mvp>   DuckDB-Wasm bundle, which must match the extension build
                         (default: eh)
   --engine <path>       Engine wasm override
+  --browser <path>      Browser executable (default: a detected Chrome/Edge,
+                        or the DFK_BROWSER environment variable)
   --timeout <ms>        Per-block timeout (default: 30000)
-  --working-dir <dir>   Directory the blocks run in (default: a temporary one)
   --report <file>       Write the full result list as JSON
   --quiet               Only report unexpected behaviour
   --help                Show this help
@@ -330,12 +312,12 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
       case '--engine':
         options.engine = next();
         break;
+      case '--browser':
+        options.browser = next();
+        break;
       case '--timeout':
         options.timeoutMs = Number(next());
         break;
-      case '--working-dir':
-        options.workingDir = next();
-        break;
       case '--report':
         options.reportFile = next();
         break;

package/dist/sql/nodeRunner.d.ts

@@ -1,51 +0,0 @@
-export interface DuckdbQueryResult {
-    numRows: number;
-    schema: {
-        fields: {
-            name: string;
-        }[];
-    };
-}
-export interface DuckdbConnection {
-    query(sql: string): Promise<DuckdbQueryResult>;
-}
-export type WasmPlatform = 'eh' | 'mvp';
-export interface RunnerOptions {
-    /**
-     * The extension to `LOAD`: a local `.duckdb_extension.wasm` path (served to
-     * the worker over a loopback http server) or an absolute `http(s)` URL.
-     */
-    extension: string;
-    /**
-     * DuckDB-Wasm platform. Must match how the extension was built — a site
-     * serving `duckfn-wasm_eh.duckdb_extension.wasm` runs the `eh` bundle.
-     */
-    platform?: WasmPlatform;
-    /** The engine wasm; defaults to the one shipped beside the worker bundle. */
-    engine?: string;
-}
-export interface RunResult {
-    rows: number;
-    columns: number;
-}
-/**
- * One DuckDB-Wasm instance at a time, driven from Node.
- *
- * `newPage()` is what a docs site does per page load: a fresh instance, a fresh
- * connection, the extension loaded again. Blocks of one page then share state
- * (a table created in one block is visible to the next), while pages stay
- * isolated — which is why the runner is used one page at a time rather than
- * over a single long-lived connection.
- */
-export declare class WasmSqlRunner {
-    #private;
-    private constructor();
-    static create(options: RunnerOptions): Promise<WasmSqlRunner>;
-    /** Drops the current instance and starts a fresh page: new instance, new connection. */
-    newPage(): Promise<void>;
-    /** Runs one block; the caller decides whether a failure is expected. */
-    run(sql: string): Promise<RunResult>;
-    /** Where the fetched extension was staged, for diagnostics. */
-    get stagingDir(): string | null;
-    close(): Promise<void>;
-}

package/dist/sql/nodeRunner.js

@@ -1,115 +0,0 @@
-import { createRequire as e } from "node:module";
-import { basename as t, dirname as n, join as r } from "node:path";
-import { mkdirSync as i, readFileSync as a, rmSync as o } from "node:fs";
-import { createServer as s } from "node:http";
-import { Worker as c } from "node:worker_threads";
-import { homedir as l } from "node:os";
-//#region src/sql/nodeRunner.ts
-var u = e(import.meta.url), d = "@duckdb/duckdb-wasm/dist/duckdb-node.cjs", f = class {
-	#e;
-	#t = /* @__PURE__ */ new Map();
-	onmessage = null;
-	onerror = null;
-	onclose = null;
-	constructor(e) {
-		this.#e = new c(u.resolve(d), { workerData: {
-			mod: e,
-			name: "",
-			type: ""
-		} }), this.#e.on("message", (e) => this.#n("message", e)), this.#e.on("error", (e) => this.#n("error", e)), this.#e.on("exit", () => this.#n("close"));
-	}
-	#n(e, t) {
-		let n = {
-			type: e,
-			data: t,
-			target: this,
-			currentTarget: this
-		}, r = this[`on${e}`];
-		typeof r == "function" && r(n);
-		for (let t of this.#t.get(e) ?? []) t(n);
-	}
-	addEventListener(e, t) {
-		this.#t.set(e, [...this.#t.get(e) ?? [], t]);
-	}
-	removeEventListener(e, t) {
-		this.#t.set(e, (this.#t.get(e) ?? []).filter((e) => e !== t));
-	}
-	postMessage(e, t) {
-		this.#e.postMessage(e, t ?? []);
-	}
-	terminate() {
-		return this.#e.terminate();
-	}
-}, p = class e {
-	#e = null;
-	#t = null;
-	#n = null;
-	#r = null;
-	#i;
-	#a;
-	#o;
-	constructor(e, t, n) {
-		this.#i = e, this.#a = t, this.#o = n;
-	}
-	static async create(t) {
-		let i = t.platform ?? "eh", a = u.resolve(`@duckdb/duckdb-wasm/dist/duckdb-node-${i}.worker.cjs`), o = t.engine ?? r(n(a), `duckdb-${i}.wasm`), s = /^https?:\/\//i.test(t.extension) ? null : await m(t.extension), c = s ? s.url : t.extension, l = new e(c, o, a);
-		return l.#n = s ? s.server : null, l.#r = s ? h(s.stagingHost, s.stagingSegment) : null, l;
-	}
-	async newPage() {
-		this.#e && (await this.#e.terminate(), this.#e = null, this.#t = null);
-		let e = await import(
-			/* @vite-ignore */
-			d
-), t = new e.AsyncDuckDB(new e.ConsoleLogger(), new f(this.#o));
-		await t.instantiate(this.#a, null), await t.open({ allowUnsignedExtensions: !0 });
-		let n = await t.connect();
-		await n.query(`LOAD '${this.#i}'`), this.#e = t, this.#t = n;
-	}
-	async run(e) {
-		let t = this.#t;
-		if (!t) throw Error("sql/verify: no page is open — call newPage() first");
-		let n = await t.query(e);
-		return {
-			rows: n.numRows,
-			columns: n.schema.fields.length
-		};
-	}
-	get stagingDir() {
-		return this.#r;
-	}
-	async close() {
-		this.#e && (await this.#e.terminate(), this.#e = null, this.#t = null), this.#n &&= (await new Promise((e) => this.#n?.close(() => e())), null);
-	}
-};
-async function m(e) {
-	let n = a(e), r = t(e), i = g(e), o = s((e, t) => {
-		t.writeHead(200, {
-			"content-type": "application/octet-stream",
-			"content-length": n.length
-		}), t.end(n);
-	}), c = process.platform === "win32" ? 80 : 0;
-	await new Promise((e, t) => {
-		o.once("error", (e) => {
-			t(/* @__PURE__ */ Error(`sql/verify: cannot serve the extension on port ${c} (${e.code}): a port in the URL would put a colon into DuckDB's staging path, which is not a legal Windows path — free port 80, or load the extension from an http(s) URL with --extension`));
-		}), o.listen(c, "localhost", e);
-	});
-	let l = o.address(), u = typeof l == "object" && l ? l.port : c, d = u === 80 ? "localhost" : `localhost:${u}`;
-	return {
-		server: o,
-		url: `http://${d}/${i}/${r}`,
-		stagingHost: d,
-		stagingSegment: i
-	};
-}
-function h(e, t) {
-	let n = r(l(), ".duckdb", "extensions", e, t);
-	return o(n, {
-		recursive: !0,
-		force: !0
-	}), i(n, { recursive: !0 }), n;
-}
-function g(e) {
-	return t(e).split(".")[0] ?? "";
-}
-//#endregion
-export { p as WasmSqlRunner };