duckfn-docs-kit 0.1.0 → 0.2.1

package/src/sql/collect.ts

@@ -0,0 +1,136 @@
+/**
+ * Node-side collector for runnable SQL blocks: walks a docs site's content and
+ * returns every fenced block whose info string is a runnable config.
+ *
+ * The metastring contract belongs to `sql/remark.ts` (one parser, exported as
+ * `parseRunnableSqlMeta`), so the blocks collected here are exactly the ones
+ * the build turns into `<dfk-sql>` — a CI check over this list therefore covers
+ * what the site actually publishes.
+ *
+ * Fences follow CommonMark closely enough for a docs tree: a closing fence has
+ * to use the same character, be at least as long, and carry no info string.
+ * That is what keeps a ```sql example *inside* a ````md wrapper (as
+ * `docs-kit/runnable-sql.md` shows the metastring) from being collected as a
+ * block of its own.
+ */
+import {readFileSync, readdirSync, statSync} from 'node:fs';
+import {join, relative, sep} from 'node:path';
+
+import {parseRunnableSqlMeta, type RunnableSqlConfig} from './remark';
+
+/** One runnable block, positioned so a failure can name it. */
+export interface RunnableSqlBlock {
+  /** Path relative to the site root, always with forward slashes. */
+  file: string;
+  /** 1-based line of the opening fence. */
+  line: number;
+  config: RunnableSqlConfig;
+  sql: string;
+}
+
+export interface CollectRunnableSqlOptions {
+  /** Site root the reported paths are relative to, and the base of a relative dir. */
+  siteDir: string;
+  /**
+   * Directories to scan (absolute, or relative to `siteDir`). Missing ones are
+   * skipped rather than reported: a site may have no translations.
+   */
+  contentDirs: readonly string[];
+  /** File extensions to scan; both `.md` and `.mdx` are markdown to us. */
+  extensions?: readonly string[];
+}
+
+const DEFAULT_EXTENSIONS = ['.md', '.mdx'] as const;
+
+/** Every runnable block under `contentDirs`, in file order, then line order. */
+export function collectRunnableSql(options: CollectRunnableSqlOptions): RunnableSqlBlock[] {
+  const {siteDir, contentDirs, extensions = DEFAULT_EXTENSIONS} = options;
+  const files: string[] = [];
+  for (const dir of contentDirs) {
+    collectFiles(join(siteDir, dir), extensions, files);
+  }
+  const blocks: RunnableSqlBlock[] = [];
+  for (const file of files.sort()) {
+    for (const block of blocksOf(readFileSync(file, 'utf8'))) {
+      blocks.push({
+        ...block,
+        file: relative(siteDir, file).split(sep).join('/'),
+      });
+    }
+  }
+  return blocks;
+}
+
+function collectFiles(dir: string, extensions: readonly string[], out: string[]): void {
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    // Nothing to scan here (typically a translation that does not exist yet).
+    return;
+  }
+  for (const name of entries) {
+    const path = join(dir, name);
+    if (statSync(path).isDirectory()) {
+      collectFiles(path, extensions, out);
+    } else if (extensions.some((extension) => name.endsWith(extension))) {
+      out.push(path);
+    }
+  }
+}
+
+interface RawBlock {
+  line: number;
+  config: RunnableSqlConfig;
+  sql: string;
+}
+
+/** The runnable blocks of one markdown document. */
+function blocksOf(text: string): RawBlock[] {
+  const lines = text.split('\n');
+  const out: RawBlock[] = [];
+  let open: {marker: string; info: string; line: number} | null = null;
+  let body: string[] = [];
+
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index] ?? '';
+    const fence = /^(`{3,}|~{3,})(.*)$/.exec(line);
+    if (!fence) {
+      if (open) {
+        body.push(line);
+      }
+      continue;
+    }
+    const [marker, info] = [fence[1] ?? '', (fence[2] ?? '').trim()];
+    if (!open) {
+      open = {marker, info, line: index + 1};
+      body = [];
+      continue;
+    }
+    const closes =
+      marker.charAt(0) === open.marker.charAt(0) && marker.length >= open.marker.length && info === '';
+    if (!closes) {
+      // A shorter or differently marked fence is content of the open block.
+      body.push(line);
+      continue;
+    }
+    if (open.info.startsWith('sql')) {
+      const config = parseRunnableSqlMeta(open.info.slice('sql'.length).trim());
+      if (config) {
+        out.push({line: open.line, config, sql: body.join('\n')});
+      }
+    }
+    open = null;
+    body = [];
+  }
+  return out;
+}
+
+/**
+ * Whether a block is expected to fail — the block's own `"expect": "error"`
+ * metadata, never its prose or its comments: an expectation that the SQL test
+ * suite acts on has to be data, not a string match on a comment.
+ */
+export function expectsError(config: RunnableSqlConfig): boolean {
+  return config.expect === 'error';
+}

package/src/sql/nodeRunner.ts

@@ -0,0 +1,298 @@
+/**
+ * Runs runnable SQL blocks in DuckDB-Wasm from Node, on the same architecture
+ * the site itself uses: the official Node **worker** target, one instance per
+ * page, one connection per page.
+ *
+ * Three platform facts shape this file, all of them measured rather than
+ * guessed (duckfn's AGENTS.md records the investigation):
+ *
+ * 1. The official Node *blocking* target (`duckdb-node-blocking.cjs`) deadlocks
+ *    when it loads an extension that opens its own connection while registering
+ *    — which is exactly what a docs site's extension may do. The worker target
+ *    is the only Node target that works.
+ * 2. A bare file name in `LOAD` is not usable on wasm: it hangs, without an
+ *    error, even with the file registered through `registerFileBuffer`. The
+ *    extension has to be fetched over http(s), like the site does it.
+ * 3. DuckDB stages that URL under
+ *    `~/.duckdb/extensions/<host>[:<port>]/<first path segment>/` — which is
+ *    why the served URL keeps a path segment, and why the directory is
+ *    pre-created: the loader's own `mkdir` is non-recursive. A colon is illegal
+ *    in a Windows path, so the port plan is per platform: Windows takes the
+ *    default port 80 (the URL then carries no port at all), POSIX takes any
+ *    free port. The directory's previous content is removed so a stale copy can
+ *    never be what runs.
+ */
+import {createServer, type Server} from 'node:http';
+import {Worker as WorkerThread} from 'node:worker_threads';
+import {createRequire} from 'node:module';
+import {mkdirSync, readFileSync, rmSync} from 'node:fs';
+import {homedir} from 'node:os';
+import {basename, dirname, join} from 'node:path';
+
+/** The slice of the Node target's surface this runner uses (it ships no types). */
+interface DuckdbNodeTarget {
+  AsyncDuckDB: new (logger: unknown, worker: unknown) => DuckdbDatabase;
+  ConsoleLogger: new () => unknown;
+}
+
+export interface DuckdbQueryResult {
+  numRows: number;
+  schema: {fields: {name: string}[]};
+}
+
+export interface DuckdbConnection {
+  query(sql: string): Promise<DuckdbQueryResult>;
+}
+
+interface DuckdbDatabase {
+  instantiate(mainModule: string, pthreadWorker: string | null): Promise<void>;
+  open(config: {allowUnsignedExtensions?: boolean}): Promise<void>;
+  connect(): Promise<DuckdbConnection>;
+  terminate(): Promise<void>;
+}
+
+const require = createRequire(import.meta.url);
+
+/** Loaded through a variable so a bundler leaves the dynamic import alone. */
+const NODE_TARGET = '@duckdb/duckdb-wasm/dist/duckdb-node.cjs';
+
+export type WasmPlatform = 'eh' | 'mvp';
+
+export interface RunnerOptions {
+  /**
+   * The extension to `LOAD`: a local `.duckdb_extension.wasm` path (served to
+   * the worker over a loopback http server) or an absolute `http(s)` URL.
+   */
+  extension: string;
+  /**
+   * DuckDB-Wasm platform. Must match how the extension was built — a site
+   * serving `duckfn-wasm_eh.duckdb_extension.wasm` runs the `eh` bundle.
+   */
+  platform?: WasmPlatform;
+  /** The engine wasm; defaults to the one shipped beside the worker bundle. */
+  engine?: string;
+}
+
+export interface RunResult {
+  rows: number;
+  columns: number;
+}
+
+/**
+ * The WebWorker surface `AsyncDuckDB` expects, backed by a worker thread.
+ *
+ * The official `createWorker()` cannot be reused here: it fetches the worker
+ * script, wraps it in a blob URL and resolves it as a file path, which is the
+ * browser's module story. The contract it relies on internally is small —
+ * `worker_threads` entry with `workerData.mod` — and that is what this
+ * reproduces.
+ */
+class WorkerShim {
+  #thread: WorkerThread;
+  #listeners = new Map<string, ((event: unknown) => void)[]>();
+  onmessage: ((event: unknown) => void) | null = null;
+  onerror: ((event: unknown) => void) | null = null;
+  onclose: ((event: unknown) => void) | null = null;
+
+  constructor(mod: string) {
+    this.#thread = new WorkerThread(require.resolve(NODE_TARGET), {
+      workerData: {mod, name: '', type: ''},
+    });
+    this.#thread.on('message', (data) => this.#emit('message', data));
+    this.#thread.on('error', (error) => this.#emit('error', error));
+    this.#thread.on('exit', () => this.#emit('close'));
+  }
+
+  #emit(type: string, data?: unknown): void {
+    const event = {type, data, target: this, currentTarget: this};
+    const handler = (this as Record<string, unknown>)[`on${type}`];
+    if (typeof handler === 'function') {
+      (handler as (event: unknown) => void)(event);
+    }
+    for (const listener of this.#listeners.get(type) ?? []) {
+      listener(event);
+    }
+  }
+
+  addEventListener(type: string, listener: (event: unknown) => void): void {
+    this.#listeners.set(type, [...(this.#listeners.get(type) ?? []), listener]);
+  }
+
+  removeEventListener(type: string, listener: (event: unknown) => void): void {
+    this.#listeners.set(
+      type,
+      (this.#listeners.get(type) ?? []).filter((candidate) => candidate !== listener),
+    );
+  }
+
+  postMessage(data: unknown, transfer?: unknown): void {
+    // The worker_threads signature always wants the second argument.
+    this.#thread.postMessage(data, (transfer ?? []) as never);
+  }
+
+  terminate(): Promise<number> | void {
+    return this.#thread.terminate();
+  }
+}
+
+/**
+ * One DuckDB-Wasm instance at a time, driven from Node.
+ *
+ * `newPage()` is what a docs site does per page load: a fresh instance, a fresh
+ * connection, the extension loaded again. Blocks of one page then share state
+ * (a table created in one block is visible to the next), while pages stay
+ * isolated — which is why the runner is used one page at a time rather than
+ * over a single long-lived connection.
+ */
+export class WasmSqlRunner {
+  #db: DuckdbDatabase | null = null;
+  #conn: DuckdbConnection | null = null;
+  #server: Server | null = null;
+  #stagingDir: string | null = null;
+  readonly #extensionUrl: string;
+  readonly #engine: string;
+  readonly #workerBundle: string;
+
+  private constructor(extensionUrl: string, engine: string, workerBundle: string) {
+    this.#extensionUrl = extensionUrl;
+    this.#engine = engine;
+    this.#workerBundle = workerBundle;
+  }
+
+  static async create(options: RunnerOptions): Promise<WasmSqlRunner> {
+    const platform = options.platform ?? 'eh';
+    const workerBundle = require.resolve(
+      `@duckdb/duckdb-wasm/dist/duckdb-node-${platform}.worker.cjs`,
+    );
+    const engine = options.engine ?? join(dirname(workerBundle), `duckdb-${platform}.wasm`);
+
+    const remote = /^https?:\/\//i.test(options.extension);
+    const served = remote ? null : await serveExtension(options.extension);
+    const url = served ? served.url : options.extension;
+
+    const runner = new WasmSqlRunner(url, engine, workerBundle);
+    runner.#server = served ? served.server : null;
+    runner.#stagingDir = served
+      ? prepareStagingDir(served.stagingHost, served.stagingSegment)
+      : null;
+    return runner;
+  }
+
+  /** Drops the current instance and starts a fresh page: new instance, new connection. */
+  async newPage(): Promise<void> {
+    if (this.#db) {
+      await this.#db.terminate();
+      this.#db = null;
+      this.#conn = null;
+    }
+    const duckdb = (await import(/* @vite-ignore */ NODE_TARGET)) as unknown as DuckdbNodeTarget;
+    const db = new duckdb.AsyncDuckDB(new duckdb.ConsoleLogger(), new WorkerShim(this.#workerBundle));
+    await db.instantiate(this.#engine, null);
+    await db.open({allowUnsignedExtensions: true});
+    const conn = await db.connect();
+    await conn.query(`LOAD '${this.#extensionUrl}'`);
+    this.#db = db;
+    this.#conn = conn;
+  }
+
+  /** Runs one block; the caller decides whether a failure is expected. */
+  async run(sql: string): Promise<RunResult> {
+    const conn = this.#conn;
+    if (!conn) {
+      throw new Error('sql/verify: no page is open — call newPage() first');
+    }
+    const table = await conn.query(sql);
+    return {rows: table.numRows, columns: table.schema.fields.length};
+  }
+
+  /** Where the fetched extension was staged, for diagnostics. */
+  get stagingDir(): string | null {
+    return this.#stagingDir;
+  }
+
+  async close(): Promise<void> {
+    if (this.#db) {
+      await this.#db.terminate();
+      this.#db = null;
+      this.#conn = null;
+    }
+    if (this.#server) {
+      await new Promise<void>((resolve) => this.#server?.close(() => resolve()));
+      this.#server = null;
+    }
+  }
+}
+
+/**
+ * Serves one extension file over loopback http, because wasm `LOAD` only works
+ * with a URL (see the file header). The server answers any path with that one
+ * file: it is a local, short-lived convenience, not a web server.
+ *
+ * The port choice is the platform split from the file header: the default port
+ * where the platform lets a non-root process take it (so the URL stays
+ * port-less, which is what Windows needs), any free port otherwise.
+ */
+async function serveExtension(file: string): Promise<{
+  server: Server;
+  url: string;
+  /** The directory DuckDB will stage the download under, and its parent. */
+  stagingHost: string;
+  stagingSegment: string;
+}> {
+  const body = readFileSync(file);
+  const name = basename(file);
+  // DuckDB stages a fetched extension under
+  // `~/.duckdb/extensions/<host>/<first path segment>/`, so the URL needs one
+  // path segment — the site serves its extension from `duckdb-extensions/`, and
+  // this mirrors that shape with the extension's own name.
+  const segment = extensionName(file);
+  const server = createServer((_request, response) => {
+    response.writeHead(200, {
+      'content-type': 'application/octet-stream',
+      'content-length': body.length,
+    });
+    response.end(body);
+  });
+  const wanted = process.platform === 'win32' ? 80 : 0;
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', (error: NodeJS.ErrnoException) => {
+      reject(
+        new Error(
+          `sql/verify: cannot serve the extension on port ${wanted} (${error.code}): a port in ` +
+            `the URL would put a colon into DuckDB's staging path, which is not a legal Windows ` +
+            `path — free port 80, or load the extension from an http(s) URL with --extension`,
+        ),
+      );
+    });
+    server.listen(wanted, 'localhost', resolve);
+  });
+  const address = server.address();
+  const bound = typeof address === 'object' && address ? address.port : wanted;
+  const host = bound === 80 ? 'localhost' : `localhost:${bound}`;
+  return {
+    server,
+    url: `http://${host}/${segment}/${name}`,
+    stagingHost: host,
+    stagingSegment: segment,
+  };
+}
+
+/**
+ * Pre-creates DuckDB's staging directory and empties it.
+ *
+ * The directory has to exist beforehand (the loader's own `mkdir` is
+ * non-recursive), and anything left in it from an earlier run would be reused
+ * instead of the extension being fetched again — a stale binary silently
+ * testing the wrong build.
+ */
+function prepareStagingDir(host: string, segment: string): string {
+  const dir = join(homedir(), '.duckdb', 'extensions', host, segment);
+  rmSync(dir, {recursive: true, force: true});
+  mkdirSync(dir, {recursive: true});
+  return dir;
+}
+
+/** The text before the first dot of the file name — the entry symbol DuckDB looks up. */
+function extensionName(file: string): string {
+  return basename(file).split('.')[0] ?? '';
+}

package/src/sql/remark.ts

@@ -38,6 +38,17 @@ export interface RunnableSqlConfig {
    * iframe, so scripts run with an opaque origin.
    */
   show?: 'table' | 'html' | 'iframe' | 'svg' | 'text';
+  /**
+   * What this block is expected to do when the docs' own SQL test suite runs it
+   * (`duckfn-docs-kit/sql/verify`). Defaults to `'ok'`; `'error'` marks a block
+   * that demonstrates a failure — the suite then *requires* it to fail, and
+   * reports it when it unexpectedly succeeds instead.
+   *
+   * This is the only source of truth for the expectation: the prose around a
+   * block, and a `-- error: …` comment inside it, are there for readers, and
+   * neither is machine-checked.
+   */
+  expect?: 'ok' | 'error';
   /**
    * The column holding the markup, for the preview renderers. A single-column
    * result is unambiguous and is used as-is.
@@ -102,8 +113,15 @@ interface ParentNode {
   children?: unknown[];
 }
 
-/** Parse the metastring; `null` means "not a runnable block, leave it alone". */
-function parseConfig(meta: string | null | undefined): RunnableSqlConfig | null {
+/**
+ * Parse the metastring; `null` means "not a runnable block, leave it alone".
+ *
+ * Exported because the block contract has two consumers: this plugin, which
+ * turns a block into `<dfk-sql>` at build time, and `sql/verify` (via
+ * `sql/collect`), which runs those same blocks in CI. Both have to agree on
+ * what counts as runnable, so there is one parser.
+ */
+export function parseRunnableSqlMeta(meta: string | null | undefined): RunnableSqlConfig | null {
   if (!meta) {
     return null;
   }
@@ -199,7 +217,7 @@ export const remarkRunnableSql: Plugin<[RunnableSqlOptions?]> =
         }
         const candidate = child as CodeNode;
         if (candidate.type === 'code' && candidate.lang === 'sql') {
-          const config = parseConfig(candidate.meta);
+          const config = parseRunnableSqlMeta(candidate.meta);
           if (config) {
             return wrapRunnableSql(candidate, config);
           }

package/src/sql/sql.css

@@ -1,7 +1,7 @@
 /* Light-DOM styles for `<dfk-sql>`: the result container. It lives in the
    element's light DOM (not the shadow root) because VTable injects a
    *document-level* stylesheet that a shadow boundary could not host — the same
-   light-DOM exception as the TOC toggle (see AGENTS.md rule 9). It is created
+   light-DOM exception as the TOC toggle (see CONVENTIONS.md rule 9). It is created
    only when a query runs, so the light DOM still starts empty (rule 11). Every
    class carries the `dfk-sql-` prefix so nothing can collide with the host site.
  *