crawldna 0.1.0

package/src/lib/llm.mjs

@@ -0,0 +1,534 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// The LLM transport layer — the ONE place that talks to a model provider.
+//
+// crawldna's judgment layer (src/engine/decide.mjs) and reshape (src/reshape.mjs)
+// are provider-agnostic: they call `chat(llm, system, user)` and never know which
+// backend answered. Two backends are supported:
+//
+//   - 'ollama'  — a local Ollama server (the default; uses the `ollama` package,
+//                 so model loading / keep-alive are handled for you).
+//   - 'openai'  — ANY OpenAI-compatible Chat Completions API, addressed by a base
+//                 URL + API key. This is the de-facto standard, so the same code
+//                 path covers OpenAI, OpenRouter, Groq, Together, Mistral, DeepSeek,
+//                 and local servers (llama.cpp, LM Studio, vLLM). Ollama itself
+//                 speaks it at http://localhost:11434/v1, so it works here too.
+//
+// A model config is the small descriptor `{ provider, model, baseUrl, apiKey }`,
+// produced once by `resolveLlm(options)` and threaded through the engine.
+
+import ollama, { Ollama } from 'ollama';
+
+const PROVIDERS = new Set(['ollama', 'openai']);
+const DEFAULT_OLLAMA_HOST = 'http://127.0.0.1:11434';
+const REQUEST_TIMEOUT_MS = 120000;
+// Reshape turns rework WHOLE documents (long outputs, slow on a local model) — the
+// 120s leash killed a legitimate "redo the original, tidied" turn live. They get a
+// longer, still-bounded budget; the crawl's small judgment calls keep the tight one.
+const RESHAPE_TIMEOUT_MS = 300000;
+const timeoutFor = (kind) => (kind === 'reshape' ? RESHAPE_TIMEOUT_MS : REQUEST_TIMEOUT_MS);
+
+// --- LLM call throttle (provider-aware) ------------------------------------
+// A LOCAL model (Ollama) is a single process that answers ~one prompt at a time,
+// so firing the crawl's parallel pages' judgment calls at it ALL AT ONCE only
+// thrashes it: each call gets slower, and Stop drags because in-flight calls must
+// drain. So we cap CONCURRENT calls PER PROVIDER — tight for local, generous for a
+// remote API (OpenAI/DeepSeek/OpenRouter/… scale horizontally, so parallelism there
+// is a clear win). The browser still renders pages in parallel; only the model
+// calls are metered.
+const PROVIDER_CONCURRENCY = { ollama: 2, openai: 16 };
+
+function createLimiter(max) {
+  let active = 0;
+  const waiters = [];
+  const pump = () => {
+    while (active < max && waiters.length) {
+      active++;
+      const { fn, resolve, reject } = waiters.shift();
+      Promise.resolve()
+        .then(fn)
+        .then(resolve, reject)
+        .finally(() => {
+          active--;
+          pump();
+        });
+    }
+  };
+  return {
+    run(fn) {
+      return new Promise((resolve, reject) => {
+        waiters.push({ fn, resolve, reject });
+        pump();
+      });
+    },
+    // Drop everything still QUEUED (not yet started) so a stopped crawl doesn't
+    // keep handing work to the model. Callers in decide.mjs `.catch()` and bias to
+    // keep/follow, so a cancelled judgment never loses content.
+    clear() {
+      const pending = waiters.splice(0);
+      for (const w of pending) w.reject(new Error('LLM call cancelled (crawl stopped)'));
+    },
+  };
+}
+
+const _limiters = new Map();
+function limiterFor(provider) {
+  let l = _limiters.get(provider);
+  if (!l) {
+    l = createLimiter(PROVIDER_CONCURRENCY[provider] || 8);
+    _limiters.set(provider, l);
+  }
+  return l;
+}
+
+/** Cancel all QUEUED (not-yet-started) LLM calls — called by run.stop() so a stop
+ *  is near-instant instead of waiting for a backlog of judgment calls to run. */
+export function abortPendingLlm() {
+  for (const l of _limiters.values()) l.clear();
+}
+
+/** Rough token estimate (≈4 chars/token) — only a FALLBACK for when a backend
+ *  doesn't report real counts; real numbers from the provider are preferred. */
+function estimateTokens(s) {
+  return Math.ceil(String(s || '').length / 4);
+}
+
+/** Reject a promise if it outruns `ms`, so a hung model call can't stall a crawl
+ *  (or its Stop) indefinitely. The Ollama client has no per-call timeout of its own. */
+function withTimeout(promise, ms, label) {
+  let t;
+  const timeout = new Promise((_, reject) => {
+    t = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(t));
+}
+
+/** Strip a trailing slash; treat empty as empty. */
+function trimUrl(raw) {
+  return String(raw || '').trim().replace(/\/+$/, '');
+}
+
+/**
+ * Build the base URL for an OpenAI-compatible API. Most providers want the `/v1`
+ * suffix already in the URL (OpenAI, Ollama), but some carry their own path
+ * (OpenRouter's `/api/v1`). So: if the user gave only an origin (no path), append
+ * `/v1` for them; otherwise respect whatever path they typed.
+ */
+function openaiBase(raw) {
+  let s = trimUrl(raw);
+  if (!s) return '';
+  try {
+    const u = new URL(s);
+    if (u.pathname === '' || u.pathname === '/') s = trimUrl(s) + '/v1';
+  } catch {
+    /* not a full URL — leave it untouched and let the request surface the error */
+  }
+  return s;
+}
+
+/** Join a base URL and a path segment with exactly one slash. */
+function joinUrl(base, path) {
+  return trimUrl(base) + '/' + String(path).replace(/^\/+/, '');
+}
+
+/**
+ * Is this descriptor the deliberate NO-AI mode? The judgment layer (decide.mjs)
+ * checks this to skip its model calls entirely and use the deterministic
+ * fallbacks it already has — same behaviour as a model outage, minus the failed
+ * calls' latency. One predicate so "AI off" is spelled the same way everywhere.
+ */
+export function llmDisabled(llm) {
+  return !llm || llm.provider === 'none';
+}
+
+/**
+ * Normalise raw options into a `{ provider, model, baseUrl, apiKey }` descriptor.
+ * Backward compatible: an absent provider means 'ollama', and `ollamaHost` keeps
+ * working. For the 'openai' provider, the API key falls back to the environment
+ * (CRAWLDNA_API_KEY, then OPENAI_API_KEY) so it never has to be put on the CLI.
+ *
+ * `noAi: true` wins over everything: it yields the provider-'none' descriptor —
+ * the crawl runs on its deterministic fallbacks (heuristic reveal, keep whole,
+ * follow all in-scope) and never contacts a model, so no model is required.
+ * Note it also drops `embedModel` (#22): no-AI means zero calls to ANY model,
+ * embeddings included — rule #6 is absolute.
+ *
+ * @param {object} [options]
+ * @param {boolean} [options.noAi]      disable AI entirely (provider 'none')
+ * @param {string} [options.provider]   'ollama' (default) | 'openai'
+ * @param {string} [options.model]
+ * @param {string} [options.embedModel] embedding model id (#22, optional — enables
+ *                                      the semantic relevance tier)
+ * @param {string} [options.baseUrl]    OpenAI-compatible API base URL
+ * @param {string} [options.apiKey]
+ * @param {string} [options.ollamaHost] Ollama server URL (legacy / ollama provider)
+ */
+export function resolveLlm(options = {}) {
+  if (options.noAi) return { provider: 'none', model: '', baseUrl: '', apiKey: '' };
+  const provider = PROVIDERS.has(String(options.provider || '').toLowerCase())
+    ? String(options.provider).toLowerCase()
+    : 'ollama';
+  const model = options.model || '';
+  const embedModel = String(options.embedModel || '').trim();
+
+  if (provider === 'openai') {
+    const apiKey =
+      options.apiKey || process.env.CRAWLDNA_API_KEY || process.env.OPENAI_API_KEY || '';
+    return { provider, model, embedModel, baseUrl: openaiBase(options.baseUrl), apiKey };
+  }
+
+  // ollama: the baseUrl is the Ollama host (no /v1).
+  const baseUrl = trimUrl(options.ollamaHost || options.baseUrl || DEFAULT_OLLAMA_HOST);
+  return { provider, model, embedModel, baseUrl, apiKey: '' };
+}
+
+// --- Ollama backend (one cached client per host) ---------------------------
+const _ollamaClients = new Map();
+function ollamaClient(host) {
+  const key = trimUrl(host);
+  if (!key) return ollama; // package default (127.0.0.1:11434)
+  let c = _ollamaClients.get(key);
+  if (!c) {
+    c = new Ollama({ host: key });
+    _ollamaClients.set(key, c);
+  }
+  return c;
+}
+
+async function ollamaChat(llm, system, user, schema, timeoutMs = REQUEST_TIMEOUT_MS) {
+  const req = {
+    model: llm.model,
+    messages: [
+      { role: 'system', content: system },
+      { role: 'user', content: user },
+    ],
+    stream: false,
+    options: { temperature: 0 },
+  };
+  // Constrained decoding: force the model to emit EXACTLY this JSON shape (Ollama uses
+  // XGrammar). Removes the #1 cause of parse failures (code fences, preambles, partial
+  // or wrong-shaped JSON) and is markedly faster — no tokens spent on formatting. Only
+  // when a schema is expected; free-form calls (reshape, the health ping) pass none.
+  if (schema) req.format = schema;
+  const res = await withTimeout(
+    ollamaClient(llm.baseUrl).chat(req),
+    timeoutMs,
+    'Ollama request',
+  );
+  const content = res?.message?.content || '';
+  // Ollama reports real token counts (prompt_eval_count / eval_count); fall back to
+  // an estimate only if a build doesn't.
+  const usage = {
+    inputTokens: res?.prompt_eval_count ?? estimateTokens(system + user),
+    outputTokens: res?.eval_count ?? estimateTokens(content),
+    cachedInputTokens: 0, // Ollama reuses its KV cache internally but doesn't report it
+  };
+  return { content, usage };
+}
+
+// --- OpenAI-compatible backend (raw fetch, zero new deps) -------------------
+
+/**
+ * Build the chat messages for an OpenAI-compatible request (#4 prompt caching).
+ *
+ * The judgment system prompts are deliberately BYTE-IDENTICAL across every call of
+ * their type (per-call data lives in the user message), so providers with automatic
+ * prefix caching (OpenAI, DeepSeek, vLLM, …) reuse them without being asked —
+ * repeat input tokens become ~10× cheaper/faster on a crawl's thousands of calls.
+ *
+ * OpenRouter is the one place an EXPLICIT marker helps: Anthropic models behind it
+ * cache only blocks tagged `cache_control`, and OpenRouter documents this content-
+ * parts form for all models (it strips the field where unsupported). Every other
+ * endpoint gets the plain-string system message, so nothing changes for them.
+ * Exported for tests.
+ */
+export function buildOpenAiMessages(llm, system, user) {
+  let sys = system;
+  try {
+    const host = new URL(llm.baseUrl).hostname;
+    if (host === 'openrouter.ai' || host.endsWith('.openrouter.ai')) {
+      sys = [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }];
+    }
+  } catch {
+    /* unparsable base URL — keep the plain string; the request will surface the error */
+  }
+  return [
+    { role: 'system', content: sys },
+    { role: 'user', content: user },
+  ];
+}
+
+async function openaiChat(llm, system, user, schema, timeoutMs = REQUEST_TIMEOUT_MS) {
+  if (!llm.baseUrl) throw new Error('no API base URL set');
+  const headers = {
+    'content-type': 'application/json',
+    ...(llm.apiKey ? { authorization: 'Bearer ' + llm.apiKey } : {}),
+  };
+  const base = {
+    model: llm.model,
+    messages: buildOpenAiMessages(llm, system, user),
+    temperature: 0,
+    stream: false,
+  };
+  // When the caller expects JSON, ask for guaranteed-valid JSON via `json_object`: it
+  // kills the code fences / preambles that break parsing, and is broadly supported
+  // (OpenAI, DeepSeek, Groq, vLLM, LM Studio…). The prompts already contain the word
+  // "JSON" (OpenAI requires it for this mode). If a provider REJECTS response_format, we
+  // retry once WITHOUT it — a crawl must never break on an endpoint lacking the feature.
+  const send = (useFormat) =>
+    fetch(joinUrl(llm.baseUrl, 'chat/completions'), {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(useFormat ? { ...base, response_format: { type: 'json_object' } } : base),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+
+  // TRANSIENT-FAILURE RETRY. On a paid API a crawl fires thousands of judgment calls;
+  // rate limits (429) and server hiccups (5xx, connection resets) are ROUTINE there,
+  // and every failed judgment call triggers the completeness-bias fallback — "follow/
+  // keep EVERYTHING" — which quietly blows the crawl off-task. So retry up to twice
+  // with backoff, honouring Retry-After. A TIMEOUT is not retried: the per-call leash
+  // is already generous, and the crawl's keep-bias makes a rare loss safe.
+  const FORMAT_REJECT = [400, 404, 415, 422, 501];
+  const RETRYABLE = new Set([429, 500, 502, 503, 529]);
+  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+  let useFormat = !!schema;
+  let r = null;
+  let lastErr = null;
+  for (let attempt = 0; attempt <= 2; attempt++) {
+    if (attempt > 0) {
+      const ra = r && Number(r.headers.get('retry-after'));
+      await sleep(Math.min(15000, ra > 0 ? ra * 1000 : 500 * 2 ** (attempt - 1)));
+    }
+    try {
+      r = await send(useFormat);
+      lastErr = null;
+    } catch (err) {
+      if (err && (err.name === 'TimeoutError' || err.name === 'AbortError')) throw err;
+      lastErr = err; // network-level hiccup (reset, DNS) — retryable like a 5xx
+      r = null;
+      continue;
+    }
+    if (r.ok) break;
+    if (useFormat && FORMAT_REJECT.includes(r.status)) {
+      useFormat = false; // provider doesn't support response_format — degrade, don't fail
+      try {
+        r = await send(false);
+        if (r.ok) break;
+      } catch (err) {
+        if (err && (err.name === 'TimeoutError' || err.name === 'AbortError')) throw err;
+        lastErr = err;
+        r = null;
+        continue;
+      }
+    }
+    if (!RETRYABLE.has(r.status)) break; // a real, non-transient error — surface it
+  }
+  if (lastErr) throw lastErr;
+  if (!r.ok) {
+    const body = await r.text().catch(() => '');
+    throw new Error(`LLM HTTP ${r.status}${body ? ': ' + body.slice(0, 200) : ''}`);
+  }
+  const d = await r.json().catch(() => null);
+  const content = d?.choices?.[0]?.message?.content || '';
+  // OpenAI-compatible APIs return token usage; estimate only if absent.
+  const u = d?.usage || {};
+  const usage = {
+    inputTokens: u.prompt_tokens ?? estimateTokens(system + user),
+    outputTokens: u.completion_tokens ?? estimateTokens(content),
+    // #4: how much of the input was served from the provider's prompt cache —
+    // OpenAI-style (prompt_tokens_details.cached_tokens) or DeepSeek-style
+    // (prompt_cache_hit_tokens). Metered so a run can SHOW the cached share growing
+    // (those tokens are ~10× cheaper); 0 when the provider doesn't report it.
+    cachedInputTokens: Number(u.prompt_tokens_details?.cached_tokens ?? u.prompt_cache_hit_tokens ?? 0) || 0,
+  };
+  return { content, usage };
+}
+
+/**
+ * One chat turn against the configured provider. Temperature 0, non-streaming.
+ * Throws on transport/HTTP errors — callers in the crawl wrap this in
+ * `.catch()` and bias toward keep/follow, so a model outage never loses content;
+ * reshape surfaces the message to the user.
+ *
+ * @param {{provider:string, model:string, baseUrl:string, apiKey:string}} llm
+ * @param {object} [schema]  optional JSON schema → constrained/guaranteed JSON output
+ * @param {string} [kind]    a label for WHAT this call is (reveal/scope/links/nav-plan/
+ *                           reshape/health) — reported to the usage sink so tokens can be
+ *                           attributed per call type, not just totalled (see src/eval).
+ * @returns {Promise<string>}
+ */
+export async function chat(llm, system, user, schema = null, kind = '') {
+  // Defensive backstop: decide.mjs short-circuits before ever reaching here in
+  // no-AI mode; anything else that leaks a 'none' descriptor (e.g. reshape) gets
+  // a clear reason instead of a misleading "no model selected".
+  if (llmDisabled(llm)) throw new Error('AI is disabled for this run (no-AI mode) — no model calls are made');
+  if (!llm.model) throw new Error('no model selected');
+  const provider = llm.provider === 'openai' ? 'openai' : 'ollama';
+  // Meter concurrent calls per provider (tight for local, generous for remote).
+  return limiterFor(provider).run(async () => {
+    const timeoutMs = timeoutFor(kind);
+    const { content, usage } = provider === 'openai'
+      ? await openaiChat(llm, system, user, schema, timeoutMs)
+      : await ollamaChat(llm, system, user, schema, timeoutMs);
+    // Report token usage to an optional sink on the descriptor (set by the crawl)
+    // so cost can be approximated. The `kind` lets the sink break the total down by
+    // call type (which judgment actually spends the tokens). Metering must never
+    // break the actual call.
+    if (typeof llm.__onUsage === 'function') {
+      try {
+        llm.__onUsage({
+          provider,
+          kind: kind || 'other',
+          inputTokens: usage.inputTokens || 0,
+          outputTokens: usage.outputTokens || 0,
+          cachedInputTokens: usage.cachedInputTokens || 0,
+        });
+      } catch {
+        /* ignore */
+      }
+    }
+    return content;
+  });
+}
+
+/**
+ * #22 — embed texts with the configured `embedModel`, through the SAME provider
+ * seam as chat: Ollama `/api/embed` or any OpenAI-compatible `/v1/embeddings`.
+ * Returns one vector per input, in input order. Usage is metered to the sink
+ * under kind 'embed', so a run's report shows exactly what the semantic tier
+ * costs next to the chat calls.
+ *
+ * Contract (rule #6): in no-AI mode this THROWS — embeddings are model calls
+ * like any other, and no-AI means zero calls to ANY model. Callers (the
+ * semantic scorer) check `llm.embedModel` first and fall back to the lexical
+ * floor; this guard is the backstop, not the routine path.
+ *
+ * @param {{provider:string, embedModel?:string, baseUrl:string, apiKey:string}} llm
+ * @param {string[]|string} texts
+ * @returns {Promise<number[][]>}
+ */
+export async function embed(llm, texts) {
+  if (llmDisabled(llm)) throw new Error('AI is disabled for this run (no-AI mode) — no model calls are made, embeddings included');
+  const model = llm && llm.embedModel;
+  if (!model) throw new Error('no embedModel configured');
+  const list = (Array.isArray(texts) ? texts : [texts]).map((t) => String(t ?? ''));
+  if (!list.length) return [];
+  const provider = llm.provider === 'openai' ? 'openai' : 'ollama';
+  return limiterFor(provider).run(async () => {
+    let vectors;
+    let usage;
+    if (provider === 'openai') {
+      if (!llm.baseUrl) throw new Error('no API base URL set');
+      const r = await fetch(joinUrl(llm.baseUrl, 'embeddings'), {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          ...(llm.apiKey ? { authorization: 'Bearer ' + llm.apiKey } : {}),
+        },
+        body: JSON.stringify({ model, input: list }),
+        signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+      });
+      if (!r.ok) {
+        const body = await r.text().catch(() => '');
+        throw new Error(`embeddings HTTP ${r.status}${body ? ': ' + body.slice(0, 200) : ''}`);
+      }
+      const d = await r.json().catch(() => null);
+      vectors = (Array.isArray(d?.data) ? d.data : [])
+        .slice()
+        .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
+        .map((e) => e.embedding);
+      usage = { inputTokens: d?.usage?.prompt_tokens ?? estimateTokens(list.join(' ')), outputTokens: 0, cachedInputTokens: 0 };
+    } else {
+      const res = await withTimeout(
+        ollamaClient(llm.baseUrl).embed({ model, input: list }),
+        REQUEST_TIMEOUT_MS,
+        'Ollama embed',
+      );
+      vectors = res?.embeddings || [];
+      usage = { inputTokens: res?.prompt_eval_count ?? estimateTokens(list.join(' ')), outputTokens: 0, cachedInputTokens: 0 };
+    }
+    if (!Array.isArray(vectors) || vectors.length !== list.length || vectors.some((v) => !Array.isArray(v) || !v.length)) {
+      throw new Error(`embedding backend returned ${Array.isArray(vectors) ? vectors.length : 0} vector(s) for ${list.length} input(s)`);
+    }
+    if (typeof llm.__onUsage === 'function') {
+      try {
+        llm.__onUsage({ provider, kind: 'embed', inputTokens: usage.inputTokens || 0, outputTokens: 0, cachedInputTokens: 0 });
+      } catch {
+        /* metering must never break the call */
+      }
+    }
+    return vectors;
+  });
+}
+
+/**
+ * One-time health check run before a crawl: confirm the configured model actually
+ * answers. The crawl's judgment calls (decide.mjs) all `.catch()` and bias toward
+ * keep/follow/reveal, so a misconfigured model (wrong key, unreachable host,
+ * un-pulled model) would otherwise degrade to heuristics SILENTLY — bad output
+ * with no explanation. This lets the caller warn LOUDLY instead. Best-effort and
+ * bounded: any throw means "not usable" with a short reason; it never throws.
+ *
+ * @param {{provider:string, model:string, baseUrl:string, apiKey:string}} llm
+ * @returns {Promise<{ ok: boolean, reason?: string }>}
+ */
+export async function checkModel(llm) {
+  if (!llm || !llm.model) return { ok: false, reason: 'no model selected' };
+  if (llm.provider === 'openai' && !llm.baseUrl) return { ok: false, reason: 'no API base URL set' };
+  try {
+    await chat(llm, 'Reply with the single word OK.', 'ping', null, 'health');
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, reason: String((err && err.message) || err).slice(0, 200) };
+  }
+}
+
+/**
+ * List the models a provider offers, for the setup UI's picker.
+ * Shape mirrors the old Ollama probe: `{ ok, models: [{ name, isCloud, size }] }`.
+ *  - ollama → GET /api/tags (cloud models report size 0 or a -cloud suffix).
+ *  - openai → GET /models (every listed model is "cloud"); needs a valid key for
+ *    providers that gate the endpoint, in which case `ok:false` lets the UI fall
+ *    back to a typed model id.
+ *
+ * @param {{provider:string, baseUrl:string, apiKey:string}} llm
+ * @returns {Promise<{ ok:boolean, models:Array<{name:string,isCloud:boolean,size:number}>, error?:string }>}
+ */
+export async function listModels(llm) {
+  if (llm.provider === 'openai') {
+    if (!llm.baseUrl) return { ok: false, models: [], error: 'no base URL' };
+    try {
+      const r = await fetch(joinUrl(llm.baseUrl, 'models'), {
+        headers: llm.apiKey ? { authorization: 'Bearer ' + llm.apiKey } : {},
+        signal: AbortSignal.timeout(10000),
+      });
+      if (!r.ok) return { ok: false, models: [], error: 'HTTP ' + r.status };
+      const d = await r.json().catch(() => null);
+      const arr = Array.isArray(d?.data) ? d.data : Array.isArray(d?.models) ? d.models : [];
+      const models = arr
+        .map((m) => ({ name: m.id || m.name || '', isCloud: true, size: 0 }))
+        .filter((m) => m.name);
+      return { ok: true, models };
+    } catch (err) {
+      return { ok: false, models: [], error: String((err && err.message) || err) };
+    }
+  }
+
+  // ollama
+  const base = trimUrl(llm.baseUrl) || 'http://localhost:11434';
+  try {
+    const r = await fetch(base + '/api/tags', { signal: AbortSignal.timeout(10000) });
+    if (!r.ok) return { ok: false, models: [] };
+    const d = await r.json();
+    const models = (d.models || []).map((m) => ({
+      name: m.name,
+      isCloud: /[:-]cloud\b/i.test(m.name || '') || m.size === 0,
+      size: m.size || 0,
+    }));
+    return { ok: true, models };
+  } catch {
+    return { ok: false, models: [] };
+  }
+}

package/src/lib/output.mjs

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Disk output (§9): a flat bundle of AI-grouped .md files plus a stable
+// manifest.json. The grouping (one file by default, several when the task asks
+// to split) is decided upstream in lib/layout.mjs; this module only writes a
+// pre-built `files` list and the manifest to a directory.
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+/**
+ * Write a bundle of grouped Markdown files + manifest.json to `dir`. When `documents`
+ * is given (the opt-in #10 per-page format), it ALSO writes one .md per page under a
+ * `documents/` subfolder plus a root `index.md` and `documents.jsonl` — pure repackaging
+ * of the same content, so a programmatic consumer can load pages individually.
+ *
+ * @param {string} dir
+ * @param {object} bundle
+ * @param {Array<{ filename: string, markdown: string }>} [bundle.files]
+ * @param {object} bundle.manifest
+ * @param {{ files?: Array<{filename,markdown}>, index?: {filename,markdown}, jsonl?: {filename,content} }} [bundle.documents]
+ * @param {{ files?: Array<{filename,markdown}> }} [bundle.states]  faithful per-state record (states/)
+ * @returns {Promise<{ dir: string, manifestPath: string, files: number, documents: number }>}
+ */
+export async function writeBundle(dir, { files = [], manifest, documents = null, states = null }) {
+  const root = path.resolve(dir);
+  await mkdir(root, { recursive: true });
+
+  for (const f of files) {
+    // basename guards against any stray path separators in a filename.
+    const abs = path.join(root, path.basename(f.filename));
+    await writeFile(abs, f.markdown, 'utf8');
+  }
+
+  let docCount = 0;
+  if (documents) {
+    if (documents.files && documents.files.length) {
+      const docDir = path.join(root, 'documents');
+      await mkdir(docDir, { recursive: true });
+      for (const f of documents.files) {
+        await writeFile(path.join(docDir, path.basename(f.filename)), f.markdown, 'utf8');
+        docCount++;
+      }
+    }
+    if (documents.index) {
+      await writeFile(path.join(root, path.basename(documents.index.filename)), documents.index.markdown, 'utf8');
+    }
+    if (documents.jsonl) {
+      await writeFile(path.join(root, path.basename(documents.jsonl.filename)), documents.jsonl.content, 'utf8');
+    }
+  }
+
+  // The faithful per-state record (reveal snapshots) — one .md per multi-state page.
+  if (states && states.files && states.files.length) {
+    const statesDir = path.join(root, 'states');
+    await mkdir(statesDir, { recursive: true });
+    for (const f of states.files) {
+      await writeFile(path.join(statesDir, path.basename(f.filename)), f.markdown, 'utf8');
+    }
+  }
+
+  const manifestPath = path.join(root, 'manifest.json');
+  await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf8');
+
+  return { dir: root, manifestPath, files: files.length, documents: docCount };
+}

package/src/lib/pool.mjs

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// A tiny fixed-size concurrency pool.
+
+/**
+ * Run `fn` over `items` with at most `concurrency` in flight.
+ * Results are returned in input order. `shouldStop()` (optional) is polled
+ * between items so a graceful stop drains quickly. Errors are captured per
+ * item as `{ __error }` rather than rejecting the whole batch.
+ */
+export async function mapPool(items, concurrency, fn, { shouldStop } = {}) {
+  const results = new Array(items.length);
+  const size = Math.max(1, Math.min(Number(concurrency) || 1, items.length || 1));
+  let cursor = 0;
+
+  async function worker() {
+    while (cursor < items.length) {
+      if (shouldStop && shouldStop()) return;
+      const i = cursor++;
+      try {
+        results[i] = await fn(items[i], i);
+      } catch (err) {
+        results[i] = { __error: err };
+      }
+    }
+  }
+
+  await Promise.all(Array.from({ length: size }, worker));
+  return results;
+}