crawldna 0.1.0

package/src/profiles/docs/sitemap.mjs

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Tier 2 of the documentation profile: sitemap enumeration.
+//
+// Follows sitemap indexes recursively, collects every <loc>, and also reads
+// Sitemap: directives from robots.txt. Filtering to the docs section is done by
+// the caller (it knows the base path).
+
+import { fetchText } from '../../lib/fetcher.mjs';
+import { XMLParser } from 'fast-xml-parser';
+import { originOf, normalizeUrl } from '../../lib/url.mjs';
+
+function toArray(x) {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+
+/**
+ * Pull { url, lastmod } from a parsed <urlset> (fast-xml-parser output). Pure — no
+ * network — so the #6 lastmod extraction is unit-testable. `lastmod` is '' when the
+ * entry omits it.
+ * @param {object} xml parsed sitemap XML
+ * @returns {Array<{url:string,lastmod:string}>}
+ */
+export function sitemapEntriesFromXml(xml) {
+  const out = [];
+  for (const u of toArray(xml && xml.urlset && xml.urlset.url)) {
+    if (u && u.loc) out.push({ url: String(u.loc).trim(), lastmod: u.lastmod != null ? String(u.lastmod).trim() : '' });
+  }
+  return out;
+}
+
+/**
+ * Collect { url, lastmod } for every page reachable from a site's sitemaps.
+ * Follows sitemap indexes recursively; the first lastmod seen for a URL wins.
+ * @param {string} baseUrl
+ * @param {object} [opts]
+ * @param {() => boolean} [opts.shouldStop]
+ * @param {number} [opts.maxDepth] sitemap-index recursion depth
+ * @returns {Promise<Array<{url:string,lastmod:string}>>}
+ */
+export async function collectSitemapEntries(baseUrl, { shouldStop, maxDepth = 4 } = {}) {
+  const origin = originOf(baseUrl);
+  if (!origin) return [];
+
+  const candidates = [
+    origin + '/sitemap.xml',
+    origin + '/sitemap_index.xml',
+    origin + '/sitemap-index.xml',
+  ];
+
+  // robots.txt may advertise sitemaps elsewhere.
+  const robots = await fetchText(origin + '/robots.txt', { accept: 'text/plain, */*' });
+  if (robots.ok && robots.text) {
+    for (const m of robots.text.matchAll(/^\s*sitemap:\s*(\S+)/gim)) {
+      candidates.push(m[1].trim());
+    }
+  }
+
+  const parser = new XMLParser({ ignoreAttributes: true, trimValues: true });
+  const seen = new Set();
+  const entries = new Map(); // url -> lastmod (first wins)
+
+  async function walk(smUrl, depth) {
+    if (depth > maxDepth || seen.has(smUrl)) return;
+    if (shouldStop && shouldStop()) return;
+    seen.add(smUrl);
+
+    const res = await fetchText(smUrl, { accept: 'application/xml, text/xml, */*' });
+    if (!res.ok || !res.text) return;
+
+    let xml;
+    try {
+      xml = parser.parse(res.text);
+    } catch {
+      return;
+    }
+
+    if (xml.sitemapindex) {
+      for (const sm of toArray(xml.sitemapindex.sitemap)) {
+        if (sm && sm.loc) await walk(String(sm.loc).trim(), depth + 1);
+      }
+    } else if (xml.urlset) {
+      for (const e of sitemapEntriesFromXml(xml)) {
+        if (!entries.has(e.url)) entries.set(e.url, e.lastmod);
+      }
+    }
+  }
+
+  for (const c of [...new Set(candidates)]) {
+    if (shouldStop && shouldStop()) break;
+    await walk(c, 0);
+  }
+
+  return [...entries].map(([url, lastmod]) => ({ url, lastmod }));
+}
+
+/**
+ * Collect all page URLs reachable from a site's sitemaps.
+ * @param {string} baseUrl
+ * @param {object} [opts]
+ * @returns {Promise<string[]>}
+ */
+export async function collectSitemapUrls(baseUrl, opts = {}) {
+  return (await collectSitemapEntries(baseUrl, opts)).map((e) => e.url);
+}
+
+/**
+ * #6 — a Map of normalizedUrl -> <lastmod> for a site's sitemap. Only URLs that
+ * actually carry a lastmod are included (a blank one is no freshness evidence).
+ * @param {string} baseUrl
+ * @param {object} [opts]
+ * @returns {Promise<Map<string,string>>}
+ */
+export async function sitemapLastmodMap(baseUrl, opts = {}) {
+  const map = new Map();
+  for (const e of await collectSitemapEntries(baseUrl, opts)) {
+    if (e.lastmod) map.set(normalizeUrl(e.url) || e.url, e.lastmod);
+  }
+  return map;
+}

package/src/profiles/docs.mjs

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Documentation task profile (§4). For docs the priorities are completeness and
+// precision, so:
+//   1. /llms-full.txt — the publisher's own complete export; use it verbatim.
+//   2. /sitemap.xml   — an authoritative page list; use it to SEED the engine.
+//   3. otherwise      — let the engine crawl from the entry, discovering pages.
+//
+// Every page (except the llms-full shortcut) goes through the browser-first
+// engine so dynamic/interaction-hidden docs (Firebase tabs, SPA nav, …) are
+// fully revealed — not just statically scraped.
+
+import { tryLlmsFull } from './docs/llms.mjs';
+import { collectSitemapUrls } from './docs/sitemap.mjs';
+import { normalizeUrl, inScope, pathOf } from '../lib/url.mjs';
+
+const now = () => new Date().toISOString();
+const bytes = (s) => Buffer.byteLength(s || '', 'utf8');
+
+/**
+ * The path prefix to constrain a docs crawl to. For "extract all documentation"
+ * the user points at *a* doc page but wants the whole docs tree, so we scope to
+ * the first path segment (the docs root: /docs, /en, /guide, …) rather than the
+ * exact entry path. Narrow it with --include when you only want one section.
+ */
+function scopePrefixFor(baseUrl) {
+  const p = pathOf(normalizeUrl(baseUrl) || baseUrl);
+  if (!p || p === '/') return null;
+  const segs = p.replace(/\/+$/, '').split('/').filter(Boolean);
+  if (!segs.length) return null;
+  return '/' + segs[0];
+}
+
+/** Keep same-site URLs under the docs path, honour include/exclude, dedupe. */
+function filterDocUrls(urls, baseUrl, options, prefix) {
+  const out = new Set();
+  for (const u of urls) {
+    const n = normalizeUrl(u, baseUrl);
+    if (!n || !inScope(n, baseUrl, options)) continue;
+    if (prefix) {
+      const p = pathOf(n);
+      if (!(p === prefix || p.startsWith(prefix + '/'))) continue;
+    }
+    out.add(n);
+  }
+  return [...out];
+}
+
+export async function runDocsProfile(target, ctx) {
+  const { url, task } = target;
+
+  // -- Tier 1: llms-full.txt (complete, verbatim, no browser) ---------------
+  const llms = await tryLlmsFull(url).catch(() => null);
+  if (llms && !ctx.shouldStop()) {
+    ctx.emit({ type: 'strategy', url, strategy: 'docs:llms-full' });
+    ctx.emit({ type: 'discover', url, count: llms.pages.length });
+    ctx.setTotal(llms.pages.length);
+    for (const p of llms.pages) {
+      if (ctx.shouldStop()) break;
+      ctx.addPage({
+        url: p.url,
+        task,
+        title: p.title,
+        markdown: p.markdown,
+        meta: { strategy: 'docs:llms-full', source: llms.sourceUrl, fetchedAt: now(), bytes: bytes(p.markdown) },
+      });
+      ctx.markProcessed(); // bar advances per page handled (kept or deduped)
+    }
+    return;
+  }
+
+  const prefix = scopePrefixFor(url);
+
+  // -- Tier 2: sitemap → seed the engine ------------------------------------
+  const sitemap = await collectSitemapUrls(url, { shouldStop: ctx.shouldStop }).catch(() => []);
+  const seeds = filterDocUrls(sitemap, url, ctx.options, prefix);
+
+  if (seeds.length > 1 && !ctx.shouldStop()) {
+    ctx.emit({ type: 'strategy', url, strategy: 'docs:sitemap' });
+    ctx.emit({ type: 'discover', url, count: seeds.length });
+    ctx.setTotal(seeds.length);
+    await ctx.runEngine(target, { seeds, announce: false, scopePrefix: prefix });
+    return;
+  }
+
+  // -- Tier 3: no page list → engine crawls from the entry and discovers ----
+  ctx.emit({
+    type: 'warn',
+    url,
+    reason: 'no-page-list',
+    message:
+      'No llms-full.txt or usable sitemap found; the engine will crawl from the entry page and ' +
+      'discover pages as it goes. Completeness is not guaranteed.',
+  });
+  await ctx.runEngine(target, { announce: true, scopePrefix: prefix });
+}

package/src/reshape.mjs

@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Phase 2 — "reshape" (chat with your extraction).
+//
+// The crawl (Phase 1) produces a faithful, VERBATIM extraction per link. This
+// step reworks those SAVED files into whatever the user asks — a table, a split,
+// a filtered subset, a regroup — on demand, as many times as they want, reusing
+// the same extraction as context (like querying a knowledge base). The crawl's
+// own files are never touched; every reshape output lands under <scan>/chat/ and
+// the conversation is recorded in that scan's session. Value-faithful by design:
+// aiReshape copies every kept value exactly and never invents one.
+
+import {
+  getRun,
+  readRunFile,
+  readChatFile,
+  getChatSession,
+  saveChatSession,
+  writeChatFile,
+} from './lib/runs.mjs';
+import { aiReshape } from './engine/decide.mjs';
+import { resolveLlm } from './lib/llm.mjs';
+import { slug } from './lib/url.mjs';
+import { verifyValues, fidelityBanner, stripFidelityBanner } from './lib/faithful.mjs';
+import { simhash, hamming } from './lib/simhash.mjs';
+
+// How many of the user's own prior chat outputs to surface back as context, so a
+// follow-up like "redo the table you made" can reference them without flooding
+// the prompt as the conversation grows.
+const PRODUCED_CONTEXT = 6;
+
+function stripFrontMatter(md) {
+  const m = String(md || '').match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/);
+  return m ? String(md).slice(m[0].length) : String(md || '');
+}
+
+function ensureTrailingNewline(s) {
+  const t = String(s || '');
+  return t.endsWith('\n') ? t : t + '\n';
+}
+
+/** Sanitise a model-proposed filename to a safe, unique `*.md` name. */
+function sanitizeChatName(raw, used) {
+  const base = slug(String(raw || '').replace(/\.md$/i, '')) || 'reshaped';
+  let name = `${base}.md`;
+  let n = 2;
+  while (used.has(name) || name === 'session.json') name = `${base}-${n++}.md`;
+  used.add(name);
+  return name;
+}
+
+/** Find a scan in a run manifest by id (''/undefined = the only/first scan). */
+function findScan(manifest, scanId) {
+  const scans = manifest.scans || [];
+  const sid = String(scanId || '');
+  return scans.find((s) => String(s.scanId || '') === sid) || (sid ? null : scans[0]) || null;
+}
+
+/**
+ * Run one reshape turn over a saved scan's extraction.
+ *
+ * @param {object} a
+ * @param {string} a.runId
+ * @param {string} [a.scanId]   '' = the run's only/first scan
+ * @param {string} a.message    the user's request
+ * @param {string} a.model      model id/name
+ * @param {string} [a.provider] 'ollama' (default) | 'openai'
+ * @param {string} [a.host]     Ollama host override (provider 'ollama')
+ * @param {string} [a.baseUrl]  OpenAI-compatible API base URL (provider 'openai')
+ * @param {string} [a.apiKey]   API key (provider 'openai')
+ * @param {string} [a.embedModel] #22 — optional embedding model id: upgrades the
+ *                              context retrieval to semantic section ranking
+ *                              (cross-language asks pull the right sections)
+ * @param {string} [a.cacheDir] runs-cache override
+ * @param {boolean} [a.verify]  #11 fidelity check (default true): value-like atoms of
+ *                              every produced file are verified against the FULL crawled
+ *                              sources; unverifiable ones are flagged with a warning
+ *                              banner inside the file instead of served silently.
+ * @returns {Promise<{ reply: string, files: Array<{ filename, bytes, at, fidelity? }>, truncated: boolean, contextMode?: string }>}
+ */
+export async function reshape({ runId, scanId = '', message, model, provider, host, baseUrl, apiKey, embedModel, cacheDir, verify = true }) {
+  if (!String(message || '').trim()) throw new Error('empty message');
+  if (!model) throw new Error('no model selected');
+  const llm = resolveLlm({ provider, model, embedModel, ollamaHost: host, baseUrl, apiKey });
+  const opts = cacheDir ? { cacheDir } : {};
+
+  const { manifest } = await getRun(runId, opts);
+  const scan = findScan(manifest, scanId);
+  if (!scan) throw new Error('scan not found');
+  const sid = String(scan.scanId || '');
+
+  // The scan's ORIGINAL crawled files, each as an IDENTIFIABLE document (filename
+  // + on-disk size + source URLs). Passing identity — not an anonymous blob — is
+  // what lets the model honour references like "the original md" / "the 4574b
+  // file" and treat these as the default thing to reshape.
+  const documents = [];
+  for (const f of scan.files || []) {
+    try {
+      const raw = await readRunFile(runId, sid, f.filename, opts);
+      documents.push({
+        filename: f.filename,
+        title: f.title || '',
+        bytes: typeof f.bytes === 'number' ? f.bytes : Buffer.byteLength(raw, 'utf8'),
+        sources: Array.isArray(f.pages) && f.pages.length ? f.pages : scan.url ? [scan.url] : [],
+        content: stripFrontMatter(raw),
+      });
+    } catch {
+      /* skip an unreadable file rather than fail the whole turn */
+    }
+  }
+  if (!documents.some((d) => d.content.trim())) throw new Error('no extracted content for this link');
+
+  const session = await getChatSession(runId, sid, opts);
+
+  // The user's own prior outputs from THIS chat, as clearly-separate context so a
+  // follow-up can revise one ("redo the table you made") — never confused with
+  // the originals.
+  const produced = [];
+  for (const f of (session.files || []).slice(-PRODUCED_CONTEXT)) {
+    try {
+      // Strip our own fidelity banner: it is a warning for the USER, not content the
+      // model should iterate on (or copy into new files).
+      produced.push({
+        filename: f.filename,
+        bytes: f.bytes,
+        content: stripFidelityBanner(stripFrontMatter(await readChatFile(runId, sid, f.filename, opts))),
+      });
+    } catch {
+      /* a derived file may have been removed — skip it */
+    }
+  }
+
+  const { reply, files, truncated, contextMode } = await aiReshape({
+    llm,
+    documents,
+    produced,
+    instruction: message,
+    history: (session.messages || []).map((m) => ({ role: m.role, content: m.content })),
+  });
+
+  // Deterministic guards on what the model produced, BEFORE anything is saved.
+  //
+  // (1) RE-EMISSION FILTER: models re-deliver earlier files under new names as the
+  // conversation grows (observed live: three near-identical "pagination" docs in one
+  // chat). A produced file whose SimHash is within Hamming 3 of a file already in this
+  // chat (or of another file from this same turn) is skipped, with a note — never
+  // silently.
+  //
+  // (2) FIDELITY CHECK (#11): every value-like atom (numbers, URLs, inline code, quoted
+  // literals, code lines) of a kept file is verified against the FULL crawled sources —
+  // not the model's context — plus the user's own instruction. Unverifiable values are
+  // flagged with a clearly tool-generated banner INSIDE the file, and reported in the
+  // result, instead of being served as if they were extracted facts.
+  const sourceTexts = documents.map((d) => d.content);
+  const priorHashes = produced.map((p) => simhash(p.content));
+  const used = new Set((session.files || []).map((f) => f.filename));
+  const savedFiles = [];
+  const notes = [];
+  for (const file of files) {
+    const raw = String(file.content);
+    const sh = simhash(raw);
+    if (priorHashes.some((h) => hamming(sh, h) <= 3)) {
+      notes.push(`(Skipped "${file.filename}" — near-identical to a file already produced in this chat.)`);
+      continue;
+    }
+    priorHashes.push(sh);
+
+    let fidelity = null;
+    let content = raw;
+    if (verify) {
+      const v = verifyValues(raw, sourceTexts, { allow: message });
+      fidelity = { checked: v.total, verified: v.verified, unverified: v.unverified.slice(0, 20) };
+      if (v.unverified.length) content = fidelityBanner(v) + '\n\n' + raw;
+    }
+
+    const name = sanitizeChatName(file.filename, used);
+    const body = ensureTrailingNewline(content);
+    await writeChatFile(runId, sid, name, body, opts);
+    savedFiles.push({
+      filename: name,
+      bytes: Buffer.byteLength(body, 'utf8'),
+      at: new Date().toISOString(),
+      ...(fidelity ? { fidelity } : {}),
+    });
+  }
+  const replyOut = [reply, ...notes].filter(Boolean).join('\n\n');
+
+  // Record the turn (user + assistant) and register the new files.
+  const now = new Date().toISOString();
+  session.messages = session.messages || [];
+  session.messages.push({ role: 'user', content: String(message), at: now });
+  session.messages.push({
+    role: 'assistant',
+    content: replyOut,
+    files: savedFiles.map((f) => f.filename),
+    at: now,
+  });
+  session.files = [...(session.files || []), ...savedFiles];
+  await saveChatSession(runId, sid, session, opts);
+
+  return { reply: replyOut, files: savedFiles, truncated, contextMode };
+}
+
+export default reshape;