@zvndev/darkmown 2.4.1-next.7.ge25dbba → 2.5.0

package/src/tools/outline.js

@@ -0,0 +1,321 @@
+// ---------------------------------------------------------------------------
+// TOOLS: outline(files, entry) and refs(files, entry, name)
+//
+// `outline` answers "what is in this page, and where", as a flat list of
+// declarations with real line numbers. `refs` answers "if I change this symbol,
+// what else moves", which is the query that turns a one-edit model into a
+// multi-edit one.
+//
+// Both are built on the symbol table `compileFromMemory` now returns, which the
+// compiler's own directive handlers record as they parse. There is no second
+// parser here on purpose: a parser beside the compiler drifts, and the day it
+// disagrees is the day a model is told something true about a file that will not
+// compile.
+//
+// The one thing the compiler cannot cheaply supply is a line number for a `{ … }`
+// binding in prose, because markdown is rendered a chunk at a time with no line
+// index in scope. The compiler records those reads WITHOUT a line but WITH the
+// exact expression text, and this module locates them by searching the source
+// for that compiler-supplied string. That is a substring search for a known
+// needle, not a parse: the compiler still decides what is true.
+// ---------------------------------------------------------------------------
+
+import { compileFromMemory } from "../compiler.js";
+import { posixKey, shortPath } from "./validate.js";
+
+/** Longest a `detail` may be before it is elided. A multi-line JSON seed is
+ *  otherwise the widest thing in the outline and tells the model nothing. */
+const DETAIL_MAX = 52;
+
+/**
+ * @param {string} s
+ * @returns {string}
+ */
+function short(s) {
+  const one = String(s ?? "")
+    .replace(/\s+/g, " ")
+    .trim();
+  return one.length > DETAIL_MAX ? `${one.slice(0, DETAIL_MAX - 1)}…` : one;
+}
+
+/**
+ * Map a symbol's absolute compile path back to the caller's own file key.
+ * The virtual root is an implementation detail of `compileFromMemory`; a model
+ * that never saw `/site/pages/` must not be handed it back.
+ * @param {string} abs
+ * @param {Record<string, string>} files
+ * @returns {string}
+ */
+function fileKeyOf(abs, files) {
+  // Both sides go through `posixKey`: the compiler's half is platform-native
+  // (a Windows host resolves the virtual root to `C:\site\pages\…`) and the
+  // caller's half is always POSIX, so comparing them raw matches on one
+  // platform and silently fails on the other.
+  const want = posixKey(abs);
+  return Object.keys(files).find((k) => posixKey(k) === want) ?? want;
+}
+
+/**
+ * Resolve the line(s) a line-less read sits on, by finding the compiler's own
+ * expression text in the source.
+ *
+ * Matches only inside a `{ … }` binding, comparing the brace's whole trimmed
+ * contents to the expression. A bare substring search is not good enough and the
+ * failure is not theoretical: searching for `count` matched the heading "Click
+ * counter", its own `:state count = 0` declaration, and the `count++` inside a
+ * button, turning one real read into four. Anything a tool reports has to be
+ * something the model can act on.
+ *
+ * A binding repeated on several lines is several reads, so every match is
+ * returned rather than just the first.
+ *
+ * @param {string} source
+ * @param {string} expr
+ * @returns {number[]} 1-based lines.
+ */
+function locate(source, expr) {
+  const needle = norm(expr);
+  if (!needle) return [];
+  /** @type {number[]} */
+  const out = [];
+  source.split("\n").forEach((line, i) => {
+    for (const m of line.matchAll(/\{([^{}]*)\}/g)) {
+      if (norm(m[1]) === needle) {
+        out.push(i + 1);
+        break;
+      }
+    }
+  });
+  return out;
+}
+
+/**
+ * Collapse whitespace so `{ cart | count }` and `{cart|count}` compare equal.
+ * @param {string} s
+ * @returns {string}
+ */
+function norm(s) {
+  return String(s ?? "")
+    .replace(/\s+/g, "")
+    .trim();
+}
+
+/**
+ * Compile and return one symbol per source construct, with every line resolved.
+ *
+ * Two foldings happen here, both forced by how the compiler works rather than by
+ * taste. A statically-unrolled `@loop` recompiles its body once per row, so a
+ * directive inside it records N identical symbols at one `file:line`; and a prose
+ * read inside such a loop records N line-less reads. Both collapse to one.
+ *
+ * The `compiled` half is the compiler's own result, derived from the function
+ * rather than restated here: widening it to `object` compiled fine while these
+ * lived under bench/, and silently lost every property check the moment they
+ * became part of the package.
+ *
+ * @typedef {ReturnType<typeof compileFromMemory>} Compiled
+ * @typedef {import("../compiler/context.js").Symbol} Symbol
+ *
+ * @param {Record<string, string>} files
+ * @param {string} entry
+ * @returns {{ok: true, symbols: Symbol[], compiled: Compiled} | {ok: false, error: string}}
+ */
+export function symbolsOf(files, entry) {
+  if (!files || !entry || !(entry in files)) {
+    return { ok: false, error: `no such file "${shortPath(entry)}" in this project` };
+  }
+  let compiled;
+  try {
+    compiled = compileFromMemory(files, entry);
+  } catch (err) {
+    return { ok: false, error: String(/** @type {any} */ (err)?.message ?? err) };
+  }
+
+  const seen = new Set();
+  /** @type {Symbol[]} */
+  const symbols = [];
+  for (const sym of compiled.symbols ?? []) {
+    const file = fileKeyOf(sym.file ?? entry, files);
+    const lines = sym.line != null ? [sym.line] : locate(files[file] ?? "", sym.detail ?? "");
+    // A read whose expression cannot be found is still a real read; keep it with
+    // a null line rather than silently dropping a fact the compiler established.
+    for (const line of lines.length ? lines : [null]) {
+      const id = `${file}|${line}|${sym.kind}|${sym.name}|${sym.detail ?? ""}`;
+      if (seen.has(id)) continue;
+      seen.add(id);
+      symbols.push({ ...sym, file, line });
+    }
+  }
+  symbols.sort((a, b) =>
+    a.file === b.file ? (a.line ?? 1e9) - (b.line ?? 1e9) : a.file < b.file ? -1 : 1
+  );
+  return { ok: true, symbols, compiled };
+}
+
+/**
+ * TOOL ENTRY POINT: outline(files, entry)
+ *
+ * @param {Record<string, string>} files
+ * @param {string} entry
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function outline(files, entry) {
+  const got = symbolsOf(files, entry);
+  if (!got.ok) return { ok: false, text: got.error, data: {} };
+
+  const byFile = new Map();
+  for (const s of got.symbols) {
+    if (!byFile.has(s.file)) byFile.set(s.file, []);
+    byFile.get(s.file).push(s);
+  }
+  // The entry always leads, even when it declares nothing, so a model asking for
+  // an outline of an empty page gets an answer rather than silence.
+  if (!byFile.has(entry)) byFile.set(entry, []);
+
+  const out = [];
+  for (const [file, syms] of [...byFile.entries()].sort((a, b) =>
+    a[0] === entry ? -1 : b[0] === entry ? 1 : a[0] < b[0] ? -1 : 1
+  )) {
+    const lines = (files[file] ?? "").split("\n").length;
+    out.push(`${shortPath(file)}  (${lines} lines)`);
+    if (!syms.length) {
+      out.push("  (no directives: plain markdown)");
+      continue;
+    }
+    for (const s of syms) {
+      // A block's span is printed as `:16-21`, because a model that is only
+      // told where a loop STARTS will replace its header and strand the closer.
+      const at = s.line == null ? "" : s.endLine ? `:${s.line}-${s.endLine}` : `:${s.line}`;
+      const tag = s.kind === "loop" && s.reactive ? "loop*" : s.kind;
+      out.push(`  ${at.padEnd(8)} ${tag.padEnd(9)} ${short(s.detail ?? s.name)}`);
+    }
+  }
+  const reactive = got.compiled.assets?.runtime;
+  out.push("");
+  out.push(
+    `${reactive ? "reactive page (ships the runtime)" : "static page (ships zero JavaScript)"}. loop* = reactive loop.`
+  );
+
+  return {
+    ok: true,
+    text: out.join("\n"),
+    data: { symbols: got.symbols, runtime: Boolean(reactive) }
+  };
+}
+
+/** Symbol kinds that WRITE the state key they name. */
+const WRITE_KINDS = new Set(["state", "store", "computed", "theme", "fetch", "action", "form"]);
+
+/**
+ * TOOL ENTRY POINT: refs(files, entry, name)
+ *
+ * Every place a symbol is declared, written, or read, across the entry and
+ * everything it includes. This is the query behind impact expansion: "what else
+ * needs to change" is a lookup, not something a model should be recalling.
+ *
+ * @param {Record<string, string>} files
+ * @param {string} entry
+ * @param {string} [name] Symbol name, e.g. `cart`. Missing is reported, not thrown.
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function refs(files, entry, name) {
+  const got = symbolsOf(files, entry);
+  if (!got.ok) return { ok: false, text: got.error, data: {} };
+  if (!name || typeof name !== "string") {
+    return { ok: false, text: 'refs needs a symbol name, e.g. refs("cart")', data: {} };
+  }
+
+  // A dotted read of `cart.total` is a reference to `cart`, and a loop over
+  // `products` targets it too. Match the head segment, not the whole string.
+  const head = (/** @type {string} */ s) => String(s ?? "").split(".")[0];
+  const hits = got.symbols.filter((s) => head(s.name) === name || head(s.target ?? "") === name);
+
+  if (!hits.length) {
+    const known = [...new Set(got.symbols.map((s) => head(s.name)))].filter(Boolean).sort();
+    return {
+      ok: true,
+      text: `"${name}" is not declared or referenced anywhere.\nThis page has: ${known.join(", ") || "nothing"}`,
+      data: { name, refs: [] }
+    };
+  }
+
+  const rows = hits.map((s) => ({
+    role:
+      s.kind === "read"
+        ? "read"
+        : s.kind === "loop"
+          ? "read"
+          : s.kind === "if"
+            ? "read"
+            : WRITE_KINDS.has(s.kind)
+              ? s.kind === "action"
+                ? "write"
+                : "declare"
+              : "read",
+    ...s
+  }));
+
+  const text = [
+    name,
+    ...rows.map(
+      (r) =>
+        `  ${r.role.padEnd(8)} ${short(r.detail ?? r.name).padEnd(DETAIL_MAX)} ${shortPath(r.file)}${r.line != null ? `:${r.line}` : ""}`
+    )
+  ].join("\n");
+
+  return { ok: true, text, data: { name, refs: rows } };
+}
+
+/**
+ * TOOL ENTRY POINT: deps(files, entry)
+ *
+ * What this page pulls in (includes, data files, collections) and, for any other
+ * page in the project, whether it pulls in the same thing. The reverse direction
+ * is what makes "I changed a partial, what breaks" answerable.
+ *
+ * @param {Record<string, string>} files
+ * @param {string} entry
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function deps(files, entry) {
+  const got = symbolsOf(files, entry);
+  if (!got.ok) return { ok: false, text: got.error, data: {} };
+
+  const reads = [...(got.compiled.deps ?? [])]
+    .map((d) => fileKeyOf(d, files))
+    .filter((d) => d !== entry)
+    .sort();
+  const collections = [...(got.compiled.collectionsUsed ?? [])].sort();
+
+  // Reverse edges: every OTHER page that also reads this entry. Compiling each
+  // one is affordable because it is in memory, and a page that fails to compile
+  // is reported rather than skipped silently.
+  const usedBy = [];
+  const broken = [];
+  for (const other of Object.keys(files)) {
+    if (other === entry || !/\.(wd|md)$/.test(other)) continue;
+    const o = symbolsOf(files, other);
+    if (!o.ok) {
+      broken.push(other);
+      continue;
+    }
+    if ([...(o.compiled.deps ?? [])].map((d) => fileKeyOf(d, files)).includes(entry)) {
+      usedBy.push(other);
+    }
+  }
+
+  const out = [`${shortPath(entry)}`];
+  out.push(reads.length ? `  includes: ${reads.map(shortPath).join(", ")}` : "  includes: nothing");
+  if (collections.length) out.push(`  loops collections: ${collections.join(", ")}`);
+  out.push(
+    usedBy.length ? `  included by: ${usedBy.map(shortPath).join(", ")}` : "  included by: nothing"
+  );
+  for (const b of broken)
+    out.push(`  note: ${shortPath(b)} does not compile, so its includes are unknown`);
+
+  return {
+    ok: true,
+    text: out.join("\n"),
+    data: { entry, includes: reads, collections, usedBy, broken }
+  };
+}

package/src/tools/parse.js

@@ -0,0 +1,81 @@
+// ---------------------------------------------------------------------------
+// Reading a tool call out of a small model's reply.
+//
+// Its own module so it can be tested without a GPU. It first lived inside the
+// runner, which meant importing the parser started a 50-task sweep — a thing
+// worth not repeating.
+//
+// Deliberately forgiving about the wrapper and strict about the content. Small
+// models fence their JSON about half the time and narrate around it the rest,
+// and none of that is a capability failure worth scoring. What IS worth scoring
+// is malformed or invented calls, so every rejection returns a reason the model
+// can act on rather than throwing.
+// ---------------------------------------------------------------------------
+
+/**
+ * @typedef {object} ToolCall
+ * @property {string} tool
+ * @property {object} args
+ */
+
+/**
+ * Pull the first complete JSON object out of a reply.
+ *
+ * Scans with a brace counter that is string- and escape-aware, because the one
+ * thing a Darkmown edit reliably contains is `{ count }`, and a naive
+ * `indexOf("}")` truncates exactly the calls that matter most.
+ *
+ * @param {string} reply
+ * @returns {{ok: true, call: ToolCall} | {ok: false, error: string}}
+ */
+export function parseToolCall(reply) {
+  const text = String(reply ?? "").replace(/```(?:json)?/g, "");
+  const start = text.indexOf("{");
+  if (start === -1) {
+    return {
+      ok: false,
+      error: 'no JSON object in that reply. Reply with exactly {"tool": "outline", "args": {}}'
+    };
+  }
+
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = start; i < text.length; i++) {
+    const c = text[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (c === "\\") {
+      escaped = true;
+      continue;
+    }
+    if (c === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+    if (c === "{") depth++;
+    else if (c === "}") {
+      depth--;
+      if (depth !== 0) continue;
+      const slice = text.slice(start, i + 1);
+      let parsed;
+      try {
+        parsed = JSON.parse(slice);
+      } catch (err) {
+        const why = err instanceof Error ? err.message : String(err);
+        return { ok: false, error: `that is not valid JSON (${why})` };
+      }
+      if (typeof parsed?.tool !== "string") {
+        return { ok: false, error: 'that JSON object has no "tool" field' };
+      }
+      return {
+        ok: true,
+        call: { tool: parsed.tool, args: parsed.args ?? parsed.arguments ?? {} }
+      };
+    }
+  }
+  return { ok: false, error: "that JSON object is never closed" };
+}

package/src/tools/validate.js

@@ -0,0 +1,169 @@
+// ---------------------------------------------------------------------------
+// TOOL: validate(files, entry)
+//
+// Compiles a whole virtual project with the REAL compiler and answers in the
+// model's own vocabulary: either "it compiles, here is what it became", or one
+// error with its code, its location, the corrective template, and a concrete
+// example line.
+//
+// This is the tool the agent loop leans on hardest, and it is nearly free:
+// `compileFromMemory` runs in process against an in-memory reader, no disk, so
+// a model can afford to check after every edit rather than at the end.
+//
+// The error path deliberately surfaces `err.wd` (the structured companion the
+// framework already attaches) instead of re-parsing the prose message. That is
+// the same discipline as the outline tool: never re-derive what the compiler
+// already knows.
+// ---------------------------------------------------------------------------
+
+import { compileFromMemory } from "../compiler.js";
+
+/**
+ * Reduce any path the compiler hands back to the caller's own key form.
+ *
+ * The caller addresses files with POSIX keys (`site/pages/index.wd`), but
+ * `compileFromMemory` resolves them against its virtual root with
+ * `path.resolve`, which is PLATFORM-NATIVE: on Windows the same file comes back
+ * as `C:\site\pages\index.wd`. Comparing that to the caller's keys matches
+ * nothing, so every symbol gets filed under a path that is not in the project
+ * and every symbol-targeted edit fails. The virtual root is not a real
+ * filesystem and its separator is an accident of the host, so normalising here
+ * is the fix rather than a workaround.
+ *
+ * @param {string} p
+ * @returns {string}
+ */
+export function posixKey(p) {
+  return String(p ?? "")
+    .replace(/\\/g, "/")
+    .replace(/^[A-Za-z]:/, "")
+    .replace(/^\/+/, "");
+}
+
+/**
+ * Shorten a virtual absolute path to what the author would call the file.
+ * The model never saw `/site/pages/`, so echoing it back is noise it may copy.
+ * @param {string} p
+ * @returns {string}
+ */
+export function shortPath(p) {
+  if (!p) return "";
+  return posixKey(p)
+    .replace(/^site\/pages\//, "")
+    .replace(/^site\/_\//, "_/");
+}
+
+/** A virtual project path as it appears mid-sentence, either separator. */
+const EMBEDDED_PATH = /(?:[A-Za-z]:)?[\\/]?site[\\/](?:pages|_)[\\/][^\s,)]*/g;
+
+/**
+ * The same shortening applied to paths EMBEDDED in a sentence.
+ *
+ * `shortPath` only anchors at the start of a string, which covered the location
+ * this tool prints and missed the one place the model actually reads: the
+ * compiler's own message, which names the file mid-sentence ("Malformed @loop in
+ * /site/pages/index.wd:3"). A model that is shown the virtual root will use it,
+ * and the path it then writes resolves to nothing.
+ *
+ * Whole tokens are matched and rewritten rather than separators being replaced
+ * globally, so a Windows path is normalised without touching anything else in
+ * the sentence.
+ *
+ * @param {string} s
+ * @returns {string}
+ */
+function shortenPaths(s) {
+  return String(s ?? "").replace(EMBEDDED_PATH, (match) => {
+    // A trailing `:3` line marker is part of the location, not the path.
+    const [, file, line] = /^(.*?)(:\d+)?$/.exec(match) ?? [];
+    return `${shortPath(file ?? match)}${line ?? ""}`;
+  });
+}
+
+/**
+ * TOOL ENTRY POINT.
+ *
+ * @param {Record<string, string>} files Project-relative path → contents, e.g.
+ *   `{"site/pages/index.wd": "---\ntitle: X\n---\n"}`.
+ * @param {string} entry Which file to compile, in the same key form.
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function validate(files, entry) {
+  if (!files || typeof files !== "object") {
+    return { ok: false, text: "validate needs a files object", data: { code: "TOOL_ARGS" } };
+  }
+  if (!entry || !(entry in files)) {
+    return {
+      ok: false,
+      text: `no such file "${shortPath(entry)}". The project has: ${Object.keys(files).map(shortPath).join(", ")}`,
+      data: { code: "TOOL_ARGS" }
+    };
+  }
+
+  let compiled;
+  try {
+    compiled = compileFromMemory(files, entry);
+  } catch (err) {
+    // Every author-facing compile error carries the compiler's own structured
+    // payload on `err.wd` (see wdError in src/compiler/context.js). Naming that
+    // type rather than restating its fields is what keeps this in step when a
+    // field is added there.
+    const wd = /** @type {Partial<import("../compiler/context.js").WdErrorInfo>} */ (
+      /** @type {any} */ (err)?.wd ?? {}
+    );
+    // `file` is optional: a pure string->string pass (compileSkin) has no file
+    // to name, so fall back to the entry the caller asked about.
+    const where = wd.file
+      ? `${shortPath(wd.file)}${wd.line ? `:${wd.line}` : ""}`
+      : shortPath(entry);
+    // Strip the "[CODE] " prefix and the trailing "Use: …" the message already
+    // carries, because both are re-emitted below in a fixed position. A model
+    // copying from a single predictable shape beats one parsing prose.
+    const message = shortenPaths(
+      String(/** @type {any} */ (err)?.message ?? err)
+        .replace(/^\[[A-Z]{2}\d{3}\]\s*/, "")
+        .replace(/\s*Use:\s*[\s\S]*$/, "")
+    ).trim();
+
+    const lines = [`${wd.code ?? "ERROR"} at ${where}`, `  ${message}`];
+    if (wd.hint) lines.push(`  use: ${shortenPaths(wd.hint)}`);
+    if (wd.example) lines.push(`  e.g. ${shortenPaths(wd.example)}`);
+
+    return {
+      ok: false,
+      text: lines.join("\n"),
+      data: {
+        code: wd.code ?? null,
+        file: shortPath(wd.file ?? entry),
+        line: wd.line ?? null,
+        message,
+        hint: wd.hint ?? null,
+        example: wd.example ?? null
+      }
+    };
+  }
+
+  // A clean compile still has things worth telling the model, and one of them is
+  // load-bearing: whether the page went reactive. A brochure page that quietly
+  // starts shipping the runtime is a regression the framework cares about more
+  // than almost anything else, and it is invisible in the HTML.
+  const skins = Object.keys(compiled.assets?.skins ?? {}).length;
+  const facts = [
+    compiled.assets?.runtime ? "reactive (ships the runtime)" : "static (ships zero JavaScript)",
+    `${skins} skin${skins === 1 ? "" : "s"}`
+  ];
+  const lines = [`compiles. ${facts.join(", ")}.`];
+  for (const w of compiled.warnings ?? []) lines.push(`  warning: ${w}`);
+
+  return {
+    ok: true,
+    text: lines.join("\n"),
+    data: {
+      runtime: Boolean(compiled.assets?.runtime),
+      skins,
+      warnings: [...(compiled.warnings ?? [])],
+      title: compiled.meta?.title ?? null,
+      html: compiled.html
+    }
+  };
+}

package/types/builder.d.ts

@@ -118,6 +118,27 @@ export type EmittedPage = {
     assets: import("./compiler.js").Assets;
     warnings: string[];
 };
+/**
+ * Site-wide metadata resolved from the home page's frontmatter.
+ */
+export type SiteIdentity = {
+    /**
+     * Absolute origin without a trailing slash, or "".
+     */
+    siteUrl: string;
+    /**
+     * Site title (also the RSS channel title).
+     */
+    title: string;
+    /**
+     * Site description (also the RSS channel description).
+     */
+    description: string;
+    /**
+     * `"allow"` or `"deny"`, from `ai_crawlers:`.
+     */
+    aiCrawlers: string;
+};
 export type Paths = import("./config.js").Paths;
 export type Assets = import("./compiler.js").Assets;
 export type Meta = import("./compiler.js").Meta;