@zvndev/darkmown 2.4.1-next.7.ge25dbba → 2.5.0

package/src/tools/apply.js

@@ -0,0 +1,274 @@
+// ---------------------------------------------------------------------------
+// TOOL: apply(files, entry, edits)
+//
+// Applies targeted edits to one file and hands back the new content. It does not
+// write anything and does not compile: the caller applies, then validates, so a
+// rejected edit costs nothing.
+//
+// WHY THERE ARE THREE WAYS TO SAY WHERE
+//
+// The obvious design is anchored edits only, ruling out line numbers because
+// "small models cannot count lines and a stale number silently corrupts a
+// file". The first half of that is the wrong worry and the second half is real
+// but fixable.
+//
+// Measured across file sizes, quoting a line back verbatim gets HARDER as a
+// file grows, and the anchor is where anchored editing dies: it accounts for
+// 42% of failed tasks at 16 lines and 67% at 244. The model is not failing to
+// count. It is failing to recall a string it read once.
+//
+// So this tool accepts three ways to say where, and the model is expected to use
+// whichever the previous tool call already handed it:
+//
+//   { anchor: "…" }   verbatim source text, must appear exactly once
+//   { line: 7 }       a line number, as printed by `outline`
+//   { symbol: "cart" } a declaration, as named by `outline` / `refs`
+//
+// The objection to line numbers was staleness, and staleness is what the design
+// prevents rather than what it ignores: every target is resolved against the
+// snapshot passed to THIS call, all edits in a call are applied against that one
+// snapshot in descending line order, and a `line` target must still contain what
+// the caller expected if it says so. The model is never counting; it is quoting
+// a number a tool just printed.
+// ---------------------------------------------------------------------------
+
+import { symbolsOf } from "./outline.js";
+import { shortPath } from "./validate.js";
+
+/** The operations a caller may ask for. Closed, and validated per edit. */
+export const OPS = new Set(["replace", "insert_after", "insert_before", "delete"]);
+
+/**
+ * One edit, as a caller (or a model) writes it.
+ *
+ * Every field is optional because a malformed edit is REPORTED, not thrown: the
+ * whole point of this surface is that a model gets a sentence back telling it
+ * what to fix. Typing them as required would describe a contract this module
+ * deliberately does not enforce at the boundary.
+ *
+ * @typedef {object} Edit
+ * @property {string} [op] One of {@link OPS}.
+ * @property {number} [line] 1-based line, as printed by `outline`.
+ * @property {string} [symbol] A symbol target, e.g. `state:cart`.
+ * @property {string} [anchor] Exact source text to find.
+ * @property {string} [text] Replacement or inserted text.
+ * @property {string} [expect] What the caller believed is at `line`, for staleness.
+ */
+
+/** Symbol kinds that DEFINE the name they carry, as opposed to referencing it. */
+const DECLARING_KINDS = new Set([
+  "state",
+  "store",
+  "computed",
+  "theme",
+  "fetch",
+  "form",
+  "field",
+  "loop"
+]);
+
+/**
+ * Resolve one edit's target to a 1-based inclusive line range in `source`.
+ *
+ * Every rejection names what was wrong AND what would have worked, because a
+ * rejection a model cannot act on is just a failed attempt.
+ *
+ * @param {string} source
+ * @param {Edit} edit
+ * @param {import("../compiler/context.js").Symbol[]} symbols
+ * @returns {{ok: true, from: number, to: number} | {ok: false, error: string}}
+ */
+export function resolveTarget(source, edit, symbols) {
+  const lines = source.split("\n");
+
+  if (typeof edit.line === "number") {
+    if (!Number.isInteger(edit.line) || edit.line < 1 || edit.line > lines.length) {
+      return {
+        ok: false,
+        error: `line ${edit.line} is outside the file, which has ${lines.length} lines. To add something at the end, use {"op": "insert_after", "line": ${lines.length}}.`
+      };
+    }
+    // Optional belt and braces: if the caller says what it expected to find, a
+    // snapshot that moved under it is caught here instead of corrupting the file.
+    if (typeof edit.expect === "string" && !lines[edit.line - 1].includes(edit.expect)) {
+      return {
+        ok: false,
+        error: `line ${edit.line} is "${lines[edit.line - 1].trim()}", not the expected "${edit.expect}". Run outline again; the file has changed.`
+      };
+    }
+    return { ok: true, from: edit.line, to: edit.line };
+  }
+
+  if (typeof edit.symbol === "string") {
+    // `state:cart` disambiguates when a name is both declared and looped.
+    const [maybeKind, maybeName] = edit.symbol.includes(":")
+      ? edit.symbol.split(":")
+      : [null, edit.symbol];
+    // Unqualified means "where this is DECLARED". A read or a conditional that
+    // mentions the name is a reference, not a definition, and is targeted by the
+    // line `refs` prints for it. Without this, the most natural target form is
+    // ambiguous on every page that both declares and uses a symbol, which is
+    // every page worth editing.
+    const candidates = symbols.filter(
+      (s) => s.line != null && s.name === maybeName && (!maybeKind || s.kind === maybeKind)
+    );
+    const hits = maybeKind ? candidates : candidates.filter((s) => DECLARING_KINDS.has(s.kind));
+    if (hits.length === 0) {
+      const known = [
+        ...new Set(symbols.filter((s) => s.line != null).map((s) => `${s.kind}:${s.name}`))
+      ].sort();
+      return {
+        ok: false,
+        error: `no symbol "${edit.symbol}". This file declares: ${known.join(", ") || "nothing"}`
+      };
+    }
+    if (hits.length > 1) {
+      return {
+        ok: false,
+        error: `"${edit.symbol}" is ambiguous: ${hits.map((h) => `${h.kind}:${h.name} at line ${h.line}`).join(", ")}. Qualify it as kind:name, or use a line.`
+      };
+    }
+    // A block directive spans its whole construct. Replacing a `@loop` means
+    // replacing the loop, not decapitating it and stranding the `@endloop`,
+    // which is the most common way a symbol-targeted edit breaks a file.
+    //
+    // `line` is nullable on a Symbol — a prose read the compiler could not
+    // locate carries none — but `candidates` above already excluded every one
+    // of those, so a hit always has a line. The cast records that guarantee
+    // instead of re-checking it: the check that used to sit here could not be
+    // reached, and unreachable error paths are worse than none, because they
+    // read as handled cases and are never exercised.
+    const at = /** @type {number} */ (hits[0].line);
+    return { ok: true, from: at, to: hits[0].endLine ?? at };
+  }
+
+  if (typeof edit.anchor === "string") {
+    const anchor = edit.anchor.replace(/\n+$/, "");
+    if (!anchor.trim()) return { ok: false, error: "an empty anchor matches nothing" };
+    const anchorLines = anchor.split("\n");
+    /** @type {number[]} */
+    const starts = [];
+    for (let i = 0; i + anchorLines.length <= lines.length; i++) {
+      if (anchorLines.every((a, k) => lines[i + k] === a)) starts.push(i + 1);
+    }
+    if (starts.length === 0) {
+      return {
+        ok: false,
+        error: `that anchor is not in the file, character for character. Use outline to get a line number instead of quoting.`
+      };
+    }
+    if (starts.length > 1) {
+      return {
+        ok: false,
+        error: `that anchor appears ${starts.length} times (lines ${starts.join(", ")}). Add a neighbouring line to make it unique, or use a line number.`
+      };
+    }
+    return { ok: true, from: starts[0], to: starts[0] + anchorLines.length - 1 };
+  }
+
+  return {
+    ok: false,
+    error: `this edit says nothing about where it goes. Give one of: {"line": 7}, {"symbol": "cart"}, or {"anchor": "the exact source line"}.`
+  };
+}
+
+/**
+ * TOOL ENTRY POINT.
+ *
+ * @param {Record<string, string>} files
+ * @param {string} entry File to edit, in the same key form as `files`.
+ * @param {Edit[]} [edits] Missing or empty is reported, not thrown.
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function apply(files, entry, edits) {
+  if (!files || !(entry in files)) {
+    return { ok: false, text: `no such file "${shortPath(entry)}" in this project`, data: {} };
+  }
+  if (!Array.isArray(edits) || edits.length === 0) {
+    return { ok: false, text: "apply needs a non-empty list of edits", data: {} };
+  }
+
+  const source = files[entry];
+  // The symbol table is only needed for `symbol:` targets, and a file that does
+  // not compile still has anchors and line numbers. So a compile failure here is
+  // not fatal: it only removes the symbol vocabulary.
+  const got = symbolsOf(files, entry);
+  const symbols = got.ok ? got.symbols.filter((s) => s.file === entry) : [];
+
+  /** @type {{from: number, to: number, edit: Edit}[]} */
+  const resolved = [];
+  for (const [i, edit] of edits.entries()) {
+    if (!edit?.op || !OPS.has(edit.op)) {
+      return {
+        ok: false,
+        text: `edit ${i + 1}: unknown op "${edit?.op}". Use one of: ${[...OPS].join(", ")}`,
+        data: {}
+      };
+    }
+    if (edit.op !== "delete" && typeof edit.text !== "string") {
+      return { ok: false, text: `edit ${i + 1}: ${edit.op} needs "text"`, data: {} };
+    }
+    if (typeof edit.symbol === "string" && !got.ok) {
+      return {
+        ok: false,
+        text: `edit ${i + 1}: cannot use a symbol target because the file does not compile (${got.error}). Fix the error first, or target a line.`,
+        data: {}
+      };
+    }
+    const target = resolveTarget(source, edit, symbols);
+    if (!target.ok) return { ok: false, text: `edit ${i + 1}: ${target.error}`, data: {} };
+    resolved.push({ from: target.from, to: target.to, edit });
+  }
+
+  // Overlap is a caller bug that would otherwise produce a plausible-looking but
+  // wrong file, which is the single worst outcome for an automated loop.
+  const ordered = [...resolved].sort((a, b) => a.from - b.from);
+  for (let i = 1; i < ordered.length; i++) {
+    if (ordered[i].from <= ordered[i - 1].to) {
+      return {
+        ok: false,
+        text: `two edits target overlapping lines (${ordered[i - 1].from}-${ordered[i - 1].to} and ${ordered[i].from}-${ordered[i].to}). Combine them into one edit.`,
+        data: {}
+      };
+    }
+  }
+
+  // Descending, so an earlier edit never shifts a later one's line numbers.
+  const lines = source.split("\n");
+  for (const { from, to, edit } of [...resolved].sort((a, b) => b.from - a.from)) {
+    const body = edit.op === "delete" ? [] : (edit.text ?? "").replace(/\n$/, "").split("\n");
+    if (edit.op === "replace" || edit.op === "delete")
+      lines.splice(from - 1, to - from + 1, ...body);
+    else if (edit.op === "insert_after") lines.splice(to, 0, ...body);
+    else lines.splice(from - 1, 0, ...body);
+  }
+
+  const content = lines.join("\n");
+  const changed = countChanged(source, content);
+  return {
+    ok: true,
+    text: `applied ${resolved.length} edit${resolved.length === 1 ? "" : "s"} to ${shortPath(entry)} (${changed} line${changed === 1 ? "" : "s"} differ)`,
+    data: { content, files: { ...files, [entry]: content }, edits: resolved.length, changed }
+  };
+}
+
+/**
+ * How many lines differ, as a blunt blast-radius signal for the caller.
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+function countChanged(a, b) {
+  const x = a.split("\n");
+  const y = b.split("\n");
+  let same = 0;
+  const pool = [...y];
+  for (const line of x) {
+    const i = pool.indexOf(line);
+    if (i !== -1) {
+      same++;
+      pool.splice(i, 1);
+    }
+  }
+  return Math.max(x.length, y.length) - same;
+}

package/src/tools/grammar.js

@@ -0,0 +1,171 @@
+// ---------------------------------------------------------------------------
+// TOOL: grammar(categories)
+//
+// Hands back only the cheatsheet rows an edit actually needs, plus the closed
+// enum of action tokens when the edit involves an action.
+//
+// Why this is a tool rather than a fixed prompt block. Measured against small
+// local models, the cheatsheet behaves as a load-bearing runtime asset and not
+// as documentation: revising it moved a 1.5B model by 22 points, and a single
+// bad example line was copied verbatim into four consecutive attempts. A model
+// shown only the rows it needs has fewer wrong rows available to copy, and
+// spends fewer tokens getting them.
+//
+// Everything here is derived from `directiveCatalog()`, which the compiler
+// generates from its own tables. Nothing is hand-transcribed, so a directive
+// that changes in the compiler changes here in the same commit or the drift
+// guard in the tests fails.
+// ---------------------------------------------------------------------------
+
+import { CATALOG_ACTION_TOKENS, directiveCatalog } from "../catalog.js";
+
+/**
+ * Every directive name, grouped into the category a person would ask for.
+ *
+ * Deliberately exhaustive and deliberately hand-maintained: `tools.test.mjs`
+ * asserts that this covers the catalog exactly, so adding a directive to the
+ * compiler without categorising it here fails a test rather than silently
+ * dropping the directive out of every grammar slice.
+ *
+ * The names match `errorAreas` in the catalog, so a compile error in area
+ * "loops" can be answered with `grammar(["loops"])` without a second mapping.
+ */
+export const CATEGORIES = {
+  state: [":state", ":store", ":computed", ":theme"],
+  loops: ["@loop"],
+  actions: [":button", ":effect", ":every"],
+  forms: [
+    ":form",
+    ":input",
+    ":textarea",
+    ":select",
+    ":checkbox",
+    ":radio",
+    ":submit",
+    ":bind",
+    ":slider"
+  ],
+  fetch: [":fetch"],
+  structure: ["@include", ":::", ":if", ":carousel"],
+  media: [":video", ":audio", ":embed"]
+};
+
+/** Categories that also need the loop clause list and loop variables. */
+const NEEDS_LOOP_EXTRAS = new Set(["loops"]);
+/** Categories that also need the closed action-token enum. */
+const NEEDS_ACTION_TOKENS = new Set(["actions"]);
+/** Categories whose expressions use the comparison operators. */
+const NEEDS_PREDICATES = new Set(["loops", "structure", "state"]);
+
+/** @returns {string[]} Every category name, for callers that want to enumerate. */
+export function categoryNames() {
+  return Object.keys(CATEGORIES);
+}
+
+/**
+ * One directive rendered as the model should write it.
+ *
+ * Syntax line then a concrete example, because small models copy `[optional]`
+ * bracket notation into the file verbatim. The example is always a real
+ * compilable line.
+ *
+ * @param {{name: string, syntax: string, description: string, example: string}} d
+ * @returns {string}
+ */
+function renderDirective(d) {
+  const lines = [`${d.name}  ${d.syntax}`, `  ${d.description}`];
+  if (d.example && d.example !== d.syntax) lines.push(`  e.g. ${d.example}`);
+  return lines.join("\n");
+}
+
+/**
+ * TOOL ENTRY POINT.
+ *
+ * @param {string[]} [categories] Category names. Omitted or empty means all of
+ *   them, which is the whole cheatsheet and is the thing this tool exists to
+ *   avoid sending.
+ * Failures use the same {ok, text, data} shape as every other tool. It used to
+ * answer {ok: false, error}, which `Session` never reads, so an unknown category
+ * handed the model an undefined message.
+ *
+ * @returns {{ok: boolean, text: string, data: object}}
+ */
+export function grammar(categories) {
+  const all = categoryNames();
+  const want = !categories || categories.length === 0 ? all : categories;
+
+  const unknown = want.filter((c) => !all.includes(c));
+  if (unknown.length) {
+    return {
+      ok: false,
+      text: `unknown categor${unknown.length === 1 ? "y" : "ies"} ${unknown.map((u) => `"${u}"`).join(", ")}. Available: ${all.join(", ")}`,
+      data: { code: "TOOL_ARGS" }
+    };
+  }
+
+  const catalog = directiveCatalog();
+  const byName = new Map(catalog.directives.map((d) => [d.name, d]));
+  const wanted = want.flatMap((c) => CATEGORIES[/** @type {keyof CATEGORIES} */ (c)] ?? []);
+  const directives = /** @type {NonNullable<ReturnType<typeof byName.get>>[]} */ (
+    wanted.map((n) => byName.get(n)).filter(Boolean)
+  );
+
+  const sections = [directives.map(renderDirective).join("\n\n")];
+  // Widened up front: the optional sections below add their own keys, and an
+  // inferred literal type would reject each one.
+  /** @type {{categories: string[], directives: string[], loopClauses?: string[],
+   *          actionTokens?: string[], predicateOps?: string[]}} */
+  const data = { categories: want, directives: directives.map((d) => d.name) };
+
+  if (want.some((c) => NEEDS_LOOP_EXTRAS.has(c))) {
+    sections.push(
+      "Loop clauses, all optional, in this exact order after `into <item>`:\n" +
+        catalog.loopClauses.map((c) => `  ${c.name}  ${c.syntax ?? ""}`.trimEnd()).join("\n") +
+        "\nInside a loop body you may also use: " +
+        catalog.loopVariables.map((v) => `{ ${v.name} }`).join("  ")
+    );
+    data.loopClauses = catalog.loopClauses.map((c) => c.name);
+  }
+
+  if (want.some((c) => NEEDS_ACTION_TOKENS.has(c))) {
+    sections.push(
+      "Action tokens. This list is closed; anything else is a compile error:\n" +
+        catalog.actionOps.map((a) => `  ${a.syntax}   ${a.description}`).join("\n")
+    );
+    data.actionTokens = [...CATALOG_ACTION_TOKENS];
+  }
+
+  if (want.some((c) => NEEDS_PREDICATES.has(c))) {
+    sections.push(
+      "Comparison operators for `where` and `:if`: " +
+        catalog.predicateOps.map((o) => o.name).join(" ") +
+        `\nJoin conditions with: ${catalog.predicateJoiners.join(" ")}`
+    );
+    data.predicateOps = catalog.predicateOps.map((o) => o.name);
+  }
+
+  return { ok: true, text: sections.join("\n\n"), data };
+}
+
+/**
+ * How much a slice saves against sending everything.
+ *
+ * The plan claims filtering the cheatsheet is both a context saving and a
+ * hallucination guard. The saving half is measurable, so it gets measured
+ * rather than asserted.
+ *
+ * @param {string[]} categories
+ * @returns {{sliceChars: number, fullChars: number, saved: number, savedPct: number}}
+ */
+export function grammarCost(categories) {
+  const slice = grammar(categories);
+  const full = grammar();
+  const sliceChars = slice.ok ? slice.text.length : 0;
+  const fullChars = full.ok ? full.text.length : 0;
+  return {
+    sliceChars,
+    fullChars,
+    saved: fullChars - sliceChars,
+    savedPct: fullChars ? Math.round((100 * (fullChars - sliceChars)) / fullChars) : 0
+  };
+}

package/src/tools/index.js

@@ -0,0 +1,170 @@
+// ---------------------------------------------------------------------------
+// The tool surface, as one dispatch point.
+//
+// Six tools, all derived from the compiler rather than written beside it, all
+// answering in text a model can nearly paste.
+//
+//   grammar(categories)          only the cheatsheet rows this edit needs
+//   outline(entry)               what is in the page, with line numbers
+//   refs(name)                   every declare / write / read of a symbol
+//   deps(entry)                  what this page pulls in, and what pulls it in
+//   apply(entry, edits)          targeted edits against a snapshot
+//   validate(entry)              compile it for real, or say exactly what broke
+//
+// A `Session` holds the project files so a model never has to restate them, and
+// so `apply` and `outline` are guaranteed to be looking at the same snapshot.
+// Everything is in memory: a full compile is milliseconds, which is what makes
+// "validate after every edit" affordable rather than aspirational.
+// ---------------------------------------------------------------------------
+
+import { apply } from "./apply.js";
+import { grammar } from "./grammar.js";
+import { deps, outline, refs } from "./outline.js";
+import { parseToolCall } from "./parse.js";
+import { validate } from "./validate.js";
+
+export { apply, deps, grammar, outline, parseToolCall, refs, validate };
+
+/**
+ * The tool list as a model should be told about it: name, when to reach for it,
+ * and the exact argument shape. Kept here rather than in a prompt file so the
+ * description can never drift from the dispatcher.
+ */
+export const TOOL_SPECS = [
+  {
+    name: "outline",
+    args: "{}",
+    use: "List everything declared in the page with its line number. Call this FIRST; every other tool's targets come from it."
+  },
+  {
+    name: "refs",
+    args: '{ "name": "cart" }',
+    use: "Find every place a symbol is declared, written, or read. Call this before changing a symbol, to see what else moves."
+  },
+  {
+    name: "deps",
+    args: "{}",
+    use: "What this page includes, and which other pages include it."
+  },
+  {
+    name: "grammar",
+    args: '{ "categories": ["state", "actions"] }',
+    use: "Get the exact syntax for one area. Categories: state, loops, actions, forms, fetch, structure, media."
+  },
+  {
+    name: "apply",
+    args: '{ "edits": [{ "op": "replace", "line": 7, "text": ":store cart = [] persist" }] }',
+    use: 'Make an edit. Say WHERE with one of "line" (from outline), "symbol" (e.g. "state:cart"), or "anchor" (exact source text). Ops: replace, insert_after, insert_before, delete.'
+  },
+  {
+    name: "validate",
+    args: "{}",
+    use: "Compile the page for real. Do this after every apply."
+  }
+];
+
+/**
+ * A working copy of a project plus the tools that operate on it.
+ *
+ * Edits are staged in the session, not written to disk, so a caller can apply,
+ * validate, and roll back without touching a filesystem. `history` exists so a
+ * run can be scored later on how many calls it took, which is the number that
+ * decides whether the tool surface is worth its context budget.
+ */
+export class Session {
+  /**
+   * @param {Record<string, string>} files Project-relative path → contents.
+   * @param {string} entry The file being edited.
+   */
+  constructor(files, entry) {
+    this.files = { ...files };
+    this.entry = entry;
+    this.original = files[entry];
+    /** @type {{tool: string, args: object, ok: boolean}[]} */
+    this.history = [];
+  }
+
+  /**
+   * Dispatch one tool call.
+   *
+   * An unknown tool name is answered with the list rather than an exception,
+   * because a model that guessed a name can recover from a list and cannot
+   * recover from a stack trace.
+   *
+   * @param {string} tool
+   * @param {object} [args]
+   * @returns {{ok: boolean, text: string, data?: object}}
+   */
+  call(tool, args = {}) {
+    const result = this.#dispatch(tool, args);
+    this.history.push({ tool, args, ok: result.ok });
+    return result;
+  }
+
+  /**
+   * The union of every argument any tool takes. Declaring it beats `object`,
+   * which type-checked nothing and let a renamed field through silently.
+   *
+   * @typedef {{entry?: string, name?: string, categories?: string[],
+   *            edits?: object[]}} ToolArgs
+   *
+   * @param {string} tool
+   * @param {ToolArgs} args
+   * @returns {{ok: boolean, text: string, data?: object}}
+   */
+  #dispatch(tool, args) {
+    switch (tool) {
+      case "outline":
+        return outline(this.files, args.entry ?? this.entry);
+      case "refs":
+        return refs(this.files, args.entry ?? this.entry, args.name);
+      case "deps":
+        return deps(this.files, args.entry ?? this.entry);
+      case "grammar":
+        return grammar(args.categories);
+      case "validate":
+        return validate(this.files, args.entry ?? this.entry);
+      case "apply": {
+        const target = args.entry ?? this.entry;
+        const result = apply(this.files, target, args.edits);
+        // Staging on success is what keeps outline and apply on one snapshot.
+        if (result.ok)
+          this.files = /** @type {{files: Record<string, string>}} */ (result.data).files;
+        return result;
+      }
+      default:
+        return {
+          ok: false,
+          text: `no tool called "${tool}". Available: ${TOOL_SPECS.map((t) => t.name).join(", ")}`
+        };
+    }
+  }
+
+  /** @returns {string} The current content of the file under edit. */
+  content() {
+    return this.files[this.entry];
+  }
+
+  /** Discard all staged edits. @returns {void} */
+  reset() {
+    this.files = { ...this.files, [this.entry]: this.original };
+  }
+}
+
+/**
+ * The tool list rendered for a system prompt.
+ *
+ * Same discipline as the cheatsheet: concrete argument examples, never
+ * `[optional]` bracket notation, because small models copy schematic notation
+ * into their output verbatim.
+ *
+ * @returns {string}
+ */
+export function toolPrompt() {
+  return [
+    "You have these tools. Call one at a time, as a single JSON object:",
+    '{"tool": "outline", "args": {}}',
+    "",
+    ...TOOL_SPECS.map((t) => `${t.name}  args: ${t.args}\n  ${t.use}`)
+  ].join("\n");
+}