@ferax564/noma-cli 0.11.0

package/src/renderer-json.ts

@@ -0,0 +1,9 @@
+import type { DocumentNode } from "./ast.js";
+
+export interface JsonRenderOptions {
+  pretty?: boolean;
+}
+
+export function renderJson(doc: DocumentNode, options: JsonRenderOptions = {}): string {
+  return JSON.stringify(doc, null, options.pretty === false ? 0 : 2);
+}

package/src/renderer-llm.ts

@@ -0,0 +1,320 @@
+import type { DirectiveNode, DocumentNode, Node, SectionNode } from "./ast.js";
+import { walk } from "./ast.js";
+import { inlineToPlain } from "./inline.js";
+
+export interface RenderLlmOptions {
+  /**
+   * When set, ::memory directives whose `last_seen` attribute is older than
+   * `days` from `now` are omitted from the output, AND ::memory_index body
+   * lines whose [[wikilinks]] resolve only to omitted memories are dropped
+   * (no dangling references in the LLM context).
+   *
+   * Durable memory types (`user`, `feedback`) are kept regardless of
+   * `last_seen` unless they carry `expired=true`. Time-window staleness
+   * applies only to `project` and `reference` memories by default, since
+   * those are the types whose facts go stale over calendar time.
+   */
+  excludeStale?: { now: Date; days: number };
+  /** Include only matching AST node types or directive names, plus ancestors. */
+  select?: string[];
+  /** Omit matching AST node types or directive names and their children. */
+  exclude?: string[];
+  /** Maximum output length in characters, trimmed at a line boundary when possible. */
+  budget?: number;
+}
+
+interface RenderCtx extends RenderLlmOptions {
+  excludedMemoryIds: Set<string>;
+  selectSet: Set<string>;
+  excludeSet: Set<string>;
+}
+
+const STALE_OPT_IN_TYPES = new Set(["project", "reference"]);
+
+/**
+ * Deterministic plain-text export designed for LLM context windows.
+ * Stable line ordering, explicit semantic tags, no HTML noise.
+ */
+export function renderLlm(doc: DocumentNode, options: RenderLlmOptions = {}): string {
+  const ctx: RenderCtx = {
+    ...options,
+    excludedMemoryIds: computeExcludedMemoryIds(doc, options),
+    selectSet: normalizeSelectors(options.select),
+    excludeSet: normalizeSelectors(options.exclude),
+  };
+  const out: string[] = [];
+  if (doc.meta.title) out.push(`# ${String(doc.meta.title)}`);
+  for (const child of doc.children) {
+    if (shouldEmit(child, ctx, false)) emit(child, out, 0, ctx, false);
+  }
+  const rendered = out.join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n";
+  return applyBudget(rendered, options.budget);
+}
+
+function normalizeSelectors(values: string[] | undefined): Set<string> {
+  const out = new Set<string>();
+  for (const raw of values ?? []) {
+    for (const part of raw.split(/[,\s]+/)) {
+      const clean = part.trim().toLowerCase();
+      if (clean) out.add(clean);
+    }
+  }
+  return out;
+}
+
+function applyBudget(text: string, budget: number | undefined): string {
+  if (budget === undefined || !Number.isFinite(budget) || budget <= 0) return text;
+  if (text.length <= budget) return text;
+  const marker = `\n\n[LLM context truncated at ${budget} characters]\n`;
+  if (budget <= marker.length) return marker.slice(0, budget);
+  const limit = budget - marker.length;
+  const cut = text.lastIndexOf("\n", limit);
+  return text.slice(0, cut > 0 ? cut : limit).trimEnd() + marker;
+}
+
+function computeExcludedMemoryIds(
+  doc: DocumentNode,
+  options: RenderLlmOptions,
+): Set<string> {
+  const excluded = new Set<string>();
+  if (!options.excludeStale) return excluded;
+  for (const node of walk(doc)) {
+    if (node.type !== "directive") continue;
+    if (node.name !== "memory") continue;
+    if (!node.id) continue;
+    if (isStale(node, options.excludeStale)) excluded.add(node.id);
+  }
+  return excluded;
+}
+
+function emit(
+  node: Node,
+  out: string[],
+  depth: number,
+  opts: RenderCtx,
+  forceSubtree: boolean,
+): void {
+  switch (node.type) {
+    case "document":
+      for (const child of node.children) {
+        if (shouldEmit(child, opts, forceSubtree)) emit(child, out, depth, opts, forceSubtree);
+      }
+      return;
+    case "section":
+      emitSection(node, out, depth, opts, forceSubtree);
+      return;
+    case "paragraph":
+      out.push(inlineToPlain(node.content));
+      out.push("");
+      return;
+    case "code":
+      out.push("```" + (node.lang ?? ""));
+      out.push(node.content);
+      out.push("```");
+      out.push("");
+      return;
+    case "list":
+      for (const item of node.items) {
+        out.push(`- ${inlineToPlain(item.content)}`);
+      }
+      out.push("");
+      return;
+    case "list_item":
+      out.push(`- ${inlineToPlain(node.content)}`);
+      return;
+    case "quote":
+      for (const line of node.content.split("\n")) out.push(`> ${line}`);
+      out.push("");
+      return;
+    case "thematic_break":
+      out.push("---");
+      out.push("");
+      return;
+    case "table": {
+      const widths = node.header.map((h, i) =>
+        Math.max(
+          h.length,
+          ...node.rows.map((r) => (r[i] ?? "").length),
+          3,
+        ),
+      );
+      const fmt = (cells: string[]) =>
+        "| " +
+        cells.map((c, i) => c.padEnd(widths[i] ?? c.length)).join(" | ") +
+        " |";
+      out.push(fmt(node.header));
+      out.push(
+        "| " +
+          widths.map((w, i) => {
+            const a = node.align[i];
+            const dashes = "-".repeat(Math.max(3, w));
+            if (a === "center") return `:${dashes.slice(0, -2)}-:`;
+            if (a === "right") return `${dashes.slice(0, -1)}:`;
+            if (a === "left") return `:${dashes.slice(0, -1)}`;
+            return dashes;
+          }).join(" | ") +
+          " |",
+      );
+      for (const row of node.rows) out.push(fmt(row));
+      out.push("");
+      return;
+    }
+    case "directive":
+      emitDirective(node, out, depth, opts, forceSubtree);
+      return;
+    case "frontmatter":
+      return;
+    default: {
+      const _exhaustive: never = node;
+      void _exhaustive;
+    }
+  }
+}
+
+function emitSection(
+  node: SectionNode,
+  out: string[],
+  depth: number,
+  opts: RenderCtx,
+  forceSubtree: boolean,
+): void {
+  const hashes = "#".repeat(node.level);
+  out.push(`${hashes} ${node.title}${node.id ? `  [#${node.id}]` : ""}`);
+  out.push("");
+  const childForce = forceSubtree || matchesSelector(node, opts.selectSet);
+  for (const child of node.children) {
+    if (shouldEmit(child, opts, childForce)) emit(child, out, depth, opts, childForce);
+  }
+}
+
+const VERBATIM_BODY = new Set(["diagram", "plotly", "math"]);
+
+function emitDirective(
+  node: DirectiveNode,
+  out: string[],
+  depth: number,
+  opts: RenderCtx,
+  forceSubtree: boolean,
+): void {
+  if (node.name === "html" || node.name === "svg" || node.name === "script") {
+    out.push(`[${node.name.toUpperCase()} escape-hatch block omitted from LLM context]`);
+    out.push("");
+    return;
+  }
+  if (node.name === "memory" && opts.excludeStale && isStale(node, opts.excludeStale)) {
+    return;
+  }
+  const tag = node.name.toUpperCase();
+  const attrs = Object.entries(node.attrs)
+    .map(([k, v]) => `${k}=${JSON.stringify(v)}`)
+    .join(" ");
+  out.push(`[${tag}${attrs ? " " + attrs : ""}]`);
+  const isIndexWithExclusions =
+    node.name === "memory_index" && opts.excludedMemoryIds.size > 0;
+  const childForce = forceSubtree || matchesSelector(node, opts.selectSet);
+  if (VERBATIM_BODY.has(node.name) && node.body !== undefined) {
+    out.push(node.body);
+  } else if (node.children.length === 0 && node.body !== undefined) {
+    const body = isIndexWithExclusions
+      ? filterMemoryIndexBody(node.body, opts.excludedMemoryIds)
+      : node.body;
+    out.push(inlineToPlain(body));
+  } else if (isIndexWithExclusions) {
+    for (const child of node.children)
+      emitFilteredIndexChild(child, out, depth + 1, opts);
+  } else {
+    for (const child of node.children) {
+      if (shouldEmit(child, opts, childForce)) emit(child, out, depth + 1, opts, childForce);
+    }
+  }
+  out.push(`[/${tag}]`);
+  out.push("");
+}
+
+function shouldEmit(node: Node, opts: RenderCtx, forceSubtree: boolean): boolean {
+  if (matchesSelector(node, opts.excludeSet)) return false;
+  if (forceSubtree || opts.selectSet.size === 0) return true;
+  if (matchesSelector(node, opts.selectSet)) return true;
+  return hasSelectableDescendant(node, opts);
+}
+
+function hasSelectableDescendant(node: Node, opts: RenderCtx): boolean {
+  for (const child of childrenOf(node)) {
+    if (matchesSelector(child, opts.excludeSet)) continue;
+    if (matchesSelector(child, opts.selectSet)) return true;
+    if (hasSelectableDescendant(child, opts)) return true;
+  }
+  return false;
+}
+
+function matchesSelector(node: Node, selectors: Set<string>): boolean {
+  if (selectors.size === 0) return false;
+  if (selectors.has(node.type)) return true;
+  return node.type === "directive" && selectors.has(node.name.toLowerCase());
+}
+
+function childrenOf(node: Node): Node[] {
+  if (node.type === "document" || node.type === "section" || node.type === "directive") {
+    return node.children;
+  }
+  if (node.type === "list") return node.items;
+  return [];
+}
+
+const WIKILINK_RE = /\[\[([a-zA-Z_][\w\-./:]*)\]\]/g;
+
+function filterMemoryIndexBody(body: string, excluded: Set<string>): string {
+  if (excluded.size === 0) return body;
+  return body
+    .split("\n")
+    .filter((line) => {
+      const matches = [...line.matchAll(WIKILINK_RE)];
+      if (matches.length === 0) return true;
+      return !matches.every((m) => excluded.has(m[1]!));
+    })
+    .join("\n");
+}
+
+function emitFilteredIndexChild(
+  node: Node,
+  out: string[],
+  depth: number,
+  opts: RenderCtx,
+): void {
+  if (node.type === "list") {
+    const survivors = node.items.filter((item) => {
+      const matches = [...item.content.matchAll(WIKILINK_RE)];
+      if (matches.length === 0) return true;
+      return !matches.every((m) => opts.excludedMemoryIds.has(m[1]!));
+    });
+    if (survivors.length === 0) return;
+    for (const item of survivors)
+      out.push(`- ${inlineToPlain(item.content)}`);
+    out.push("");
+    return;
+  }
+  if (node.type === "paragraph") {
+    const matches = [...node.content.matchAll(WIKILINK_RE)];
+    const allExcluded =
+      matches.length > 0 && matches.every((m) => opts.excludedMemoryIds.has(m[1]!));
+    if (allExcluded) return;
+    out.push(inlineToPlain(node.content));
+    out.push("");
+    return;
+  }
+  emit(node, out, depth, opts, false);
+}
+
+function isStale(
+  node: DirectiveNode,
+  cfg: { now: Date; days: number },
+): boolean {
+  const ls = node.attrs.last_seen;
+  if (typeof ls !== "string" || !ls) return false;
+  const t = Date.parse(ls);
+  if (Number.isNaN(t)) return false;
+  const type = typeof node.attrs.type === "string" ? node.attrs.type : "";
+  const expired = node.attrs.expired === true;
+  if (!STALE_OPT_IN_TYPES.has(type) && !expired) return false;
+  return cfg.now.getTime() - t > cfg.days * 24 * 60 * 60 * 1000;
+}

package/src/renderer-noma.ts

@@ -0,0 +1,220 @@
+import yaml from "js-yaml";
+import { slugify } from "./parser.js";
+import type {
+  Attrs,
+  AttrValue,
+  CodeNode,
+  DirectiveNode,
+  DocumentNode,
+  FrontmatterNode,
+  ListNode,
+  Node,
+  ParagraphNode,
+  QuoteNode,
+  SectionNode,
+  TableNode,
+  ThematicBreakNode,
+} from "./ast.js";
+
+export interface NomaRenderOptions {
+  /** Drop internal meta keys (filename, pos) from frontmatter. Default: true. */
+  stripInternal?: boolean;
+}
+
+const INTERNAL_META_KEYS = new Set(["filename"]);
+
+/**
+ * AST → .noma source. Designed for roundtrip: `parse(renderNoma(doc))` should
+ * yield a structurally equal AST (modulo positions). Foundation for `noma patch`.
+ */
+export function renderNoma(doc: DocumentNode, options: NomaRenderOptions = {}): string {
+  const stripInternal = options.stripInternal !== false;
+  const out: string[] = [];
+  const ctx = buildContext(doc);
+
+  const hasFrontmatterNode = doc.children[0]?.type === "frontmatter";
+  if (!hasFrontmatterNode) {
+    const metaEntries = Object.entries(doc.meta).filter(
+      ([k]) => !stripInternal || !INTERNAL_META_KEYS.has(k),
+    );
+    if (metaEntries.length > 0) {
+      out.push("---");
+      out.push(yaml.dump(Object.fromEntries(metaEntries)).trimEnd());
+      out.push("---");
+      out.push("");
+    }
+  }
+
+  for (const child of doc.children) {
+    out.push(renderNode(child, 2, ctx));
+    out.push("");
+  }
+
+  return out.join("\n").replace(/\n{3,}/g, "\n\n").replace(/\n+$/, "\n");
+}
+
+interface RenderCtx {
+  /** Aliases that the parser/loader will re-derive on parse and so don't
+   *  need to be emitted on the heading. Filename slug + frontmatter list. */
+  regenAliases: Set<string>;
+}
+
+function buildContext(doc: DocumentNode): RenderCtx {
+  const regenAliases = new Set<string>();
+  if (Array.isArray(doc.meta.aliases)) {
+    for (const a of doc.meta.aliases) {
+      if (typeof a === "string" && a.trim()) regenAliases.add(a.trim());
+    }
+  }
+  if (typeof doc.meta.filename === "string") {
+    const base = doc.meta.filename.replace(/\\/g, "/").split("/").pop() ?? "";
+    const stem = base.replace(/\.noma$/i, "").replace(/^\d+[-_]/, "");
+    const slug = slugify(stem);
+    if (slug) regenAliases.add(slug);
+  }
+  return { regenAliases };
+}
+
+function renderNode(node: Node, colons: number, ctx: RenderCtx): string {
+  switch (node.type) {
+    case "document":
+      return node.children.map((c) => renderNode(c, colons, ctx)).join("\n\n");
+    case "section":
+      return renderSection(node, colons, ctx);
+    case "paragraph":
+      return renderParagraph(node);
+    case "code":
+      return renderCode(node);
+    case "list":
+      return renderList(node);
+    case "list_item":
+      return `- ${node.content}`;
+    case "quote":
+      return renderQuote(node);
+    case "thematic_break":
+      return renderThematicBreak(node);
+    case "table":
+      return renderTable(node);
+    case "directive":
+      return renderDirective(node, colons, ctx);
+    case "frontmatter":
+      return `---\n${node.raw}\n---`;
+    default: {
+      const _exhaustive: never = node;
+      void _exhaustive;
+      return "";
+    }
+  }
+}
+
+function renderSection(node: SectionNode, colons: number, ctx: RenderCtx): string {
+  const hashes = "#".repeat(Math.max(1, Math.min(6, node.level)));
+  const attrs = headingAttrs(node, ctx);
+  const head = attrs ? `${hashes} ${node.title} ${attrs}` : `${hashes} ${node.title}`;
+  if (node.children.length === 0) return head;
+  const inner = node.children.map((c) => renderNode(c, colons, ctx)).join("\n\n");
+  return `${head}\n\n${inner}`;
+}
+
+function headingAttrs(node: SectionNode, ctx: RenderCtx): string {
+  const explicitId =
+    node.id && node.id !== slugify(node.title) ? node.id : undefined;
+  // Drop aliases the parser/loader will re-derive (frontmatter list, filename
+  // slug). Anything else came from explicit `{aliases="..."}` in source and
+  // must be kept to round-trip.
+  const aliases = (node.aliases ?? []).filter((a) => !ctx.regenAliases.has(a));
+  const parts: string[] = [];
+  if (explicitId) parts.push(`id="${explicitId}"`);
+  if (aliases.length > 0) {
+    parts.push(`aliases="${aliases.join(",")}"`);
+  }
+  return parts.length > 0 ? `{${parts.join(" ")}}` : "";
+}
+
+function renderParagraph(node: ParagraphNode): string {
+  return node.content;
+}
+
+function renderCode(node: CodeNode): string {
+  return "```" + (node.lang ?? "") + "\n" + node.content + "\n```";
+}
+
+function renderList(node: ListNode): string {
+  if (node.ordered) {
+    return node.items.map((it, i) => `${i + 1}. ${it.content}`).join("\n");
+  }
+  return node.items.map((it) => `- ${it.content}`).join("\n");
+}
+
+function renderQuote(node: QuoteNode): string {
+  return node.content
+    .split("\n")
+    .map((l) => (l ? `> ${l}` : ">"))
+    .join("\n");
+}
+
+function renderThematicBreak(_node: ThematicBreakNode): string {
+  return "---";
+}
+
+function renderTable(node: TableNode): string {
+  const widths = node.header.map((h, i) =>
+    Math.max(h.length, ...node.rows.map((r) => (r[i] ?? "").length), 3),
+  );
+  const fmtRow = (cells: string[]) =>
+    "| " +
+    cells.map((c, i) => c.padEnd(widths[i] ?? c.length)).join(" | ") +
+    " |";
+  const sep =
+    "| " +
+    widths
+      .map((w, i) => {
+        const a = node.align[i];
+        if (a === "center") return ":" + "-".repeat(Math.max(3, w - 2)) + ":";
+        if (a === "right") return "-".repeat(Math.max(3, w - 1)) + ":";
+        if (a === "left") return ":" + "-".repeat(Math.max(3, w - 1));
+        return "-".repeat(Math.max(3, w));
+      })
+      .join(" | ") +
+    " |";
+  return [fmtRow(node.header), sep, ...node.rows.map(fmtRow)].join("\n");
+}
+
+function renderDirective(node: DirectiveNode, colons: number, ctx: RenderCtx): string {
+  const fence = ":".repeat(colons);
+  const attrs = serializeAttrs(node.attrs);
+  const open = `${fence}${node.name}${attrs ? attrs : ""}`;
+  const close = fence;
+
+  if (node.children.length === 0) {
+    if (node.body !== undefined && node.body !== "") {
+      return `${open}\n${node.body}\n${close}`;
+    }
+    return `${open}\n${close}`;
+  }
+
+  const childColons = colons + 1;
+  const inner = node.children.map((c) => renderNode(c, childColons, ctx)).join("\n\n");
+  return `${open}\n${inner}\n${close}`;
+}
+
+function serializeAttrs(attrs: Attrs): string {
+  const entries = Object.entries(attrs);
+  if (entries.length === 0) return "";
+  const parts = entries.map(([k, v]) => serializeAttr(k, v));
+  return `{${parts.join(" ")}}`;
+}
+
+function serializeAttr(key: string, value: AttrValue): string {
+  if (value === true) return key;
+  if (value === false) return `${key}=false`;
+  if (typeof value === "number") return `${key}=${value}`;
+  const s = String(value);
+  if (s.includes('"')) {
+    if (s.includes("'")) {
+      return `${key}="${s.replace(/"/g, '\\"')}"`;
+    }
+    return `${key}='${s}'`;
+  }
+  return `${key}="${s}"`;
+}