depa-codument 0.4.1

package/src/cli/engineering/registry.ts

@@ -0,0 +1,230 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { parseXnl, stringifyLineBlock, wordToString, XnlParseError } from 'xnl-core';
+import type { XnlNode, DataElementNode, XnlWord } from 'xnl-core';
+
+/**
+ * codument engineering registry adapter.
+ *
+ * The registry is a plain, host-git-versioned working tree of `.xnl` files under
+ * `codument/engineering/<plane>/<category>/...`. We load/save via xnl-core (parse +
+ * lineBlock formatter) and index nodes by their namespaced id. No parallel vcs repo
+ * is persisted; node-level 3-way merge (see merge.ts) uses xnl-vfs ephemerally.
+ */
+
+export interface EngineeringNodeRef {
+  /** Namespaced id, e.g. `backend.howto.orders.add_endpoint`. */
+  id: string;
+  node: DataElementNode;
+  /** Path relative to the engineering dir, e.g. `backend/howto/orders.xnl`. */
+  file: string;
+  /** `engineering://<plane>/<category>/<topic>/<name>` derived from the file path + id. */
+  uri: string;
+}
+
+export interface EngineeringRegistry {
+  dir: string;
+  /** relPath -> top-level nodes of that file (preserves order for save). */
+  files: Map<string, XnlNode[]>;
+  /** namespaced id -> ref. */
+  index: Map<string, EngineeringNodeRef>;
+}
+
+export function isDataElement(node: XnlNode | undefined): node is DataElementNode {
+  return Boolean(node && typeof node === 'object' && (node as DataElementNode).kind === 'DataElement');
+}
+
+/** Read a node's namespaced id from `#word` or `metadata.id`. */
+export function readNodeId(node: XnlNode): string | undefined {
+  if (!isDataElement(node)) return undefined;
+  const fromWord = wordToString((node as { id?: XnlWord }).id);
+  if (fromWord) return fromWord;
+  const metaId = node.metadata?.id;
+  if (typeof metaId === 'string') return metaId;
+  if (metaId && typeof metaId === 'object' && (metaId as XnlWord).kind === 'Word') {
+    return wordToString(metaId as XnlWord);
+  }
+  return undefined;
+}
+
+/** Last segment of a namespaced id (`a.b.name` -> `name`). */
+export function nodeName(id: string): string {
+  const parts = id.split('.');
+  return parts[parts.length - 1];
+}
+
+/**
+ * Namespace prefix of a namespaced id (`a.b.name` -> `a.b`, `name` -> ``).
+ * The namespace is the id minus its trailing `name` segment; a bare name yields
+ * the empty string. Used by id↔path alignment and engineering-uri derivation.
+ */
+export function idNamespace(id: string): string {
+  const parts = id.split('.');
+  return parts.slice(0, -1).join('.');
+}
+
+/** Build `engineering://<plane>/<category>/<topic>/<name>` from a file path and node id. */
+export function engineeringUri(relFile: string, id: string): string {
+  const segs = relFile.split(path.sep).filter(Boolean);
+  const plane = segs[0] ?? 'global';
+  const category = segs[1] ?? 'overview';
+  const topic = (segs[2] ?? 'index').replace(/\.xnl$/i, '');
+  return `engineering://${plane}/${category}/${topic}/${nodeName(id)}`;
+}
+
+function isHidden(name: string): boolean {
+  return name.startsWith('.');
+}
+
+function walkXnlFiles(dir: string, base: string, out: string[]): void {
+  if (!fs.existsSync(dir)) return;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (isHidden(entry.name)) continue; // skip .node-meta, .tmp, .xnl-vcs, etc.
+    const abs = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      walkXnlFiles(abs, base, out);
+    } else if (entry.isFile() && entry.name.endsWith('.xnl')) {
+      out.push(path.relative(base, abs));
+    }
+  }
+}
+
+/** Load the engineering registry from a working tree directory. */
+export function loadEngineeringRegistry(dir: string): EngineeringRegistry {
+  const files = new Map<string, XnlNode[]>();
+  const index = new Map<string, EngineeringNodeRef>();
+  const relFiles: string[] = [];
+  walkXnlFiles(dir, dir, relFiles);
+  relFiles.sort();
+
+  for (const relFile of relFiles) {
+    const content = fs.readFileSync(path.join(dir, relFile), 'utf-8');
+    const nodes = parseXnl(content, { textBlockStyle: true }).nodes;
+    files.set(relFile, nodes);
+    for (const node of nodes) {
+      const id = readNodeId(node);
+      if (!id || !isDataElement(node)) continue;
+      if (index.has(id)) {
+        throw new Error(
+          `Duplicate engineering node id '${id}' in '${relFile}' and '${index.get(id)!.file}'`,
+        );
+      }
+      index.set(id, { id, node, file: relFile, uri: engineeringUri(relFile, id) });
+    }
+  }
+
+  return { dir, files, index };
+}
+
+/**
+ * A non-fatal load issue collected by {@link loadEngineeringRegistrySafe}: an XNL
+ * parse error (`syntax`) or a cross-file duplicate id (`duplicate-id`). The
+ * validate engine maps these onto its own finding shape.
+ */
+export interface LoadIssue {
+  kind: 'syntax' | 'duplicate-id';
+  /** Path relative to the registry dir. */
+  file: string;
+  /** 1-based line, when known (parse errors carry one). */
+  line?: number;
+  message: string;
+  /** For duplicate ids: the other file that already defined the id. */
+  otherFile?: string;
+}
+
+export interface SafeRegistryResult {
+  registry: EngineeringRegistry;
+  issues: LoadIssue[];
+}
+
+/**
+ * Load the engineering registry without throwing. Files that fail XNL parsing are
+ * skipped (their error is collected as a `syntax` issue); duplicate ids are
+ * collected as `duplicate-id` issues (first definition wins in the index)
+ * instead of aborting the load. The non-safe {@link loadEngineeringRegistry}
+ * keeps its throwing contract for existing callers (lint, merge, archive).
+ */
+export function loadEngineeringRegistrySafe(dir: string): SafeRegistryResult {
+  const files = new Map<string, XnlNode[]>();
+  const index = new Map<string, EngineeringNodeRef>();
+  const issues: LoadIssue[] = [];
+  const relFiles: string[] = [];
+  walkXnlFiles(dir, dir, relFiles);
+  relFiles.sort();
+
+  for (const relFile of relFiles) {
+    const content = fs.readFileSync(path.join(dir, relFile), 'utf-8');
+    let nodes: XnlNode[];
+    try {
+      nodes = parseXnl(content, { textBlockStyle: true }).nodes;
+    } catch (err) {
+      if (err instanceof XnlParseError) {
+        issues.push({ kind: 'syntax', file: relFile, line: err.line, message: err.message });
+        continue;
+      }
+      throw err;
+    }
+    files.set(relFile, nodes);
+    for (const node of nodes) {
+      const id = readNodeId(node);
+      if (!id || !isDataElement(node)) continue;
+      const existing = index.get(id);
+      if (existing) {
+        issues.push({
+          kind: 'duplicate-id',
+          file: relFile,
+          otherFile: existing.file,
+          message: `Duplicate engineering node id '${id}' in '${relFile}' and '${existing.file}'`,
+        });
+        continue;
+      }
+      index.set(id, { id, node, file: relFile, uri: engineeringUri(relFile, id) });
+    }
+  }
+
+  return { registry: { dir, files, index }, issues };
+}
+
+export interface SerializeOptions {
+  /**
+   * Marker factory for markerless text blocks. Defaults to xnl-core's ULID factory,
+   * which preserves XNL text nodes' escape-free property (a markerless `?>` block
+   * gets a stable unique marker on first write, then round-trips unchanged).
+   * Tests may inject a deterministic factory; production MUST keep the ULID default.
+   */
+  textMarkerFactory?: () => string;
+  /**
+   * Block text style (default true for engineering files): render each text element's
+   * content on its own indented line(s) between `<tag ?m>` and `</?m>`, so registry
+   * files stay readable and diff-friendly. Paired with `parseXnl(.., {textBlockStyle})`.
+   */
+  textBlockStyle?: boolean;
+}
+
+/** Serialize top-level nodes of a file to XNL (lineBlock pretty, block text style). */
+export function serializeEngineeringFile(nodes: XnlNode[], opts: SerializeOptions = {}): string {
+  return nodes.map((n) => stringifyLineBlock(n, { textBlockStyle: true, ...opts })).join('\n\n') + '\n';
+}
+
+/** Write one engineering file back to the working tree. */
+export function saveEngineeringFile(
+  dir: string,
+  relFile: string,
+  nodes: XnlNode[],
+  opts: SerializeOptions = {},
+): void {
+  const abs = path.join(dir, relFile);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  fs.writeFileSync(abs, serializeEngineeringFile(nodes, opts), 'utf-8');
+}
+
+/** Save the whole registry back to its working tree. */
+export function saveEngineeringRegistry(
+  registry: EngineeringRegistry,
+  targetDir = registry.dir,
+  opts: SerializeOptions = {},
+): void {
+  for (const [relFile, nodes] of registry.files) {
+    saveEngineeringFile(targetDir, relFile, nodes, opts);
+  }
+}

package/src/cli/engineering/schema.ts

@@ -0,0 +1,126 @@
+import type { XnlNode, DataElementNode, TextElementNode } from 'xnl-core';
+import { isDataElement, readNodeId, type EngineeringRegistry } from './registry';
+
+/**
+ * Node schema validation for the engineering registry: kind vocabulary and
+ * minimal required representations per engineering knowledge kind.
+ * See std/spec/engineering-node-schema.md.
+ */
+
+export const ENGINEERING_KINDS = [
+  'overview',
+  'howto',
+  'rule',
+  'example',
+  'reference',
+  'troubleshooting',
+  'runbook',
+  'code-map',
+] as const;
+
+type Element = DataElementNode | TextElementNode;
+
+function isElement(node: XnlNode | undefined): node is Element {
+  return Boolean(
+    node && typeof node === 'object' &&
+      ((node as Element).kind === 'DataElement' || (node as Element).kind === 'TextElement'),
+  );
+}
+
+function bodyChildren(node: DataElementNode): Element[] {
+  return (node.body ?? []).filter(isElement);
+}
+
+function childTags(node: DataElementNode): Set<string> {
+  return new Set(bodyChildren(node).map((c) => c.tag));
+}
+
+function metaString(node: DataElementNode, key: string): string | undefined {
+  const v = node.metadata?.[key];
+  return typeof v === 'string' ? v : undefined;
+}
+
+export function nodeKind(node: DataElementNode): string | undefined {
+  return metaString(node, 'kind');
+}
+
+export function validateEngineeringNode(node: XnlNode): string[] {
+  const errors: string[] = [];
+  if (!isDataElement(node)) return errors;
+  const id = readNodeId(node);
+  const where = id ? `#${id}` : `<${node.tag}>`;
+
+  if (!id) {
+    errors.push(`${where}: node has no id (use #<plane>.<category>.<topic>.<name>)`);
+  }
+
+  const kind = nodeKind(node);
+  if (!kind) {
+    errors.push(`${where}: missing 'kind'`);
+    return errors;
+  }
+
+  const isShell = kind.includes(':');
+  if (!isShell && !(ENGINEERING_KINDS as readonly string[]).includes(kind)) {
+    errors.push(`${where}: unknown engineering kind '${kind}'`);
+  }
+
+  const tags = childTags(node);
+  const req = (cond: boolean, msg: string) => {
+    if (!cond) errors.push(`${where} (kind=${kind}): ${msg}`);
+  };
+
+  switch (kind) {
+    case 'overview':
+      req(tags.has('desc'), 'overview requires a <desc> block');
+      req(tags.has('mental-model'), 'overview requires a <mental-model> block');
+      break;
+    case 'howto':
+      req(tags.has('when-to-use'), 'howto requires a <when-to-use> block');
+      req(tags.has('steps'), 'howto requires a <steps> block');
+      req(tags.has('verification'), 'howto requires a <verification> block');
+      break;
+    case 'rule':
+      req(tags.has('rule'), 'rule requires a <rule> block');
+      req(tags.has('rationale'), 'rule requires a <rationale> block');
+      req(tags.has('enforcement'), 'rule requires an <enforcement> block');
+      break;
+    case 'example':
+      req(tags.has('scenario'), 'example requires a <scenario> block');
+      req(tags.has('walkthrough'), 'example requires a <walkthrough> block');
+      break;
+    case 'reference':
+      req(tags.has('scope'), 'reference requires a <scope> block');
+      req(tags.has('source-of-truth'), 'reference requires a <source-of-truth> block');
+      req(tags.has('update-procedure'), 'reference requires an <update-procedure> block');
+      break;
+    case 'troubleshooting':
+      req(tags.has('symptoms'), 'troubleshooting requires a <symptoms> block');
+      req(tags.has('diagnosis'), 'troubleshooting requires a <diagnosis> block');
+      req(tags.has('fix'), 'troubleshooting requires a <fix> block');
+      break;
+    case 'runbook':
+      req(tags.has('preconditions'), 'runbook requires a <preconditions> block');
+      req(tags.has('steps'), 'runbook requires a <steps> block');
+      req(tags.has('verification'), 'runbook requires a <verification> block');
+      req(tags.has('rollback'), 'runbook requires a <rollback> block');
+      break;
+    case 'code-map':
+      req(tags.has('scope'), 'code-map requires a <scope> block');
+      req(tags.has('paths'), 'code-map requires a <paths> block');
+      req(tags.has('update-procedure'), 'code-map requires an <update-procedure> block');
+      break;
+    default:
+      break;
+  }
+
+  return errors;
+}
+
+export function validateEngineeringRegistry(registry: EngineeringRegistry): string[] {
+  const errors: string[] = [];
+  for (const nodes of registry.files.values()) {
+    for (const node of nodes) errors.push(...validateEngineeringNode(node));
+  }
+  return errors;
+}

package/src/cli/engineering/validate.ts

@@ -0,0 +1,286 @@
+import * as path from 'path';
+import type { DataElementNode, AttributeMap } from 'xnl-core';
+import {
+  loadEngineeringRegistrySafe,
+  isDataElement,
+  readNodeId,
+  nodeName,
+} from './registry';
+import { validateEngineeringNode } from './schema';
+
+/**
+ * Engineering validation engine. Checks every engineering `.xnl` file across
+ * XNL syntax, node-schema semantics, hierarchy, and URI reference shape.
+ */
+
+export type ValidateLayer = 'syntax' | 'schema' | 'hierarchy';
+export type ValidateSeverity = 'error' | 'warning';
+
+export interface ValidateFinding {
+  file: string;
+  line?: number;
+  layer: ValidateLayer;
+  severity: ValidateSeverity;
+  message: string;
+}
+
+export type ValidateMode = 'registry' | 'deltas';
+
+export interface ValidateOptions {
+  /**
+   * registry: `<plane>/<category>/<topic>.xnl` or `<plane>/<category>/<topic>/index.xnl`.
+   * deltas: same layout under a track's `engineering_deltas/`.
+   */
+  mode?: ValidateMode;
+}
+
+const KNOWN_PLANES = new Set([
+  'global',
+  'backend',
+  'surface',
+  'runtime',
+  'storage',
+  'pipelines',
+  'agents',
+  'operations',
+  'cli',
+]);
+
+const KNOWN_CATEGORIES = new Set([
+  'overview',
+  'howto',
+  'rules',
+  'examples',
+  'reference',
+  'troubleshooting',
+  'runbooks',
+  'code-map',
+]);
+
+const ENGINEERING_SCHEME = 'engineering://';
+const MODELING_SCHEME = 'modeling://';
+const BEHAVIOR_SCHEME = 'behavior://';
+const DECISION_SCHEME = 'decision://';
+
+interface PathLoc {
+  plane: string;
+  category: string;
+  topic: string;
+}
+
+function pathLoc(relFile: string): PathLoc {
+  const segs = relFile.split(path.sep).filter(Boolean);
+  const plane = segs[0] ?? '';
+  const category = segs[1] ?? '';
+  let topic = segs[2] ?? '';
+  if (topic === 'index.xnl' && segs.length >= 3) topic = segs[1] ?? '';
+  topic = topic.replace(/\.xnl$/i, '');
+  return { plane, category, topic };
+}
+
+function collectRefs(meta: AttributeMap | undefined): string[] {
+  const out: string[] = [];
+  const visit = (v: unknown): void => {
+    if (typeof v === 'string') {
+      if (
+        v.startsWith(ENGINEERING_SCHEME) ||
+        v.startsWith(MODELING_SCHEME) ||
+        v.startsWith(BEHAVIOR_SCHEME) ||
+        v.startsWith(DECISION_SCHEME)
+      ) {
+        out.push(v);
+      }
+      return;
+    }
+    if (Array.isArray(v)) {
+      for (const item of v) visit(item);
+      return;
+    }
+    if (v && typeof v === 'object') {
+      for (const item of Object.values(v as Record<string, unknown>)) visit(item);
+    }
+  };
+  if (meta) for (const v of Object.values(meta)) visit(v);
+  return out;
+}
+
+function checkIdPathAlignment(node: DataElementNode, relFile: string, loc: PathLoc): ValidateFinding[] {
+  const findings: ValidateFinding[] = [];
+  const id = readNodeId(node);
+  if (!id) return findings;
+  const parts = id.split('.');
+
+  if (parts.length < 4) {
+    findings.push({
+      file: relFile,
+      layer: 'hierarchy',
+      severity: 'error',
+      message: `#${id}: id must be '#<plane>.<category>.<topic>.<name>'`,
+    });
+    return findings;
+  }
+
+  const [idPlane, idCategory, idTopic] = parts;
+  if (idPlane !== loc.plane) {
+    findings.push({
+      file: relFile,
+      layer: 'hierarchy',
+      severity: 'error',
+      message: `#${id}: id plane '${idPlane}' does not match path plane '${loc.plane}' (${relFile})`,
+    });
+  }
+  if (idCategory !== loc.category) {
+    findings.push({
+      file: relFile,
+      layer: 'hierarchy',
+      severity: 'error',
+      message: `#${id}: id category '${idCategory}' does not match path category '${loc.category}' (${relFile})`,
+    });
+  }
+  if (idTopic !== loc.topic) {
+    findings.push({
+      file: relFile,
+      layer: 'hierarchy',
+      severity: 'error',
+      message: `#${id}: id topic '${idTopic}' does not match path topic '${loc.topic}' (${relFile})`,
+    });
+  }
+
+  return findings;
+}
+
+function checkReferences(node: DataElementNode, relFile: string, knownEngineeringUris: Set<string>): ValidateFinding[] {
+  const findings: ValidateFinding[] = [];
+  const id = readNodeId(node);
+  const where = id ? `#${id}` : `<${node.tag}>`;
+
+  for (const ref of collectRefs(node.metadata)) {
+    if (ref.startsWith(ENGINEERING_SCHEME)) {
+      if (!knownEngineeringUris.has(ref)) {
+        findings.push({
+          file: relFile,
+          layer: 'hierarchy',
+          severity: 'error',
+          message: `${where}: dangling engineering reference '${ref}'`,
+        });
+      }
+      continue;
+    }
+
+    const [scheme, minParts] = ref.startsWith(MODELING_SCHEME)
+      ? [MODELING_SCHEME, 3]
+      : ref.startsWith(BEHAVIOR_SCHEME)
+        ? [BEHAVIOR_SCHEME, 2]
+        : ref.startsWith(DECISION_SCHEME)
+          ? [DECISION_SCHEME, 1]
+          : ['', 0];
+    if (!scheme) continue;
+    const rest = ref.slice(scheme.length);
+    if (rest.split('/').filter(Boolean).length < minParts) {
+      findings.push({
+        file: relFile,
+        layer: 'hierarchy',
+        severity: 'error',
+        message: `${where}: malformed reference '${ref}'`,
+      });
+    }
+  }
+
+  return findings;
+}
+
+function checkLayout(relFiles: Iterable<string>): ValidateFinding[] {
+  const findings: ValidateFinding[] = [];
+  const planes = new Set<string>();
+  for (const relFile of relFiles) {
+    const loc = pathLoc(relFile);
+    if (loc.plane) planes.add(loc.plane);
+    if (!loc.plane || !loc.category || !loc.topic) {
+      findings.push({
+        file: relFile,
+        layer: 'hierarchy',
+        severity: 'error',
+        message: `engineering file path must be '<plane>/<category>/<topic>.xnl' or '<plane>/<category>/<topic>/index.xnl'`,
+      });
+      continue;
+    }
+    if (!KNOWN_CATEGORIES.has(loc.category)) {
+      findings.push({
+        file: relFile,
+        layer: 'hierarchy',
+        severity: 'warning',
+        message: `unknown engineering category '${loc.category}' (custom categories are allowed; verify it is intentional)`,
+      });
+    }
+  }
+
+  if (!planes.has('global')) {
+    findings.push({
+      file: '.',
+      layer: 'hierarchy',
+      severity: 'warning',
+      message: `engineering registry has no 'global' plane; this is allowed but cross-plane knowledge usually belongs there`,
+    });
+  }
+
+  for (const plane of planes) {
+    if (plane && !KNOWN_PLANES.has(plane)) {
+      findings.push({
+        file: '.',
+        layer: 'hierarchy',
+        severity: 'warning',
+        message: `unknown engineering plane '${plane}' (custom planes are allowed; verify it is intentional)`,
+      });
+    }
+  }
+
+  return findings;
+}
+
+export function validateEngineeringTree(dir: string, opts: ValidateOptions = {}): ValidateFinding[] {
+  void opts;
+  const findings: ValidateFinding[] = [];
+  const { registry, issues } = loadEngineeringRegistrySafe(dir);
+
+  for (const issue of issues) {
+    if (issue.kind === 'syntax') {
+      findings.push({
+        file: issue.file,
+        line: issue.line,
+        layer: 'syntax',
+        severity: 'error',
+        message: issue.message,
+      });
+    } else {
+      findings.push({ file: issue.file, layer: 'hierarchy', severity: 'error', message: issue.message });
+    }
+  }
+
+  const knownEngineeringUris = new Set<string>();
+  for (const [relFile, nodes] of registry.files) {
+    const loc = pathLoc(relFile);
+    for (const node of nodes) {
+      const id = readNodeId(node);
+      if (!id || !isDataElement(node)) continue;
+      knownEngineeringUris.add(`${ENGINEERING_SCHEME}${loc.plane}/${loc.category}/${loc.topic}/${nodeName(id)}`);
+    }
+  }
+
+  for (const [relFile, nodes] of registry.files) {
+    const loc = pathLoc(relFile);
+    for (const node of nodes) {
+      if (!isDataElement(node)) continue;
+      for (const msg of validateEngineeringNode(node)) {
+        findings.push({ file: relFile, layer: 'schema', severity: 'error', message: msg });
+      }
+      findings.push(...checkIdPathAlignment(node, relFile, loc));
+      findings.push(...checkReferences(node, relFile, knownEngineeringUris));
+    }
+  }
+
+  const layoutFiles = new Set<string>(registry.files.keys());
+  for (const issue of issues) if (issue.kind === 'syntax') layoutFiles.add(issue.file);
+  findings.push(...checkLayout(layoutFiles));
+
+  return findings;
+}