mirror-mind 0.0.1 → 0.32.1

package/ts/src/db/schemaState.ts

@@ -0,0 +1,91 @@
+// Schema-state guard for the database seam (CR018, RS003 database audit).
+//
+// CV22 freezes the SQLite schema as a compatibility contract. This guard is
+// its enforcement point: before serving a command, the front door asserts that
+// the database's `_migrations` bookkeeping matches the migration set this core
+// was built against, converting silent schema drift into an explicit,
+// actionable error.
+//
+// `KNOWN_MIGRATION_IDS` is the authoritative migration set. It began as the
+// Python list plus whatever TypeScript authored on its own (CV22.DS6.US2 made
+// TS ⊇ Python), and `tests/unit/test_ts_schema_contract.py` held the other
+// side of that contract so a new Python migration could not land without
+// extending this list.
+//
+// CV22.DS10.TS5 collapses the two-custodian model: TypeScript carries all
+// seventeen, Python carries sixteen and is being deleted, so there is no second
+// list to stay a prefix of. The remedies below name `runtime migrate` rather
+// than an interpreter, for the same reason.
+
+import { PROGRAM } from "#util/program.ts";
+import type { Database } from "./database.ts";
+
+/** Raised when the database's migration state does not match this TS build. */
+export class SchemaStateError extends Error {}
+
+/**
+ * Every migration this core knows, in order. Until CV22.DS10.TS5 the list was
+ * the Python engine's plus the TypeScript-authored tail; since then the
+ * TypeScript engine is the only custodian, and the list is simply its own.
+ */
+export const KNOWN_MIGRATION_IDS: readonly string[] = [
+  "001_project_to_travessia",
+  "002_create_attachments",
+  "003_create_tasks",
+  "004_tasks_temporal_fields",
+  "005_travessia_to_journey",
+  "006_create_llm_calls",
+  "007_create_identity_descriptors",
+  "008_create_memories_fts",
+  "009_memories_reinforcement_columns",
+  "010_create_consolidations",
+  "011_create_operation_runs",
+  "012_create_operation_run_events",
+  "013_create_exploratory_stories",
+  "014_create_identity_integrations",
+  "015_create_builder_workbench",
+  "016_builder_workbench_display_codes",
+  // TS-authored, no Python counterpart (CV22.DS6.US2 — first TS ⊋ Python migration).
+  "017_journey_parent_column",
+];
+
+/**
+ * Assert that the database's migration ledger matches this build. Three
+ * failure modes, each named in the error: `_migrations` absent (not a
+ * bootstrapped Mirror database), a known id missing (database older than this
+ * core — `runtime migrate` brings it forward), an unknown id present (database
+ * migrated by a newer core than the running one — `runtime update` brings the
+ * core forward, for a clone and a package alike).
+ */
+export function assertSchemaState(db: Database): void {
+  let rows: { id: string }[];
+  try {
+    rows = db.prepare("SELECT id FROM _migrations").all() as { id: string }[];
+  } catch {
+    throw new SchemaStateError(
+      "database has no _migrations table — not a bootstrapped Mirror database. " +
+        "Point Mirror at the right file, or create one: any write command bootstraps a missing database.",
+    );
+  }
+  const applied = new Set(rows.map((row) => row.id));
+  // Every known migration is required now. Until CV22.DS10.TS5 this filtered
+  // out the TS-authored tail, because Python could not apply those and the read
+  // path did not need them -- a split that only made sense while two engines
+  // shared custody. TypeScript carries all seventeen and is the only engine
+  // left to apply them, so "pending" means pending.
+  const missingRequired = KNOWN_MIGRATION_IDS.filter((id) => !applied.has(id));
+  if (missingRequired.length > 0) {
+    throw new SchemaStateError(
+      `database schema is older than this TS core (pending migrations: ${missingRequired.join(", ")}). ` +
+        "Run `runtime migrate` to bring it forward, then retry.",
+    );
+  }
+  const known = new Set(KNOWN_MIGRATION_IDS);
+  const unknown = rows.map((row) => row.id).filter((id) => !known.has(id));
+  if (unknown.length > 0) {
+    throw new SchemaStateError(
+      `database schema is newer than this TS core (unknown migrations: ${unknown.join(", ")}). ` +
+        `Update this Mirror installation so its core matches the database; run: ${PROGRAM} runtime update`,
+    );
+  }
+}

package/ts/src/db/schemaTsDivergence.ts

@@ -0,0 +1,114 @@
+// TS ⊇ Python schema divergence — CV22.DS6.US2.
+//
+// Through DS6.TS1 the TS-created schema had to be structurally IDENTICAL to the
+// committed Python snapshot. US2 is the first story where TS owns schema custody
+// enough to author schema Python lacks: the `identity.parent_journey` column and
+// its partial index. This module is the single, enumerated source of truth for
+// exactly what TS adds beyond Python, so every guard (schema.test.ts,
+// migrations.test.ts regression, the Navigator parity script) compares a
+// TS-created inventory to the Python snapshot THROUGH it — an unexpected drift
+// (anything not listed here) still fails loudly, in either direction.
+
+import { isDeepStrictEqual } from "node:util";
+import type { ColumnInventory, SchemaInventory } from "./schemaInventory.ts";
+
+/** Columns TS adds to a Python-snapshot table, appended after its columns. */
+const TS_ONLY_COLUMNS: Record<string, ColumnInventory[]> = {
+  identity: [{ name: "parent_journey", type: "TEXT", notnull: 0, dflt_value: null, pk: 0 }],
+};
+
+/** Indexes TS creates that the Python snapshot does not have. */
+const TS_ONLY_INDEXES: Record<string, { table: string; unique: number; columns: string[] }> = {
+  idx_identity_parent_journey: { table: "identity", unique: 0, columns: ["parent_journey"] },
+};
+
+function sortedKeys(record: Record<string, unknown>): string[] {
+  return Object.keys(record).sort();
+}
+
+/**
+ * Return the list of structural problems in `actual` (a TS-created inventory)
+ * measured against the Python `snapshot` PLUS exactly the enumerated TS-only
+ * additions. Empty list == pass. Pure, so tests and the Navigator script share
+ * one definition of the intended divergence.
+ */
+export function diffTsInventoryAgainstSnapshot(
+  actual: SchemaInventory,
+  snapshot: SchemaInventory,
+): string[] {
+  const problems: string[] = [];
+
+  // Tables: no new/removed tables; a listed table may gain listed columns.
+  if (!isDeepStrictEqual(sortedKeys(actual.tables), sortedKeys(snapshot.tables))) {
+    problems.push("tables: name set differs from the Python snapshot");
+  }
+  for (const name of Object.keys(snapshot.tables)) {
+    const actualTable = actual.tables[name];
+    if (!actualTable) continue;
+    const additions = TS_ONLY_COLUMNS[name] ?? [];
+    const expectedColumns = [...snapshot.tables[name].columns, ...additions];
+    if (!isDeepStrictEqual(actualTable.columns, expectedColumns)) {
+      problems.push(`tables[${name}].columns differ from snapshot + enumerated TS-only additions`);
+    }
+    if (!isDeepStrictEqual(actualTable.foreign_keys, snapshot.tables[name].foreign_keys)) {
+      problems.push(`tables[${name}].foreign_keys differ from snapshot`);
+    }
+    if (additions.length > 0) {
+      // The raw CREATE TABLE text diverges by exactly these columns; assert each
+      // added column name is present rather than reproduce SQLite's stored text.
+      for (const column of additions) {
+        if (!(actualTable.sql ?? "").includes(column.name)) {
+          problems.push(`tables[${name}].sql is missing TS-only column ${column.name}`);
+        }
+      }
+    } else if (!isDeepStrictEqual(actualTable.sql, snapshot.tables[name].sql)) {
+      problems.push(`tables[${name}].sql differs from snapshot`);
+    }
+  }
+
+  // Indexes: snapshot indexes exact; plus the enumerated TS-only ones whose
+  // owning table is present (a TS-only index on `identity` only exists when the
+  // schema/fixture actually has an `identity` table for migration 017 to alter).
+  const applicableTsIndexes = Object.entries(TS_ONLY_INDEXES).filter(
+    ([, spec]) => spec.table in snapshot.tables,
+  );
+  const expectedIndexNames = [
+    ...Object.keys(snapshot.indexes),
+    ...applicableTsIndexes.map(([name]) => name),
+  ].sort();
+  if (!isDeepStrictEqual(sortedKeys(actual.indexes), expectedIndexNames)) {
+    problems.push("indexes: name set differs from snapshot + enumerated TS-only indexes");
+  }
+  for (const name of Object.keys(snapshot.indexes)) {
+    if (actual.indexes[name] && !isDeepStrictEqual(actual.indexes[name], snapshot.indexes[name])) {
+      problems.push(`indexes[${name}] differs from snapshot`);
+    }
+  }
+  for (const [name, spec] of applicableTsIndexes) {
+    const index = actual.indexes[name];
+    if (!index) {
+      problems.push(`missing enumerated TS-only index ${name}`);
+    } else if (
+      index.table !== spec.table ||
+      index.unique !== spec.unique ||
+      !isDeepStrictEqual(index.columns, spec.columns)
+    ) {
+      problems.push(`TS-only index ${name} has an unexpected shape`);
+    }
+  }
+
+  // Triggers: exact, both directions.
+  if (!isDeepStrictEqual(sortedKeys(actual.triggers), sortedKeys(snapshot.triggers))) {
+    problems.push("triggers: name set differs from the Python snapshot");
+  }
+  for (const name of Object.keys(snapshot.triggers)) {
+    if (
+      actual.triggers[name] &&
+      !isDeepStrictEqual(actual.triggers[name], snapshot.triggers[name])
+    ) {
+      problems.push(`triggers[${name}] differs from snapshot`);
+    }
+  }
+
+  return problems;
+}

package/ts/src/descriptor/descriptorRead.ts

@@ -0,0 +1,49 @@
+// Descriptor read primitives — the port of Store.get_descriptors_by_layer and
+// the `get_all_identity`-driven "all layers" iteration in
+// memory.cli.descriptor._cmd_list (`generate`, which calls the LLM, stays on
+// Python; see routing.ts).
+
+import type { Database } from "#db/database.ts";
+import { requireString } from "#db/rowDecode.ts";
+
+export interface DescriptorRow {
+  layer: string;
+  key: string;
+  descriptor: string;
+}
+
+function toDescriptorRow(row: Record<string, unknown>): DescriptorRow {
+  return {
+    layer: requireString(row, "layer"),
+    key: requireString(row, "key"),
+    descriptor: requireString(row, "descriptor"),
+  };
+}
+
+/** Port of `get_descriptors_by_layer`: descriptors for one layer, ordered by key. */
+export function descriptorsByLayer(db: Database, layer: string): DescriptorRow[] {
+  return db
+    .prepare("SELECT layer, key, descriptor FROM identity_descriptors WHERE layer = ? ORDER BY key")
+    .all(layer)
+    .map(toDescriptorRow);
+}
+
+/**
+ * Port of the "all layers" default path in `_cmd_list`: Python iterates
+ * `get_all_identity()` (ordered by layer, key) and keeps only the identity rows
+ * that have a matching descriptor row — an inner join driven by `identity`'s
+ * order, not a direct scan of `identity_descriptors`. An orphaned descriptor row
+ * with no identity row is silently excluded, exactly as the Python loop skips it.
+ * Pushed down to SQL per the database-seam philosophy (listing.ts precedent).
+ */
+export function allDescriptors(db: Database): DescriptorRow[] {
+  return db
+    .prepare(
+      `SELECT i.layer as layer, i.key as key, d.descriptor as descriptor
+       FROM identity i
+       JOIN identity_descriptors d ON d.layer = i.layer AND d.key = i.key
+       ORDER BY i.layer, i.key`,
+    )
+    .all()
+    .map(toDescriptorRow);
+}

package/ts/src/docs/docsLint.ts

@@ -0,0 +1,265 @@
+// Docs link/anchor checker — the TypeScript port of `src/memory/docs_lint.py`
+// (CV22.DS10.TS5, slice B).
+//
+// The Zero Python gate assigns `scripts/check_doc_links.py` to this story. The
+// script is a 57-line wrapper; the subject is this module's Python original
+// (221 lines) plus the duplicate-heading guard, which TypeScript already owns
+// through `findDuplicateRoadmapHeadings` (DS7.US8). So this port is the
+// link/anchor half, and the roadmap half is a thin adapter over existing code.
+//
+// Every behavior here is pinned by `ts/test/scripts/docsLint.test.ts`, which
+// carries all 26 cases of `tests/unit/memory/test_docs_lint.py`. That test is
+// the reason this file exists in the shape it does: each case records a
+// failure mode the checker actually caught, or once missed, during the
+// repo-wide docs audit. The Python self-test dies with its suite; the
+// assertions do not.
+
+import {
+  type Dirent,
+  existsSync,
+  readdirSync,
+  readFileSync,
+  realpathSync,
+  statSync,
+} from "node:fs";
+import { dirname, join, normalize, relative, resolve, sep } from "node:path";
+
+import { roadmapPaths } from "#builder/pullCandidates.ts";
+import { findDuplicateRoadmapHeadings } from "#builder/storyPaths.ts";
+
+const LINK_RE = /\[[^\]]*\]\(([^)]+)\)/g;
+const HEADING_RE = /^#{1,6}[ \t]+(.+?)[ \t]*$/gm;
+const FENCE_RE = /```[\s\S]*?```/g;
+const SLUG_STRIP_RE = /[^a-z0-9 _-]/g;
+
+const EXCLUDED_DIR_NAMES = new Set(["node_modules", ".venv", "__pycache__", ".git"]);
+
+// Parity fixture trees (`ts/test/fixtures/**`) are test INPUT, not
+// documentation. They carry authored-Markdown edge cases an oracle is graded
+// on -- dangling links, duplicate heading codes, malformed tables -- so
+// linting them as docs would force the fixtures to stop being fixtures.
+// Narrow on purpose: a directory named `fixtures` is skipped only beneath a
+// `test`/`tests` directory, so a real `docs/**/fixtures/` tree is still
+// checked.
+const TEST_DIR_NAMES = new Set(["test", "tests"]);
+const FIXTURE_DIR_NAME = "fixtures";
+
+// `roadmap/templates/*.md` link to a bare `index.md` breadcrumb meant to
+// resolve once the template is copied into a real story folder; `templates/`
+// itself has no `index.md`, by design. Not a broken link.
+const TEMPLATE_DIR_MARKER = "/roadmap/templates/";
+
+export interface BrokenLink {
+  readonly sourceFile: string;
+  readonly line: number;
+  readonly target: string;
+  readonly reason: string;
+}
+
+export interface DuplicateRoadmapHeading {
+  readonly code: string;
+  readonly paths: readonly string[];
+}
+
+/**
+ * Reproduce GitHub's heading-to-anchor algorithm.
+ *
+ * Lowercase, strip everything except letters/digits/spaces/underscores/
+ * hyphens (existing hyphens and underscores are PRESERVED -- `MEMORY_ENV`
+ * slugs to `memory_env`, not `memory-env`), then spaces become hyphens.
+ * Repeated hyphens are never collapsed, matching GitHub: an em dash is
+ * stripped outright and leaves its surrounding spaces behind, which is why
+ * `D1 — Local-first` slugs to `d1--local-first`.
+ */
+export function slugify(heading: string): string {
+  return heading.toLowerCase().replace(SLUG_STRIP_RE, "").replace(/ /g, "-");
+}
+
+/**
+ * Blank out fenced code blocks so link-like text inside ``` ``` is never
+ * scanned, preserving the line count exactly -- a problem reported after a
+ * fence must still point at its real source line.
+ */
+export function stripFencedCode(text: string): string {
+  return text.replace(FENCE_RE, (match) => "\n".repeat((match.match(/\n/g) ?? []).length));
+}
+
+/** Every Markdown heading's text, in document order. */
+export function extractHeadings(text: string): string[] {
+  const headings: string[] = [];
+  for (const match of text.matchAll(HEADING_RE)) headings.push(match[1] as string);
+  return headings;
+}
+
+/**
+ * Every anchor id reachable in this document. Duplicate heading text is
+ * suffixed `-1`, `-2`, ... in encounter order, as GitHub disambiguates.
+ */
+export function computeAnchors(text: string): Set<string> {
+  const anchors = new Set<string>();
+  const seen = new Map<string, number>();
+  for (const heading of extractHeadings(text)) {
+    const base = slugify(heading);
+    const count = (seen.get(base) ?? -1) + 1;
+    seen.set(base, count);
+    anchors.add(count === 0 ? base : `${base}-${count}`);
+  }
+  return anchors;
+}
+
+function toPosix(path: string): string {
+  return sep === "/" ? path : path.split(sep).join("/");
+}
+
+function isTemplatePlaceholder(sourceFile: string, targetPath: string): boolean {
+  return toPosix(sourceFile).includes(TEMPLATE_DIR_MARKER) && targetPath === "index.md";
+}
+
+/** True for a `fixtures` directory nested under a `test`/`tests` one. */
+function isTestFixturePath(path: string): boolean {
+  const parts = toPosix(path).split("/");
+  return parts.some(
+    (part, index) =>
+      part === FIXTURE_DIR_NAME && parts.slice(0, index).some((p) => TEST_DIR_NAMES.has(p)),
+  );
+}
+
+function lineOf(text: string, offset: number): number {
+  let line = 1;
+  for (let i = 0; i < offset; i += 1) if (text[i] === "\n") line += 1;
+  return line;
+}
+
+/** Every markdown file in the tree, in deterministic (sorted) order. */
+function iterMarkdownFiles(repoRoot: string): string[] {
+  const found: string[] = [];
+  const walk = (dir: string): void => {
+    let entries: Dirent[];
+    try {
+      entries = readdirSync(dir, { withFileTypes: true, encoding: "utf8" });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (EXCLUDED_DIR_NAMES.has(entry.name)) continue;
+        walk(full);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        if (!isTestFixturePath(full)) found.push(full);
+      }
+    }
+  };
+  walk(repoRoot);
+  return found.sort();
+}
+
+/** Check every relative/anchor link in one markdown file. */
+export function checkFile(sourceFile: string, repoRoot: string): BrokenLink[] {
+  const raw = readFileSync(sourceFile, "utf8");
+  const scanned = stripFencedCode(raw);
+  const relSource = toPosix(relative(repoRoot, sourceFile));
+  const problems: BrokenLink[] = [];
+  let ownAnchors: Set<string> | null = null;
+
+  for (const match of scanned.matchAll(LINK_RE)) {
+    const link = (match[1] as string).trim();
+    if (/^(https?:\/\/|mailto:)/.test(link)) continue;
+    const line = lineOf(scanned, match.index ?? 0);
+
+    if (link.startsWith("#")) {
+      ownAnchors ??= computeAnchors(raw);
+      if (!ownAnchors.has(link.slice(1))) {
+        problems.push({
+          sourceFile: relSource,
+          line,
+          target: link,
+          reason: "anchor not found in this file",
+        });
+      }
+      continue;
+    }
+
+    const hashIndex = link.indexOf("#");
+    const pathPart = hashIndex === -1 ? link : link.slice(0, hashIndex);
+    const anchorPart = hashIndex === -1 ? "" : link.slice(hashIndex + 1);
+    if (isTemplatePlaceholder(sourceFile, pathPart)) continue;
+
+    const target = normalize(join(dirname(sourceFile), pathPart));
+    if (!existsSync(target)) {
+      problems.push({
+        sourceFile: relSource,
+        line,
+        target: link,
+        reason: "target file does not exist",
+      });
+      continue;
+    }
+
+    if (anchorPart && target.endsWith(".md")) {
+      if (!computeAnchors(readFileSync(target, "utf8")).has(anchorPart)) {
+        problems.push({
+          sourceFile: relSource,
+          line,
+          target: link,
+          reason: "anchor not found in target file",
+        });
+      }
+    }
+  }
+
+  return problems;
+}
+
+/** Check every markdown file in the repository. Deterministic order. */
+export function checkRepo(repoRoot: string): BrokenLink[] {
+  const problems: BrokenLink[] = [];
+  for (const file of iterMarkdownFiles(repoRoot)) problems.push(...checkFile(file, repoRoot));
+  return problems;
+}
+
+/**
+ * Two or more roadmap `index.md` files whose heading claims the same code.
+ *
+ * Sorted by code, then by path, for stable CI output.
+ *
+ * Two resolution hazards, both real, both pinned by the self-test:
+ *
+ * 1. **The helper's contract is not Python's.** `find_duplicate_roadmap_headings`
+ *    returns ABSOLUTE directories; the TypeScript `findDuplicateRoadmapHeadings`
+ *    returns paths RELATIVE TO THE ROADMAP ROOT (`cv2-ds1-a`), because that is
+ *    what `resolveStoryDirectory` needed when DS7.US8 ported it. Same name,
+ *    different contract -- so this adapter joins them onto the roadmap root
+ *    rather than treating them as absolute. Assuming parity of a helper whose
+ *    name matched would have produced `../../../..`-prefixed garbage in CI
+ *    output.
+ * 2. **Both ends must be realpath'd or neither.** Python resolves the repo root
+ *    and the roadmap root, so a root holding `..` or (on macOS) a
+ *    `/var` -> `/private/var` symlink still yields a repo-relative path.
+ *    Resolving only one end reintroduces exactly the defect the Python test
+ *    says was caught by manual smoke-testing.
+ */
+export function checkRoadmapDuplicateHeadings(repoRoot: string): DuplicateRoadmapHeading[] {
+  const realpathOrSelf = (path: string): string => {
+    try {
+      return statSync(path).isDirectory() ? realpathSync(path) : path;
+    } catch {
+      return path;
+    }
+  };
+
+  const resolvedRoot = realpathOrSelf(resolve(repoRoot));
+  const resolvedRoadmapRoot = realpathOrSelf(resolve(roadmapPaths(repoRoot).roadmapRoot));
+
+  const duplicates = findDuplicateRoadmapHeadings(repoRoot);
+  return [...duplicates.entries()]
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+    .map(([code, directories]) => ({
+      code,
+      paths: directories
+        .map((directory) =>
+          toPosix(relative(resolvedRoot, join(resolvedRoadmapRoot, directory, "index.md"))),
+        )
+        .sort(),
+    }));
+}