@nanobpm/urban 0.69.1 → 0.71.0

package/src/context/retrieval/source.test.ts

@@ -0,0 +1,118 @@
+// Slice S4 (retrieval) — tests for the git read source. These write record
+// files straight onto disk under the S3 layout (no git needed) to prove the
+// source selects/parses/skips correctly and reads namespaces cheaply.
+
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { after, test } from "node:test";
+import { recordRelativePath } from "../git/index.ts";
+import type { MemoryRecord } from "../schema/index.ts";
+import { GitRecordSource, RecordSourceError } from "./source.ts";
+
+const TEMP_ROOTS: string[] = [];
+
+after(async () => {
+  await Promise.all(TEMP_ROOTS.map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+async function makeRoot(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), "s4-source-"));
+  TEMP_ROOTS.push(dir);
+  return dir;
+}
+
+function rec(overrides: Partial<MemoryRecord>): MemoryRecord {
+  const base: MemoryRecord = {
+    schemaVersion: 1,
+    id: "rec-1",
+    scope: "epic",
+    scopeRef: "issue-303",
+    mode: "empirical",
+    provenance: "measured",
+    authority: "authoritative",
+    statement: "the measured throughput was 42 rps",
+    createdAt: "2026-01-01T00:00:00Z",
+  };
+  return { ...base, ...overrides };
+}
+
+/** Write a record onto disk at its canonical S3 layout path. */
+async function put(root: string, record: MemoryRecord): Promise<string> {
+  const relPath = recordRelativePath(record);
+  const abs = join(root, relPath);
+  await mkdir(dirname(abs), { recursive: true });
+  await writeFile(abs, JSON.stringify(record, null, 2), "utf8");
+  return relPath;
+}
+
+test("constructor rejects a relative substrate root", () => {
+  assert.throws(() => new GitRecordSource("relative/path"), RecordSourceError);
+});
+
+test("list() on an empty / missing store is [] (cold read degrades gracefully)", async () => {
+  const root = await makeRoot();
+  const source = new GitRecordSource(root);
+  assert.deepEqual(await source.list(), []);
+  assert.deepEqual(await source.list({ scope: "epic", scopeRef: "nope" }), []);
+});
+
+test("list() reads every record under the layout, in deterministic path order", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "b", scope: "epic", scopeRef: "issue-303" }));
+  await put(root, rec({ id: "a", scope: "repo", scopeRef: undefined, provenance: "human", mode: "normative", authority: "authoritative" }));
+  const source = new GitRecordSource(root);
+  const all = await source.list();
+  assert.equal(all.length, 2);
+  // Sorted by substrate-relative path.
+  assert.deepEqual([...all].map((s) => s.path), [...all].map((s) => s.path).sort());
+  const ids = all.map((s) => s.record.id).sort();
+  assert.deepEqual(ids, ["a", "b"]);
+});
+
+test("list(selector) reads only one namespace partition", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "x", scope: "epic", scopeRef: "issue-303" }));
+  await put(root, rec({ id: "y", scope: "epic", scopeRef: "issue-999" }));
+  await put(root, rec({ id: "z", scope: "repo", scopeRef: "nano-ide", provenance: "human", mode: "normative" }));
+  const source = new GitRecordSource(root);
+
+  const ns = await source.list({ scope: "epic", scopeRef: "issue-303" });
+  assert.deepEqual(ns.map((s) => s.record.id), ["x"]);
+
+  const scopeWide = await source.list({ scope: "epic" });
+  assert.deepEqual(scopeWide.map((s) => s.record.id).sort(), ["x", "y"]);
+});
+
+test("non-record and corrupt files under the layout are skipped, not fatal", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "good" }));
+  // A corrupt JSON file at a record-looking path.
+  const badPath = join(root, "records", "epic", "issue-303", "broken.json");
+  await mkdir(dirname(badPath), { recursive: true });
+  await writeFile(badPath, "{ not json", "utf8");
+  // A valid JSON file that is NOT a valid record.
+  const notRecord = join(root, "records", "epic", "issue-303", "notarecord.json");
+  await writeFile(notRecord, JSON.stringify({ hello: "world" }), "utf8");
+  // A non-json file is ignored by isRecordPath.
+  await writeFile(join(root, "records", "epic", "issue-303", "README.md"), "hi", "utf8");
+
+  const source = new GitRecordSource(root);
+  const all = await source.list();
+  assert.deepEqual(all.map((s) => s.record.id), ["good"]);
+});
+
+test("read() returns a single record by layout path, undefined when absent/invalid", async () => {
+  const root = await makeRoot();
+  const relPath = await put(root, rec({ id: "solo" }));
+  const source = new GitRecordSource(root);
+
+  const found = await source.read(relPath);
+  assert.equal(found?.record.id, "solo");
+
+  assert.equal(await source.read("records/epic/issue-303/missing.json"), undefined);
+  // A path outside the layout is rejected.
+  assert.equal(await source.read("../escape.json"), undefined);
+  assert.equal(await source.read("secrets.json"), undefined);
+});

package/src/context/retrieval/source.ts

@@ -0,0 +1,218 @@
+// Slice S4 (retrieval) — the READ seam over the git system-of-record.
+//
+// Retrieval treats git as the authoritative store: records live on disk under
+// the exact layout S3 defines (`records/<scope>/<scopeRef-bucket>/<id>.json`),
+// and this module reads them back. It is deliberately abstracted behind a
+// {@link RecordSource} interface so the derived hot cache, the query layer, and
+// a future PII/mutable backend all read through one seam rather than assuming a
+// public-git working copy. {@link GitRecordSource} is the default MVP: it walks
+// a resolved substrate working copy (the local checkout S1 produces, pinned to
+// its ref) using the S3 layout helpers to select record files.
+//
+// The source NEVER caches — it always reflects what is on disk (== the git ref
+// that is checked out). Caching is the derived index's job (see cache.ts), and
+// the index is disposable: a cold/empty index falls back to reading here, so
+// retrieval is correct even before any index is built.
+
+import type { Dirent } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
+import { isAbsolute, join, relative, sep } from "node:path";
+import {
+  isRecordPath,
+  LAYOUT_ROOT,
+  recordDir,
+  scopeDir,
+} from "../git/index.ts";
+import {
+  type MemoryRecord,
+  type MemoryScope,
+  validateMemoryRecord,
+} from "../schema/index.ts";
+
+/** A record as it is stored in the substrate: its layout path + the record. */
+export interface StoredRecord {
+  /** Substrate-root-relative POSIX path the record lives at (S3 layout). */
+  readonly path: string;
+  /** The validated memory record parsed from that file. */
+  readonly record: MemoryRecord;
+}
+
+/**
+ * A path-scoped selector for a namespace partition in the S3 layout. Reading a
+ * single `(scope, scopeRef)` — or a whole `scope` — is a cheap structured query
+ * because the layout partitions records into one directory per namespace, so the
+ * source can walk just that subtree instead of the whole store.
+ */
+export interface NamespaceSelector {
+  readonly scope: MemoryScope;
+  /** When present, narrows to one namespace bucket; absent lists the scope. */
+  readonly scopeRef?: string;
+}
+
+/**
+ * The read seam retrieval is built on. A future mutable/PII backend can
+ * implement this differently (e.g. against a database) without the cache, the
+ * query layer, or callers changing.
+ */
+export interface RecordSource {
+  /**
+   * List every stored record, optionally narrowed to one namespace partition
+   * (a cheap, path-scoped read that walks only that subtree of the S3 layout).
+   * Invalid / unparseable files under the layout root are skipped, not fatal —
+   * a single corrupt file must never make the whole store unreadable.
+   */
+  list(selector?: NamespaceSelector): Promise<readonly StoredRecord[]>;
+  /**
+   * Read a single record by its substrate-relative layout path, or `undefined`
+   * when it is absent or not a valid record file.
+   */
+  read(relPath: string): Promise<StoredRecord | undefined>;
+}
+
+/** The minimal resolved-handle shape the read source consumes (from S1). */
+export interface ResolvedReadHandle {
+  /** Absolute path to the local substrate working copy (checked out at `ref`). */
+  readonly localPath: string;
+  /** The git ref the working copy is pinned to (informational for reads). */
+  readonly ref?: string;
+}
+
+/** Thrown when the underlying substrate cannot be read at all. */
+export class RecordSourceError extends Error {
+  constructor(message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "RecordSourceError";
+  }
+}
+
+function isEnoent(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    error.code === "ENOENT"
+  );
+}
+
+/** Convert a native absolute path to a substrate-root-relative POSIX path. */
+function toPosixRelative(root: string, absPath: string): string {
+  return relative(root, absPath).split(sep).join("/");
+}
+
+/**
+ * The default git-backed read source: reads records straight off a local
+ * substrate working copy. Git is the system of record; this holds no state of
+ * its own, so every call reflects the current on-disk (== checked-out ref)
+ * content. That is what makes the derived index safe to throw away and rebuild.
+ */
+export class GitRecordSource implements RecordSource {
+  readonly #root: string;
+
+  constructor(handle: ResolvedReadHandle | string) {
+    const root = typeof handle === "string" ? handle : handle.localPath;
+    if (!isAbsolute(root)) {
+      throw new RecordSourceError(
+        `substrate root must be an absolute path, got: ${root}`,
+      );
+    }
+    this.#root = root;
+  }
+
+  /** Absolute path of the substrate working copy this source reads from. */
+  get root(): string {
+    return this.#root;
+  }
+
+  async list(selector?: NamespaceSelector): Promise<readonly StoredRecord[]> {
+    const subtree = selectorSubtree(selector);
+    const walkRoot = join(this.#root, subtree);
+
+    let entries: Dirent[];
+    try {
+      entries = await readdir(walkRoot, { recursive: true, withFileTypes: true });
+    } catch (error) {
+      // A missing subtree (e.g. an empty store, or a namespace with no records)
+      // is an empty result, not an error — a cold read must degrade gracefully.
+      if (isEnoent(error)) {
+        return [];
+      }
+      throw new RecordSourceError(`failed to list substrate at ${walkRoot}`, {
+        cause: error,
+      });
+    }
+
+    const results: StoredRecord[] = [];
+    for (const entry of entries) {
+      if (!entry.isFile()) {
+        continue;
+      }
+      const absPath = join(entry.parentPath, entry.name);
+      const relPath = toPosixRelative(this.#root, absPath);
+      if (!isRecordPath(relPath)) {
+        continue;
+      }
+      const stored = await this.#readAbsolute(absPath, relPath);
+      if (stored !== undefined) {
+        results.push(stored);
+      }
+    }
+    // Deterministic order regardless of filesystem enumeration order.
+    results.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    return results;
+  }
+
+  async read(relPath: string): Promise<StoredRecord | undefined> {
+    const normalised = relPath.replace(/\\/g, "/").replace(/^\.\//, "");
+    if (!isRecordPath(normalised)) {
+      return undefined;
+    }
+    return this.#readAbsolute(join(this.#root, normalised), normalised);
+  }
+
+  async #readAbsolute(
+    absPath: string,
+    relPath: string,
+  ): Promise<StoredRecord | undefined> {
+    let raw: string;
+    try {
+      raw = await readFile(absPath, "utf8");
+    } catch (error) {
+      if (isEnoent(error)) {
+        return undefined;
+      }
+      throw new RecordSourceError(`failed to read record at ${relPath}`, {
+        cause: error,
+      });
+    }
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      // A corrupt / non-JSON file under the layout is skipped, not fatal.
+      return undefined;
+    }
+
+    const result = validateMemoryRecord(parsed);
+    if (!result.ok) {
+      // A file that does not validate against the S2 schema is not a record we
+      // can safely surface — skip it rather than return an unchecked shape.
+      return undefined;
+    }
+    return { path: relPath, record: result.record };
+  }
+}
+
+/**
+ * The layout subtree to walk for a selector: one namespace bucket, one whole
+ * scope, or the entire layout root. Keeps path-scoping in one place so the
+ * cache and query layers never re-derive layout paths themselves.
+ */
+export function selectorSubtree(selector?: NamespaceSelector): string {
+  if (selector === undefined) {
+    return LAYOUT_ROOT;
+  }
+  return selector.scopeRef === undefined
+    ? scopeDir(selector.scope)
+    : recordDir(selector.scope, selector.scopeRef);
+}