@nanobpm/urban 0.69.0 → 0.70.0

package/src/context/retrieval/retriever.test.ts

@@ -0,0 +1,187 @@
+// Slice S4 (retrieval) — integration tests. The substrate is populated via S3's
+// REAL write path (ContextWriter over a local temp git repo); nothing touches
+// the network. This proves retrieval reads the exact layout S3 writes and that
+// the derived cache is a correct, disposable projection of git.
+
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, test } from "node:test";
+import { promisify } from "node:util";
+import { ContextWriter } from "../git/index.ts";
+import type { MemoryRecord } from "../schema/index.ts";
+import { ContextRetriever } from "./retriever.ts";
+
+const execFileAsync = promisify(execFile);
+const TEMP_ROOTS: string[] = [];
+
+after(async () => {
+  await Promise.all(TEMP_ROOTS.map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+async function git(cwd: string, ...args: string[]): Promise<void> {
+  await execFileAsync("git", args, { cwd });
+}
+
+/** A temp git repo with an initial commit on `main`, used as the substrate. */
+async function makeSubstrate(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), "s4-substrate-"));
+  TEMP_ROOTS.push(dir);
+  await git(dir, "init");
+  await writeFile(join(dir, "README.md"), "# context substrate\n", "utf8");
+  await git(dir, "add", "-A");
+  await git(dir, "-c", "user.name=seed", "-c", "user.email=seed@nanobpm.local", "commit", "--no-gpg-sign", "-m", "seed");
+  await git(dir, "branch", "-M", "main");
+  return dir;
+}
+
+function rec(overrides: Partial<MemoryRecord>): MemoryRecord {
+  const base: MemoryRecord = {
+    schemaVersion: 1,
+    id: "rec-1",
+    scope: "epic",
+    scopeRef: "issue-303",
+    mode: "empirical",
+    provenance: "measured",
+    authority: "authoritative",
+    statement: "the measured throughput was 42 rps",
+    createdAt: "2026-01-01T00:00:00Z",
+  };
+  return { ...base, ...overrides };
+}
+
+/** Populate a fresh substrate with a mix of records via S3's write path. */
+async function seededSubstrate(): Promise<{ dir: string; writer: ContextWriter }> {
+  const dir = await makeSubstrate();
+  const writer = new ContextWriter({ localPath: dir, ref: "main" });
+  await writer.appendRecord(rec({ id: "epic-measured", scope: "epic", scopeRef: "issue-303", statement: "the measured throughput was 42 rps" }));
+  await writer.appendRecord(rec({ id: "epic-human", scope: "epic", scopeRef: "issue-303", provenance: "human", mode: "normative", authority: "authoritative", statement: "prefer explicit errors over silent fallbacks" }));
+  await writer.appendRecord(rec({ id: "epic-other-ns", scope: "epic", scopeRef: "issue-999", provenance: "instance", mode: "empirical", authority: "authoritative", statement: "instance run observed a retry storm" }));
+  await writer.appendRecord(rec({ id: "repo-norm", scope: "repo", scopeRef: "nano-ide", provenance: "human", mode: "normative", authority: "authoritative", statement: "all writes go through governance" }));
+  await writer.appendRecord(rec({ id: "corpus-fact", scope: "corpus", scopeRef: undefined, provenance: "measured", mode: "empirical", authority: "authoritative", statement: "throughput baseline across the corpus" }));
+  return { dir, writer };
+}
+
+test("retrieves all records written via the S3 write path", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+  const all = await retriever.all();
+  assert.deepEqual(all.map((s) => s.record.id).sort(), [
+    "corpus-fact",
+    "epic-human",
+    "epic-measured",
+    "epic-other-ns",
+    "repo-norm",
+  ]);
+});
+
+test("is correct on a COLD cache (reads through to git without warming)", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+  assert.equal(retriever.cache.isWarm, false);
+
+  // A path-scoped query on a cold cache reads just that partition and does NOT
+  // warm the whole index — retrieval is correct before any rebuild.
+  const ns = await retriever.byNamespace("epic", "issue-303");
+  assert.deepEqual(ns.map((s) => s.record.id).sort(), ["epic-human", "epic-measured"]);
+  assert.equal(retriever.cache.isWarm, false);
+});
+
+test("structured / path-scoped query by scope + namespace", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+
+  const oneNs = await retriever.query({ scope: "epic", scopeRef: "issue-303" });
+  assert.deepEqual(oneNs.map((s) => s.record.id).sort(), ["epic-human", "epic-measured"]);
+  assert.equal(retriever.cache.isWarm, false, "single-namespace query must not force a full build");
+
+  const wholeScope = await retriever.query({ scope: "epic" });
+  assert.deepEqual(wholeScope.map((s) => s.record.id).sort(), ["epic-human", "epic-measured", "epic-other-ns"]);
+});
+
+test("frontmatter filtering on typed record fields (OR-sets, ANDed clauses)", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+
+  const humans = await retriever.query({ provenance: "human" });
+  assert.deepEqual(humans.map((s) => s.record.id).sort(), ["epic-human", "repo-norm"]);
+
+  const measuredOrInstance = await retriever.query({ provenance: ["measured", "instance"] });
+  assert.deepEqual(measuredOrInstance.map((s) => s.record.id).sort(), ["corpus-fact", "epic-measured", "epic-other-ns"]);
+
+  const normativeHumansInEpic = await retriever.query({ scope: "epic", mode: "normative", provenance: "human" });
+  assert.deepEqual(normativeHumansInEpic.map((s) => s.record.id), ["epic-human"]);
+});
+
+test("text search across record bodies, composable with filters", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+
+  const throughput = await retriever.search("throughput");
+  assert.deepEqual(throughput.map((s) => s.record.id).sort(), ["corpus-fact", "epic-measured"]);
+
+  const throughputMeasuredInEpic = await retriever.search("throughput", { scope: "epic" });
+  assert.deepEqual(throughputMeasuredInEpic.map((s) => s.record.id), ["epic-measured"]);
+});
+
+test("limit caps the result set", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+  const limited = await retriever.query({ limit: 2 });
+  assert.equal(limited.length, 2);
+});
+
+test("the derived cache is disposable: invalidate/rebuild re-projects from git", async () => {
+  const { dir, writer } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+
+  // Warm the cache with the current store.
+  const before = await retriever.all();
+  assert.equal(retriever.cache.isWarm, true);
+  const beforeCount = before.length;
+
+  // Land a new record via the S3 write path AFTER warming.
+  await writer.appendRecord(rec({ id: "late-arrival", scope: "epic", scopeRef: "issue-303", statement: "measured tail latency dropped" }));
+
+  // The warm cache still serves the old projection (it is a snapshot, not live).
+  assert.equal((await retriever.all()).length, beforeCount);
+
+  // Invalidate → the next read re-projects from git and sees the new record.
+  retriever.invalidate();
+  assert.equal(retriever.cache.isWarm, false);
+  const after = await retriever.all();
+  assert.equal(after.length, beforeCount + 1);
+  assert.ok(after.some((s) => s.record.id === "late-arrival"));
+
+  // rebuild() is an eager equivalent.
+  await retriever.rebuild();
+  assert.equal(retriever.cache.isWarm, true);
+});
+
+test("get() reads a single record by its S3 layout path", async () => {
+  const { dir } = await seededSubstrate();
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+  const all = await retriever.all();
+  const target = all.find((s) => s.record.id === "repo-norm");
+  assert.ok(target);
+  const fetched = await retriever.get(target.path);
+  assert.equal(fetched?.record.id, "repo-norm");
+});
+
+test("a ratified agent-retro prior is retrievable; provenance/authority survive the round-trip", async () => {
+  const dir = await makeSubstrate();
+  const writer = new ContextWriter({ localPath: dir, ref: "main" });
+  // agent-retro cannot be appended directly — it must be proposed then ratified.
+  const proposal = await writer.proposePrior(
+    rec({ id: "hypothesis-1", scope: "epic", scopeRef: "issue-303", provenance: "agent-retro", mode: "normative", authority: "hypothesis", statement: "hypothesis: batching reduces retries" }),
+  );
+  await writer.ratify(proposal);
+
+  const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
+  const priors = await retriever.query({ provenance: "agent-retro" });
+  assert.deepEqual(priors.map((s) => s.record.id), ["hypothesis-1"]);
+  // The schema invariant is preserved on read: an agent-retro prior is never authoritative.
+  assert.equal(priors[0]?.record.authority, "hypothesis");
+});

package/src/context/retrieval/retriever.ts

@@ -0,0 +1,193 @@
+// Slice S4 (retrieval) — the public retriever over the git system-of-record.
+//
+// `ContextRetriever` composes the three pieces of this slice into one API:
+//   - a RecordSource (git read seam, default GitRecordSource over a resolved
+//     substrate working copy),
+//   - a DerivedRecordIndex (disposable hot cache, rebuildable from git),
+//   - the pure query model (path-scoped + frontmatter + text search).
+//
+// Correctness does not depend on the cache: every query works from a cold/empty
+// index by reading through to git, and a query narrowed to one namespace reads
+// only that partition (a cheap, path-scoped read) without warming the whole
+// index. Callers keep the cache honest by calling `invalidate()` after a write
+// (git remains the source of truth); nothing here silently serves stale data
+// beyond what the caller has chosen to keep warm.
+
+import { DerivedRecordIndex } from "./cache.ts";
+import type { EmbeddingIndex } from "./embedding.ts";
+import {
+  compileQuery,
+  type CompiledQuery,
+  matchesQuery,
+  type RetrievalQuery,
+} from "./query.ts";
+import {
+  GitRecordSource,
+  type NamespaceSelector,
+  type RecordSource,
+  type ResolvedReadHandle,
+  type StoredRecord,
+} from "./source.ts";
+
+/** Options for {@link ContextRetriever}. All optional; the defaults are safe. */
+export interface ContextRetrieverOptions {
+  /**
+   * Override the read source (seam for a future mutable/PII backend). Defaults
+   * to a {@link GitRecordSource} over the resolved handle's working copy.
+   */
+  readonly source?: RecordSource;
+  /**
+   * A future opt-in derived embedding index. Purely a forward-compat seam — the
+   * MVP does not build or require one, and no method here depends on it.
+   */
+  readonly embeddingIndex?: EmbeddingIndex;
+}
+
+/**
+ * Retrieval over the git-backed context store. Bind one to a resolved substrate
+ * handle (the local checkout S1 produces) and query it; the derived hot cache is
+ * built lazily and can be invalidated/rebuilt at will.
+ */
+export class ContextRetriever {
+  readonly #source: RecordSource;
+  readonly #index: DerivedRecordIndex;
+  readonly #embeddingIndex?: EmbeddingIndex;
+
+  constructor(
+    handle: ResolvedReadHandle | string,
+    options: ContextRetrieverOptions = {},
+  ) {
+    this.#source = options.source ?? new GitRecordSource(handle);
+    this.#index = new DerivedRecordIndex(this.#source);
+    this.#embeddingIndex = options.embeddingIndex;
+  }
+
+  /** The derived hot cache (for explicit warm/invalidate/rebuild control). */
+  get cache(): DerivedRecordIndex {
+    return this.#index;
+  }
+
+  /** The read source this retriever reads git through. */
+  get source(): RecordSource {
+    return this.#source;
+  }
+
+  /**
+   * A future opt-in embedding index, if one was supplied. `undefined` in the
+   * MVP — retrieval is fully functional without it.
+   */
+  get embeddingIndex(): EmbeddingIndex | undefined {
+    return this.#embeddingIndex;
+  }
+
+  /**
+   * Run a retrieval query. Structured/path-scoped clauses (`scope` + a single
+   * `scopeRef`) narrow the read to one layout partition when the cache is cold,
+   * so a targeted query is cheap and never forces a full index build; otherwise
+   * the frontmatter + text clauses filter the candidate set in memory. Results
+   * are ordered by layout path for determinism and capped by `limit`.
+   */
+  async query(query: RetrievalQuery = {}): Promise<readonly StoredRecord[]> {
+    const compiled = compileQuery(query);
+    const candidates = await this.#candidates(compiled);
+    const matched = candidates.filter((stored) =>
+      matchesQuery(stored.record, compiled),
+    );
+    return applyLimit(matched, compiled);
+  }
+
+  /**
+   * Path-scoped structured query: every record in one namespace partition
+   * (`scope` + optional `scopeRef`), read cheaply from just that subtree when
+   * the cache is cold. Equivalent to `query({ scope, scopeRef })` but expresses
+   * the S3-partition intent directly.
+   */
+  async byNamespace(
+    scope: NamespaceSelector["scope"],
+    scopeRef?: string,
+  ): Promise<readonly StoredRecord[]> {
+    return this.#index.namespace({ scope, scopeRef });
+  }
+
+  /**
+   * Free-text search across record bodies, optionally combined with structured
+   * frontmatter filters. Convenience over {@link query} for the common case.
+   */
+  async search(
+    text: string,
+    filters: Omit<RetrievalQuery, "text"> = {},
+  ): Promise<readonly StoredRecord[]> {
+    return this.query({ ...filters, text });
+  }
+
+  /** Every record in the store, warming the cache. */
+  async all(): Promise<readonly StoredRecord[]> {
+    return this.#index.all();
+  }
+
+  /** Read a single record by its S3 layout path (bypasses the cache). */
+  async get(relPath: string): Promise<StoredRecord | undefined> {
+    return this.#source.read(relPath);
+  }
+
+  /** Drop the derived cache so the next query re-reads from git. */
+  invalidate(): void {
+    this.#index.invalidate();
+  }
+
+  /** Eagerly rebuild the derived cache from git. */
+  async rebuild(): Promise<readonly StoredRecord[]> {
+    return this.#index.rebuild();
+  }
+
+  /**
+   * The candidate set to filter: when a query pins exactly one scope we read
+   * only that scope's subtree (cheap path-scoped read, cold-cache friendly) —
+   * pinning a single scopeRef narrows it further to that one namespace
+   * partition, otherwise the whole scope subtree (spanning its refs) is read;
+   * when no single scope is pinned we read the whole (cached) store.
+   */
+  async #candidates(query: CompiledQuery): Promise<readonly StoredRecord[]> {
+    const selector = singleNamespace(query);
+    if (selector !== undefined) {
+      return this.#index.namespace(selector);
+    }
+    return this.#index.all();
+  }
+}
+
+/**
+ * If a compiled query pins exactly ONE scope (and at most one scopeRef), return
+ * the namespace partition it lives in so retrieval can read just that subtree.
+ * Returns `undefined` when the query spans multiple scopes/refs (or none), in
+ * which case the whole store is the candidate set.
+ */
+function singleNamespace(query: CompiledQuery): NamespaceSelector | undefined {
+  const scope = onlyMember(query.scope);
+  if (scope === undefined) {
+    return undefined;
+  }
+  const scopeRef = onlyMember(query.scopeRef);
+  return { scope, ...(scopeRef === undefined ? {} : { scopeRef }) };
+}
+
+/** The sole member of a set of exactly one element, else `undefined`. */
+function onlyMember<T>(set: ReadonlySet<T> | undefined): T | undefined {
+  if (set === undefined || set.size !== 1) {
+    return undefined;
+  }
+  for (const value of set) {
+    return value;
+  }
+  return undefined;
+}
+
+function applyLimit(
+  records: readonly StoredRecord[],
+  query: CompiledQuery,
+): readonly StoredRecord[] {
+  if (query.limit === undefined || query.limit < 0) {
+    return records;
+  }
+  return records.slice(0, query.limit);
+}

package/src/context/retrieval/source.test.ts

@@ -0,0 +1,118 @@
+// Slice S4 (retrieval) — tests for the git read source. These write record
+// files straight onto disk under the S3 layout (no git needed) to prove the
+// source selects/parses/skips correctly and reads namespaces cheaply.
+
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { after, test } from "node:test";
+import { recordRelativePath } from "../git/index.ts";
+import type { MemoryRecord } from "../schema/index.ts";
+import { GitRecordSource, RecordSourceError } from "./source.ts";
+
+const TEMP_ROOTS: string[] = [];
+
+after(async () => {
+  await Promise.all(TEMP_ROOTS.map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+async function makeRoot(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), "s4-source-"));
+  TEMP_ROOTS.push(dir);
+  return dir;
+}
+
+function rec(overrides: Partial<MemoryRecord>): MemoryRecord {
+  const base: MemoryRecord = {
+    schemaVersion: 1,
+    id: "rec-1",
+    scope: "epic",
+    scopeRef: "issue-303",
+    mode: "empirical",
+    provenance: "measured",
+    authority: "authoritative",
+    statement: "the measured throughput was 42 rps",
+    createdAt: "2026-01-01T00:00:00Z",
+  };
+  return { ...base, ...overrides };
+}
+
+/** Write a record onto disk at its canonical S3 layout path. */
+async function put(root: string, record: MemoryRecord): Promise<string> {
+  const relPath = recordRelativePath(record);
+  const abs = join(root, relPath);
+  await mkdir(dirname(abs), { recursive: true });
+  await writeFile(abs, JSON.stringify(record, null, 2), "utf8");
+  return relPath;
+}
+
+test("constructor rejects a relative substrate root", () => {
+  assert.throws(() => new GitRecordSource("relative/path"), RecordSourceError);
+});
+
+test("list() on an empty / missing store is [] (cold read degrades gracefully)", async () => {
+  const root = await makeRoot();
+  const source = new GitRecordSource(root);
+  assert.deepEqual(await source.list(), []);
+  assert.deepEqual(await source.list({ scope: "epic", scopeRef: "nope" }), []);
+});
+
+test("list() reads every record under the layout, in deterministic path order", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "b", scope: "epic", scopeRef: "issue-303" }));
+  await put(root, rec({ id: "a", scope: "repo", scopeRef: undefined, provenance: "human", mode: "normative", authority: "authoritative" }));
+  const source = new GitRecordSource(root);
+  const all = await source.list();
+  assert.equal(all.length, 2);
+  // Sorted by substrate-relative path.
+  assert.deepEqual([...all].map((s) => s.path), [...all].map((s) => s.path).sort());
+  const ids = all.map((s) => s.record.id).sort();
+  assert.deepEqual(ids, ["a", "b"]);
+});
+
+test("list(selector) reads only one namespace partition", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "x", scope: "epic", scopeRef: "issue-303" }));
+  await put(root, rec({ id: "y", scope: "epic", scopeRef: "issue-999" }));
+  await put(root, rec({ id: "z", scope: "repo", scopeRef: "nano-ide", provenance: "human", mode: "normative" }));
+  const source = new GitRecordSource(root);
+
+  const ns = await source.list({ scope: "epic", scopeRef: "issue-303" });
+  assert.deepEqual(ns.map((s) => s.record.id), ["x"]);
+
+  const scopeWide = await source.list({ scope: "epic" });
+  assert.deepEqual(scopeWide.map((s) => s.record.id).sort(), ["x", "y"]);
+});
+
+test("non-record and corrupt files under the layout are skipped, not fatal", async () => {
+  const root = await makeRoot();
+  await put(root, rec({ id: "good" }));
+  // A corrupt JSON file at a record-looking path.
+  const badPath = join(root, "records", "epic", "issue-303", "broken.json");
+  await mkdir(dirname(badPath), { recursive: true });
+  await writeFile(badPath, "{ not json", "utf8");
+  // A valid JSON file that is NOT a valid record.
+  const notRecord = join(root, "records", "epic", "issue-303", "notarecord.json");
+  await writeFile(notRecord, JSON.stringify({ hello: "world" }), "utf8");
+  // A non-json file is ignored by isRecordPath.
+  await writeFile(join(root, "records", "epic", "issue-303", "README.md"), "hi", "utf8");
+
+  const source = new GitRecordSource(root);
+  const all = await source.list();
+  assert.deepEqual(all.map((s) => s.record.id), ["good"]);
+});
+
+test("read() returns a single record by layout path, undefined when absent/invalid", async () => {
+  const root = await makeRoot();
+  const relPath = await put(root, rec({ id: "solo" }));
+  const source = new GitRecordSource(root);
+
+  const found = await source.read(relPath);
+  assert.equal(found?.record.id, "solo");
+
+  assert.equal(await source.read("records/epic/issue-303/missing.json"), undefined);
+  // A path outside the layout is rejected.
+  assert.equal(await source.read("../escape.json"), undefined);
+  assert.equal(await source.read("secrets.json"), undefined);
+});