@nanobpm/urban 0.69.1 → 0.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/retrieval/cache.d.ts +35 -0
- package/dist/context/retrieval/cache.js +75 -0
- package/dist/context/retrieval/embedding.d.ts +27 -0
- package/dist/context/retrieval/embedding.js +13 -0
- package/dist/context/retrieval/index.d.ts +5 -1
- package/dist/context/retrieval/index.js +21 -1
- package/dist/context/retrieval/query.d.ts +50 -0
- package/dist/context/retrieval/query.js +62 -0
- package/dist/context/retrieval/retriever.d.ts +63 -0
- package/dist/context/retrieval/retriever.js +136 -0
- package/dist/context/retrieval/source.d.ts +71 -0
- package/dist/context/retrieval/source.js +141 -0
- package/package.json +1 -1
- package/src/context/README.md +44 -0
- package/src/context/pii/check-pii.test.ts +196 -0
- package/src/context/retrieval/NOTES.md +68 -0
- package/src/context/retrieval/cache.ts +95 -0
- package/src/context/retrieval/embedding.ts +45 -0
- package/src/context/retrieval/index.ts +47 -2
- package/src/context/retrieval/query.test.ts +72 -0
- package/src/context/retrieval/query.ts +118 -0
- package/src/context/retrieval/retriever.test.ts +187 -0
- package/src/context/retrieval/retriever.ts +193 -0
- package/src/context/retrieval/source.test.ts +118 -0
- package/src/context/retrieval/source.ts +218 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — unit tests for the pure query model.
|
|
2
|
+
|
|
3
|
+
import assert from "node:assert/strict";
|
|
4
|
+
import { test } from "node:test";
|
|
5
|
+
import type { MemoryRecord } from "../schema/index.ts";
|
|
6
|
+
import { compileQuery, matchesQuery, type RetrievalQuery } from "./query.ts";
|
|
7
|
+
|
|
8
|
+
function rec(overrides: Partial<MemoryRecord>): MemoryRecord {
|
|
9
|
+
const base: MemoryRecord = {
|
|
10
|
+
schemaVersion: 1,
|
|
11
|
+
id: "rec-1",
|
|
12
|
+
scope: "epic",
|
|
13
|
+
scopeRef: "issue-303",
|
|
14
|
+
mode: "empirical",
|
|
15
|
+
provenance: "measured",
|
|
16
|
+
authority: "authoritative",
|
|
17
|
+
statement: "the measured throughput was 42 rps",
|
|
18
|
+
createdAt: "2026-01-01T00:00:00Z",
|
|
19
|
+
};
|
|
20
|
+
return { ...base, ...overrides };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function matches(record: MemoryRecord, query: RetrievalQuery): boolean {
|
|
24
|
+
return matchesQuery(record, compileQuery(query));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
test("an empty query matches every record", () => {
|
|
28
|
+
assert.equal(matches(rec({}), {}), true);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("frontmatter clauses are ANDed", () => {
|
|
32
|
+
const record = rec({ provenance: "measured", mode: "empirical" });
|
|
33
|
+
assert.equal(matches(record, { provenance: "measured", mode: "empirical" }), true);
|
|
34
|
+
assert.equal(matches(record, { provenance: "measured", mode: "normative" }), false);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("a clause given as an array is an OR-set", () => {
|
|
38
|
+
const record = rec({ scope: "repo" });
|
|
39
|
+
assert.equal(matches(record, { scope: ["epic", "repo"] }), true);
|
|
40
|
+
assert.equal(matches(record, { scope: ["epic", "corpus"] }), false);
|
|
41
|
+
assert.equal(matches(record, { scope: "repo" }), true);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("scopeRef filter distinguishes namespaces", () => {
|
|
45
|
+
assert.equal(matches(rec({ scopeRef: "issue-303" }), { scopeRef: "issue-303" }), true);
|
|
46
|
+
assert.equal(matches(rec({ scopeRef: "issue-999" }), { scopeRef: "issue-303" }), false);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("a scopeRef filter never matches a record without a scopeRef", () => {
|
|
50
|
+
const unscoped = rec({ scope: "corpus", scopeRef: undefined });
|
|
51
|
+
assert.equal(matches(unscoped, { scopeRef: "issue-303" }), false);
|
|
52
|
+
assert.equal(matches(unscoped, {}), true);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("text search is a case-insensitive substring over the body", () => {
|
|
56
|
+
const record = rec({ statement: "The Measured Throughput was 42 RPS", subject: "latency" });
|
|
57
|
+
assert.equal(matches(record, { text: "throughput" }), true);
|
|
58
|
+
assert.equal(matches(record, { text: "THROUGHPUT" }), true);
|
|
59
|
+
assert.equal(matches(record, { text: "latency" }), true); // subject
|
|
60
|
+
assert.equal(matches(record, { text: "rec-1" }), true); // id
|
|
61
|
+
assert.equal(matches(record, { text: "absent" }), false);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("an empty text needle matches everything", () => {
|
|
65
|
+
assert.equal(matches(rec({}), { text: "" }), true);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("text search composes with structured filters (AND)", () => {
|
|
69
|
+
const record = rec({ provenance: "human", mode: "normative", statement: "prefer explicit errors" });
|
|
70
|
+
assert.equal(matches(record, { provenance: "human", text: "explicit" }), true);
|
|
71
|
+
assert.equal(matches(record, { provenance: "measured", text: "explicit" }), false);
|
|
72
|
+
});
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — the query model: frontmatter filtering + text search.
|
|
2
|
+
//
|
|
3
|
+
// A retrieval query is a conjunction of independent predicates: path-scoped
|
|
4
|
+
// namespace narrowing (scope/scopeRef), frontmatter filtering on the typed
|
|
5
|
+
// record fields (mode/provenance/authority/…), and a free-text search across
|
|
6
|
+
// the record body. Every clause is optional; an empty query matches everything.
|
|
7
|
+
// The matching here is pure and cache-agnostic — it runs identically over
|
|
8
|
+
// records read straight from git (cold cache) or served from the hot index.
|
|
9
|
+
|
|
10
|
+
import type {
|
|
11
|
+
MemoryAuthority,
|
|
12
|
+
MemoryMode,
|
|
13
|
+
MemoryProvenance,
|
|
14
|
+
MemoryRecord,
|
|
15
|
+
MemoryScope,
|
|
16
|
+
} from "../schema/index.ts";
|
|
17
|
+
|
|
18
|
+
/** A field value that may be given as a single value or an OR-set of values. */
|
|
19
|
+
export type OneOrMany<T> = T | readonly T[];
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A retrieval query. All clauses are ANDed together; a clause given as an array
|
|
23
|
+
* is an OR over its members. Absent clauses impose no constraint.
|
|
24
|
+
*/
|
|
25
|
+
export interface RetrievalQuery {
|
|
26
|
+
/** Structured / path-scoped: restrict to one or more scope ladder levels. */
|
|
27
|
+
readonly scope?: OneOrMany<MemoryScope>;
|
|
28
|
+
/** Structured / path-scoped: restrict to one or more `scopeRef` namespaces. */
|
|
29
|
+
readonly scopeRef?: OneOrMany<string>;
|
|
30
|
+
/** Frontmatter filter: normative vs empirical. */
|
|
31
|
+
readonly mode?: OneOrMany<MemoryMode>;
|
|
32
|
+
/** Frontmatter filter: where the record came from. */
|
|
33
|
+
readonly provenance?: OneOrMany<MemoryProvenance>;
|
|
34
|
+
/** Frontmatter filter: hypothesis vs authoritative. */
|
|
35
|
+
readonly authority?: OneOrMany<MemoryAuthority>;
|
|
36
|
+
/** Frontmatter filter: exact record id(s). */
|
|
37
|
+
readonly id?: OneOrMany<string>;
|
|
38
|
+
/** Frontmatter filter: exact subject(s). */
|
|
39
|
+
readonly subject?: OneOrMany<string>;
|
|
40
|
+
/**
|
|
41
|
+
* Free-text search across the record body — case-insensitive substring match
|
|
42
|
+
* over `statement`, `subject` (when present), and `id`. This is the "text
|
|
43
|
+
* search across record bodies" clause; it composes with the structured filters.
|
|
44
|
+
*/
|
|
45
|
+
readonly text?: string;
|
|
46
|
+
/** Cap the number of results returned (applied after ordering). */
|
|
47
|
+
readonly limit?: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function toSet<T>(value: OneOrMany<T> | undefined): ReadonlySet<T> | undefined {
|
|
51
|
+
if (value === undefined) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
return new Set(Array.isArray(value) ? value : [value]);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function matchesOneOf<T>(
|
|
58
|
+
allowed: ReadonlySet<T> | undefined,
|
|
59
|
+
actual: T | undefined,
|
|
60
|
+
): boolean {
|
|
61
|
+
if (allowed === undefined) {
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
return actual !== undefined && allowed.has(actual);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* A query compiled to sets for O(1) membership tests. Building it once and
|
|
69
|
+
* reusing it across a candidate list avoids re-normalising the query per record.
|
|
70
|
+
*/
|
|
71
|
+
export interface CompiledQuery {
|
|
72
|
+
readonly scope?: ReadonlySet<MemoryScope>;
|
|
73
|
+
readonly scopeRef?: ReadonlySet<string>;
|
|
74
|
+
readonly mode?: ReadonlySet<MemoryMode>;
|
|
75
|
+
readonly provenance?: ReadonlySet<MemoryProvenance>;
|
|
76
|
+
readonly authority?: ReadonlySet<MemoryAuthority>;
|
|
77
|
+
readonly id?: ReadonlySet<string>;
|
|
78
|
+
readonly subject?: ReadonlySet<string>;
|
|
79
|
+
readonly text?: string;
|
|
80
|
+
readonly limit?: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Compile a raw query into membership sets + a normalised (lowercased) needle. */
|
|
84
|
+
export function compileQuery(query: RetrievalQuery): CompiledQuery {
|
|
85
|
+
return {
|
|
86
|
+
scope: toSet(query.scope),
|
|
87
|
+
scopeRef: toSet(query.scopeRef),
|
|
88
|
+
mode: toSet(query.mode),
|
|
89
|
+
provenance: toSet(query.provenance),
|
|
90
|
+
authority: toSet(query.authority),
|
|
91
|
+
id: toSet(query.id),
|
|
92
|
+
subject: toSet(query.subject),
|
|
93
|
+
text: query.text === undefined ? undefined : query.text.toLowerCase(),
|
|
94
|
+
limit: query.limit,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** `true` iff `record` satisfies every clause of the compiled query. */
|
|
99
|
+
export function matchesQuery(record: MemoryRecord, query: CompiledQuery): boolean {
|
|
100
|
+
if (!matchesOneOf(query.scope, record.scope)) return false;
|
|
101
|
+
if (!matchesOneOf(query.scopeRef, record.scopeRef)) return false;
|
|
102
|
+
if (!matchesOneOf(query.mode, record.mode)) return false;
|
|
103
|
+
if (!matchesOneOf(query.provenance, record.provenance)) return false;
|
|
104
|
+
if (!matchesOneOf(query.authority, record.authority)) return false;
|
|
105
|
+
if (!matchesOneOf(query.id, record.id)) return false;
|
|
106
|
+
if (!matchesOneOf(query.subject, record.subject)) return false;
|
|
107
|
+
if (query.text !== undefined && !matchesText(record, query.text)) return false;
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Case-insensitive substring search across the record's textual body. */
|
|
112
|
+
function matchesText(record: MemoryRecord, needle: string): boolean {
|
|
113
|
+
if (needle === "") {
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
const haystacks = [record.statement, record.subject ?? "", record.id];
|
|
117
|
+
return haystacks.some((field) => field.toLowerCase().includes(needle));
|
|
118
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — integration tests. The substrate is populated via S3's
|
|
2
|
+
// REAL write path (ContextWriter over a local temp git repo); nothing touches
|
|
3
|
+
// the network. This proves retrieval reads the exact layout S3 writes and that
|
|
4
|
+
// the derived cache is a correct, disposable projection of git.
|
|
5
|
+
|
|
6
|
+
import assert from "node:assert/strict";
|
|
7
|
+
import { execFile } from "node:child_process";
|
|
8
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { after, test } from "node:test";
|
|
12
|
+
import { promisify } from "node:util";
|
|
13
|
+
import { ContextWriter } from "../git/index.ts";
|
|
14
|
+
import type { MemoryRecord } from "../schema/index.ts";
|
|
15
|
+
import { ContextRetriever } from "./retriever.ts";
|
|
16
|
+
|
|
17
|
+
const execFileAsync = promisify(execFile);
|
|
18
|
+
const TEMP_ROOTS: string[] = [];
|
|
19
|
+
|
|
20
|
+
after(async () => {
|
|
21
|
+
await Promise.all(TEMP_ROOTS.map((dir) => rm(dir, { recursive: true, force: true })));
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
async function git(cwd: string, ...args: string[]): Promise<void> {
|
|
25
|
+
await execFileAsync("git", args, { cwd });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A temp git repo with an initial commit on `main`, used as the substrate. */
|
|
29
|
+
async function makeSubstrate(): Promise<string> {
|
|
30
|
+
const dir = await mkdtemp(join(tmpdir(), "s4-substrate-"));
|
|
31
|
+
TEMP_ROOTS.push(dir);
|
|
32
|
+
await git(dir, "init");
|
|
33
|
+
await writeFile(join(dir, "README.md"), "# context substrate\n", "utf8");
|
|
34
|
+
await git(dir, "add", "-A");
|
|
35
|
+
await git(dir, "-c", "user.name=seed", "-c", "user.email=seed@nanobpm.local", "commit", "--no-gpg-sign", "-m", "seed");
|
|
36
|
+
await git(dir, "branch", "-M", "main");
|
|
37
|
+
return dir;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function rec(overrides: Partial<MemoryRecord>): MemoryRecord {
|
|
41
|
+
const base: MemoryRecord = {
|
|
42
|
+
schemaVersion: 1,
|
|
43
|
+
id: "rec-1",
|
|
44
|
+
scope: "epic",
|
|
45
|
+
scopeRef: "issue-303",
|
|
46
|
+
mode: "empirical",
|
|
47
|
+
provenance: "measured",
|
|
48
|
+
authority: "authoritative",
|
|
49
|
+
statement: "the measured throughput was 42 rps",
|
|
50
|
+
createdAt: "2026-01-01T00:00:00Z",
|
|
51
|
+
};
|
|
52
|
+
return { ...base, ...overrides };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Populate a fresh substrate with a mix of records via S3's write path. */
|
|
56
|
+
async function seededSubstrate(): Promise<{ dir: string; writer: ContextWriter }> {
|
|
57
|
+
const dir = await makeSubstrate();
|
|
58
|
+
const writer = new ContextWriter({ localPath: dir, ref: "main" });
|
|
59
|
+
await writer.appendRecord(rec({ id: "epic-measured", scope: "epic", scopeRef: "issue-303", statement: "the measured throughput was 42 rps" }));
|
|
60
|
+
await writer.appendRecord(rec({ id: "epic-human", scope: "epic", scopeRef: "issue-303", provenance: "human", mode: "normative", authority: "authoritative", statement: "prefer explicit errors over silent fallbacks" }));
|
|
61
|
+
await writer.appendRecord(rec({ id: "epic-other-ns", scope: "epic", scopeRef: "issue-999", provenance: "instance", mode: "empirical", authority: "authoritative", statement: "instance run observed a retry storm" }));
|
|
62
|
+
await writer.appendRecord(rec({ id: "repo-norm", scope: "repo", scopeRef: "nano-ide", provenance: "human", mode: "normative", authority: "authoritative", statement: "all writes go through governance" }));
|
|
63
|
+
await writer.appendRecord(rec({ id: "corpus-fact", scope: "corpus", scopeRef: undefined, provenance: "measured", mode: "empirical", authority: "authoritative", statement: "throughput baseline across the corpus" }));
|
|
64
|
+
return { dir, writer };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
test("retrieves all records written via the S3 write path", async () => {
|
|
68
|
+
const { dir } = await seededSubstrate();
|
|
69
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
70
|
+
const all = await retriever.all();
|
|
71
|
+
assert.deepEqual(all.map((s) => s.record.id).sort(), [
|
|
72
|
+
"corpus-fact",
|
|
73
|
+
"epic-human",
|
|
74
|
+
"epic-measured",
|
|
75
|
+
"epic-other-ns",
|
|
76
|
+
"repo-norm",
|
|
77
|
+
]);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test("is correct on a COLD cache (reads through to git without warming)", async () => {
|
|
81
|
+
const { dir } = await seededSubstrate();
|
|
82
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
83
|
+
assert.equal(retriever.cache.isWarm, false);
|
|
84
|
+
|
|
85
|
+
// A path-scoped query on a cold cache reads just that partition and does NOT
|
|
86
|
+
// warm the whole index — retrieval is correct before any rebuild.
|
|
87
|
+
const ns = await retriever.byNamespace("epic", "issue-303");
|
|
88
|
+
assert.deepEqual(ns.map((s) => s.record.id).sort(), ["epic-human", "epic-measured"]);
|
|
89
|
+
assert.equal(retriever.cache.isWarm, false);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test("structured / path-scoped query by scope + namespace", async () => {
|
|
93
|
+
const { dir } = await seededSubstrate();
|
|
94
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
95
|
+
|
|
96
|
+
const oneNs = await retriever.query({ scope: "epic", scopeRef: "issue-303" });
|
|
97
|
+
assert.deepEqual(oneNs.map((s) => s.record.id).sort(), ["epic-human", "epic-measured"]);
|
|
98
|
+
assert.equal(retriever.cache.isWarm, false, "single-namespace query must not force a full build");
|
|
99
|
+
|
|
100
|
+
const wholeScope = await retriever.query({ scope: "epic" });
|
|
101
|
+
assert.deepEqual(wholeScope.map((s) => s.record.id).sort(), ["epic-human", "epic-measured", "epic-other-ns"]);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("frontmatter filtering on typed record fields (OR-sets, ANDed clauses)", async () => {
|
|
105
|
+
const { dir } = await seededSubstrate();
|
|
106
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
107
|
+
|
|
108
|
+
const humans = await retriever.query({ provenance: "human" });
|
|
109
|
+
assert.deepEqual(humans.map((s) => s.record.id).sort(), ["epic-human", "repo-norm"]);
|
|
110
|
+
|
|
111
|
+
const measuredOrInstance = await retriever.query({ provenance: ["measured", "instance"] });
|
|
112
|
+
assert.deepEqual(measuredOrInstance.map((s) => s.record.id).sort(), ["corpus-fact", "epic-measured", "epic-other-ns"]);
|
|
113
|
+
|
|
114
|
+
const normativeHumansInEpic = await retriever.query({ scope: "epic", mode: "normative", provenance: "human" });
|
|
115
|
+
assert.deepEqual(normativeHumansInEpic.map((s) => s.record.id), ["epic-human"]);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("text search across record bodies, composable with filters", async () => {
|
|
119
|
+
const { dir } = await seededSubstrate();
|
|
120
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
121
|
+
|
|
122
|
+
const throughput = await retriever.search("throughput");
|
|
123
|
+
assert.deepEqual(throughput.map((s) => s.record.id).sort(), ["corpus-fact", "epic-measured"]);
|
|
124
|
+
|
|
125
|
+
const throughputMeasuredInEpic = await retriever.search("throughput", { scope: "epic" });
|
|
126
|
+
assert.deepEqual(throughputMeasuredInEpic.map((s) => s.record.id), ["epic-measured"]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("limit caps the result set", async () => {
|
|
130
|
+
const { dir } = await seededSubstrate();
|
|
131
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
132
|
+
const limited = await retriever.query({ limit: 2 });
|
|
133
|
+
assert.equal(limited.length, 2);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test("the derived cache is disposable: invalidate/rebuild re-projects from git", async () => {
|
|
137
|
+
const { dir, writer } = await seededSubstrate();
|
|
138
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
139
|
+
|
|
140
|
+
// Warm the cache with the current store.
|
|
141
|
+
const before = await retriever.all();
|
|
142
|
+
assert.equal(retriever.cache.isWarm, true);
|
|
143
|
+
const beforeCount = before.length;
|
|
144
|
+
|
|
145
|
+
// Land a new record via the S3 write path AFTER warming.
|
|
146
|
+
await writer.appendRecord(rec({ id: "late-arrival", scope: "epic", scopeRef: "issue-303", statement: "measured tail latency dropped" }));
|
|
147
|
+
|
|
148
|
+
// The warm cache still serves the old projection (it is a snapshot, not live).
|
|
149
|
+
assert.equal((await retriever.all()).length, beforeCount);
|
|
150
|
+
|
|
151
|
+
// Invalidate → the next read re-projects from git and sees the new record.
|
|
152
|
+
retriever.invalidate();
|
|
153
|
+
assert.equal(retriever.cache.isWarm, false);
|
|
154
|
+
const after = await retriever.all();
|
|
155
|
+
assert.equal(after.length, beforeCount + 1);
|
|
156
|
+
assert.ok(after.some((s) => s.record.id === "late-arrival"));
|
|
157
|
+
|
|
158
|
+
// rebuild() is an eager equivalent.
|
|
159
|
+
await retriever.rebuild();
|
|
160
|
+
assert.equal(retriever.cache.isWarm, true);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("get() reads a single record by its S3 layout path", async () => {
|
|
164
|
+
const { dir } = await seededSubstrate();
|
|
165
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
166
|
+
const all = await retriever.all();
|
|
167
|
+
const target = all.find((s) => s.record.id === "repo-norm");
|
|
168
|
+
assert.ok(target);
|
|
169
|
+
const fetched = await retriever.get(target.path);
|
|
170
|
+
assert.equal(fetched?.record.id, "repo-norm");
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("a ratified agent-retro prior is retrievable; provenance/authority survive the round-trip", async () => {
|
|
174
|
+
const dir = await makeSubstrate();
|
|
175
|
+
const writer = new ContextWriter({ localPath: dir, ref: "main" });
|
|
176
|
+
// agent-retro cannot be appended directly — it must be proposed then ratified.
|
|
177
|
+
const proposal = await writer.proposePrior(
|
|
178
|
+
rec({ id: "hypothesis-1", scope: "epic", scopeRef: "issue-303", provenance: "agent-retro", mode: "normative", authority: "hypothesis", statement: "hypothesis: batching reduces retries" }),
|
|
179
|
+
);
|
|
180
|
+
await writer.ratify(proposal);
|
|
181
|
+
|
|
182
|
+
const retriever = new ContextRetriever({ localPath: dir, ref: "main" });
|
|
183
|
+
const priors = await retriever.query({ provenance: "agent-retro" });
|
|
184
|
+
assert.deepEqual(priors.map((s) => s.record.id), ["hypothesis-1"]);
|
|
185
|
+
// The schema invariant is preserved on read: an agent-retro prior is never authoritative.
|
|
186
|
+
assert.equal(priors[0]?.record.authority, "hypothesis");
|
|
187
|
+
});
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — the public retriever over the git system-of-record.
|
|
2
|
+
//
|
|
3
|
+
// `ContextRetriever` composes the three pieces of this slice into one API:
|
|
4
|
+
// - a RecordSource (git read seam, default GitRecordSource over a resolved
|
|
5
|
+
// substrate working copy),
|
|
6
|
+
// - a DerivedRecordIndex (disposable hot cache, rebuildable from git),
|
|
7
|
+
// - the pure query model (path-scoped + frontmatter + text search).
|
|
8
|
+
//
|
|
9
|
+
// Correctness does not depend on the cache: every query works from a cold/empty
|
|
10
|
+
// index by reading through to git, and a query narrowed to one namespace reads
|
|
11
|
+
// only that partition (a cheap, path-scoped read) without warming the whole
|
|
12
|
+
// index. Callers keep the cache honest by calling `invalidate()` after a write
|
|
13
|
+
// (git remains the source of truth); nothing here silently serves stale data
|
|
14
|
+
// beyond what the caller has chosen to keep warm.
|
|
15
|
+
|
|
16
|
+
import { DerivedRecordIndex } from "./cache.ts";
|
|
17
|
+
import type { EmbeddingIndex } from "./embedding.ts";
|
|
18
|
+
import {
|
|
19
|
+
compileQuery,
|
|
20
|
+
type CompiledQuery,
|
|
21
|
+
matchesQuery,
|
|
22
|
+
type RetrievalQuery,
|
|
23
|
+
} from "./query.ts";
|
|
24
|
+
import {
|
|
25
|
+
GitRecordSource,
|
|
26
|
+
type NamespaceSelector,
|
|
27
|
+
type RecordSource,
|
|
28
|
+
type ResolvedReadHandle,
|
|
29
|
+
type StoredRecord,
|
|
30
|
+
} from "./source.ts";
|
|
31
|
+
|
|
32
|
+
/** Options for {@link ContextRetriever}. All optional; the defaults are safe. */
|
|
33
|
+
export interface ContextRetrieverOptions {
|
|
34
|
+
/**
|
|
35
|
+
* Override the read source (seam for a future mutable/PII backend). Defaults
|
|
36
|
+
* to a {@link GitRecordSource} over the resolved handle's working copy.
|
|
37
|
+
*/
|
|
38
|
+
readonly source?: RecordSource;
|
|
39
|
+
/**
|
|
40
|
+
* A future opt-in derived embedding index. Purely a forward-compat seam — the
|
|
41
|
+
* MVP does not build or require one, and no method here depends on it.
|
|
42
|
+
*/
|
|
43
|
+
readonly embeddingIndex?: EmbeddingIndex;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Retrieval over the git-backed context store. Bind one to a resolved substrate
|
|
48
|
+
* handle (the local checkout S1 produces) and query it; the derived hot cache is
|
|
49
|
+
* built lazily and can be invalidated/rebuilt at will.
|
|
50
|
+
*/
|
|
51
|
+
export class ContextRetriever {
|
|
52
|
+
readonly #source: RecordSource;
|
|
53
|
+
readonly #index: DerivedRecordIndex;
|
|
54
|
+
readonly #embeddingIndex?: EmbeddingIndex;
|
|
55
|
+
|
|
56
|
+
constructor(
|
|
57
|
+
handle: ResolvedReadHandle | string,
|
|
58
|
+
options: ContextRetrieverOptions = {},
|
|
59
|
+
) {
|
|
60
|
+
this.#source = options.source ?? new GitRecordSource(handle);
|
|
61
|
+
this.#index = new DerivedRecordIndex(this.#source);
|
|
62
|
+
this.#embeddingIndex = options.embeddingIndex;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The derived hot cache (for explicit warm/invalidate/rebuild control). */
|
|
66
|
+
get cache(): DerivedRecordIndex {
|
|
67
|
+
return this.#index;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The read source this retriever reads git through. */
|
|
71
|
+
get source(): RecordSource {
|
|
72
|
+
return this.#source;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A future opt-in embedding index, if one was supplied. `undefined` in the
|
|
77
|
+
* MVP — retrieval is fully functional without it.
|
|
78
|
+
*/
|
|
79
|
+
get embeddingIndex(): EmbeddingIndex | undefined {
|
|
80
|
+
return this.#embeddingIndex;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Run a retrieval query. Structured/path-scoped clauses (`scope` + a single
|
|
85
|
+
* `scopeRef`) narrow the read to one layout partition when the cache is cold,
|
|
86
|
+
* so a targeted query is cheap and never forces a full index build; otherwise
|
|
87
|
+
* the frontmatter + text clauses filter the candidate set in memory. Results
|
|
88
|
+
* are ordered by layout path for determinism and capped by `limit`.
|
|
89
|
+
*/
|
|
90
|
+
async query(query: RetrievalQuery = {}): Promise<readonly StoredRecord[]> {
|
|
91
|
+
const compiled = compileQuery(query);
|
|
92
|
+
const candidates = await this.#candidates(compiled);
|
|
93
|
+
const matched = candidates.filter((stored) =>
|
|
94
|
+
matchesQuery(stored.record, compiled),
|
|
95
|
+
);
|
|
96
|
+
return applyLimit(matched, compiled);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Path-scoped structured query: every record in one namespace partition
|
|
101
|
+
* (`scope` + optional `scopeRef`), read cheaply from just that subtree when
|
|
102
|
+
* the cache is cold. Equivalent to `query({ scope, scopeRef })` but expresses
|
|
103
|
+
* the S3-partition intent directly.
|
|
104
|
+
*/
|
|
105
|
+
async byNamespace(
|
|
106
|
+
scope: NamespaceSelector["scope"],
|
|
107
|
+
scopeRef?: string,
|
|
108
|
+
): Promise<readonly StoredRecord[]> {
|
|
109
|
+
return this.#index.namespace({ scope, scopeRef });
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Free-text search across record bodies, optionally combined with structured
|
|
114
|
+
* frontmatter filters. Convenience over {@link query} for the common case.
|
|
115
|
+
*/
|
|
116
|
+
async search(
|
|
117
|
+
text: string,
|
|
118
|
+
filters: Omit<RetrievalQuery, "text"> = {},
|
|
119
|
+
): Promise<readonly StoredRecord[]> {
|
|
120
|
+
return this.query({ ...filters, text });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Every record in the store, warming the cache. */
|
|
124
|
+
async all(): Promise<readonly StoredRecord[]> {
|
|
125
|
+
return this.#index.all();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Read a single record by its S3 layout path (bypasses the cache). */
|
|
129
|
+
async get(relPath: string): Promise<StoredRecord | undefined> {
|
|
130
|
+
return this.#source.read(relPath);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Drop the derived cache so the next query re-reads from git. */
|
|
134
|
+
invalidate(): void {
|
|
135
|
+
this.#index.invalidate();
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Eagerly rebuild the derived cache from git. */
|
|
139
|
+
async rebuild(): Promise<readonly StoredRecord[]> {
|
|
140
|
+
return this.#index.rebuild();
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The candidate set to filter: when a query pins exactly one scope we read
|
|
145
|
+
* only that scope's subtree (cheap path-scoped read, cold-cache friendly) —
|
|
146
|
+
* pinning a single scopeRef narrows it further to that one namespace
|
|
147
|
+
* partition, otherwise the whole scope subtree (spanning its refs) is read;
|
|
148
|
+
* when no single scope is pinned we read the whole (cached) store.
|
|
149
|
+
*/
|
|
150
|
+
async #candidates(query: CompiledQuery): Promise<readonly StoredRecord[]> {
|
|
151
|
+
const selector = singleNamespace(query);
|
|
152
|
+
if (selector !== undefined) {
|
|
153
|
+
return this.#index.namespace(selector);
|
|
154
|
+
}
|
|
155
|
+
return this.#index.all();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* If a compiled query pins exactly ONE scope (and at most one scopeRef), return
|
|
161
|
+
* the namespace partition it lives in so retrieval can read just that subtree.
|
|
162
|
+
* Returns `undefined` when the query spans multiple scopes/refs (or none), in
|
|
163
|
+
* which case the whole store is the candidate set.
|
|
164
|
+
*/
|
|
165
|
+
function singleNamespace(query: CompiledQuery): NamespaceSelector | undefined {
|
|
166
|
+
const scope = onlyMember(query.scope);
|
|
167
|
+
if (scope === undefined) {
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
const scopeRef = onlyMember(query.scopeRef);
|
|
171
|
+
return { scope, ...(scopeRef === undefined ? {} : { scopeRef }) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** The sole member of a set of exactly one element, else `undefined`. */
|
|
175
|
+
function onlyMember<T>(set: ReadonlySet<T> | undefined): T | undefined {
|
|
176
|
+
if (set === undefined || set.size !== 1) {
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
for (const value of set) {
|
|
180
|
+
return value;
|
|
181
|
+
}
|
|
182
|
+
return undefined;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function applyLimit(
|
|
186
|
+
records: readonly StoredRecord[],
|
|
187
|
+
query: CompiledQuery,
|
|
188
|
+
): readonly StoredRecord[] {
|
|
189
|
+
if (query.limit === undefined || query.limit < 0) {
|
|
190
|
+
return records;
|
|
191
|
+
}
|
|
192
|
+
return records.slice(0, query.limit);
|
|
193
|
+
}
|