@nanobpm/urban 0.69.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/retrieval/cache.d.ts +35 -0
- package/dist/context/retrieval/cache.js +75 -0
- package/dist/context/retrieval/embedding.d.ts +27 -0
- package/dist/context/retrieval/embedding.js +13 -0
- package/dist/context/retrieval/index.d.ts +5 -1
- package/dist/context/retrieval/index.js +21 -1
- package/dist/context/retrieval/query.d.ts +50 -0
- package/dist/context/retrieval/query.js +62 -0
- package/dist/context/retrieval/retriever.d.ts +63 -0
- package/dist/context/retrieval/retriever.js +136 -0
- package/dist/context/retrieval/source.d.ts +71 -0
- package/dist/context/retrieval/source.js +141 -0
- package/package.json +2 -2
- package/src/context/retrieval/NOTES.md +68 -0
- package/src/context/retrieval/cache.ts +95 -0
- package/src/context/retrieval/embedding.ts +45 -0
- package/src/context/retrieval/index.ts +47 -2
- package/src/context/retrieval/query.test.ts +72 -0
- package/src/context/retrieval/query.ts +118 -0
- package/src/context/retrieval/retriever.test.ts +187 -0
- package/src/context/retrieval/retriever.ts +193 -0
- package/src/context/retrieval/source.test.ts +118 -0
- package/src/context/retrieval/source.ts +218 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { NamespaceSelector, RecordSource, StoredRecord } from "./source.ts";
|
|
2
|
+
/**
|
|
3
|
+
* A derived, in-memory index over a {@link RecordSource}. Holds a full snapshot
|
|
4
|
+
* of the store once warmed; cold reads and namespace-scoped reads fall back to
|
|
5
|
+
* the source directly so a cold/empty index is always correct.
|
|
6
|
+
*/
|
|
7
|
+
export declare class DerivedRecordIndex {
|
|
8
|
+
#private;
|
|
9
|
+
constructor(source: RecordSource);
|
|
10
|
+
/** `true` once a full snapshot is loaded; `false` when cold/invalidated. */
|
|
11
|
+
get isWarm(): boolean;
|
|
12
|
+
/** The read source this index is derived from (git, by default). */
|
|
13
|
+
get source(): RecordSource;
|
|
14
|
+
/**
|
|
15
|
+
* Eagerly (re)load the full store snapshot from the source, replacing any
|
|
16
|
+
* existing projection. Cheap to call and idempotent — the whole point is that
|
|
17
|
+
* rebuilding is always safe because git remains the source of truth.
|
|
18
|
+
*/
|
|
19
|
+
rebuild(): Promise<readonly StoredRecord[]>;
|
|
20
|
+
/** Drop the projection so the next read re-reads from git. */
|
|
21
|
+
invalidate(): void;
|
|
22
|
+
/**
|
|
23
|
+
* Return the full store, warming the index on a cold read. After this the
|
|
24
|
+
* index is warm; subsequent calls serve the cached snapshot without touching
|
|
25
|
+
* the filesystem until {@link invalidate} or {@link rebuild} is called.
|
|
26
|
+
*/
|
|
27
|
+
all(): Promise<readonly StoredRecord[]>;
|
|
28
|
+
/**
|
|
29
|
+
* Return the records in one namespace partition. When the index is already
|
|
30
|
+
* warm this filters the in-memory snapshot (no I/O); when cold it does a
|
|
31
|
+
* cheap, path-scoped read of JUST that partition from the source WITHOUT
|
|
32
|
+
* warming the whole index — so a targeted query never pays to load the store.
|
|
33
|
+
*/
|
|
34
|
+
namespace(selector: NamespaceSelector): Promise<readonly StoredRecord[]>;
|
|
35
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — the DERIVED hot read cache/index.
|
|
2
|
+
//
|
|
3
|
+
// Git is the system of record; this index is a disposable, rebuildable-from-git
|
|
4
|
+
// projection kept hot for fast repeated reads. It is NEVER authoritative:
|
|
5
|
+
// - a cold (never-built) index transparently reads through to the source on
|
|
6
|
+
// first use, so retrieval is correct even before any rebuild;
|
|
7
|
+
// - `invalidate()` drops the projection so the next read re-reads from git;
|
|
8
|
+
// - `rebuild()` re-reads the whole store eagerly.
|
|
9
|
+
// Because it holds nothing git does not, throwing it away can never lose data.
|
|
10
|
+
/**
|
|
11
|
+
* A derived, in-memory index over a {@link RecordSource}. Holds a full snapshot
|
|
12
|
+
* of the store once warmed; cold reads and namespace-scoped reads fall back to
|
|
13
|
+
* the source directly so a cold/empty index is always correct.
|
|
14
|
+
*/
|
|
15
|
+
export class DerivedRecordIndex {
|
|
16
|
+
#source;
|
|
17
|
+
// `undefined` == cold (never loaded / invalidated). An empty array is a valid
|
|
18
|
+
// WARM state (a store that genuinely has no records), distinct from cold.
|
|
19
|
+
#snapshot;
|
|
20
|
+
constructor(source) {
|
|
21
|
+
this.#source = source;
|
|
22
|
+
}
|
|
23
|
+
/** `true` once a full snapshot is loaded; `false` when cold/invalidated. */
|
|
24
|
+
get isWarm() {
|
|
25
|
+
return this.#snapshot !== undefined;
|
|
26
|
+
}
|
|
27
|
+
/** The read source this index is derived from (git, by default). */
|
|
28
|
+
get source() {
|
|
29
|
+
return this.#source;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Eagerly (re)load the full store snapshot from the source, replacing any
|
|
33
|
+
* existing projection. Cheap to call and idempotent — the whole point is that
|
|
34
|
+
* rebuilding is always safe because git remains the source of truth.
|
|
35
|
+
*/
|
|
36
|
+
async rebuild() {
|
|
37
|
+
const snapshot = await this.#source.list();
|
|
38
|
+
this.#snapshot = snapshot;
|
|
39
|
+
return snapshot;
|
|
40
|
+
}
|
|
41
|
+
/** Drop the projection so the next read re-reads from git. */
|
|
42
|
+
invalidate() {
|
|
43
|
+
this.#snapshot = undefined;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Return the full store, warming the index on a cold read. After this the
|
|
47
|
+
* index is warm; subsequent calls serve the cached snapshot without touching
|
|
48
|
+
* the filesystem until {@link invalidate} or {@link rebuild} is called.
|
|
49
|
+
*/
|
|
50
|
+
async all() {
|
|
51
|
+
if (this.#snapshot !== undefined) {
|
|
52
|
+
return this.#snapshot;
|
|
53
|
+
}
|
|
54
|
+
return this.rebuild();
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Return the records in one namespace partition. When the index is already
|
|
58
|
+
* warm this filters the in-memory snapshot (no I/O); when cold it does a
|
|
59
|
+
* cheap, path-scoped read of JUST that partition from the source WITHOUT
|
|
60
|
+
* warming the whole index — so a targeted query never pays to load the store.
|
|
61
|
+
*/
|
|
62
|
+
async namespace(selector) {
|
|
63
|
+
if (this.#snapshot !== undefined) {
|
|
64
|
+
return this.#snapshot.filter((stored) => inNamespace(stored, selector));
|
|
65
|
+
}
|
|
66
|
+
return this.#source.list(selector);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** `true` iff a stored record belongs to the given namespace partition. */
|
|
70
|
+
function inNamespace(stored, selector) {
|
|
71
|
+
if (stored.record.scope !== selector.scope) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
return (selector.scopeRef === undefined || stored.record.scopeRef === selector.scopeRef);
|
|
75
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { RetrievalQuery } from "./query.ts";
|
|
2
|
+
import type { StoredRecord } from "./source.ts";
|
|
3
|
+
/** A semantic match: a stored record plus the backend's similarity score. */
|
|
4
|
+
export interface SemanticMatch {
|
|
5
|
+
readonly stored: StoredRecord;
|
|
6
|
+
/** Backend-defined similarity score; higher is more similar. */
|
|
7
|
+
readonly score: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* The seam a future opt-in embedding index would implement. A retriever could
|
|
11
|
+
* accept one and offer `semanticQuery`, but the MVP neither ships nor requires
|
|
12
|
+
* an implementation — retrieval is fully correct git-only without it.
|
|
13
|
+
*
|
|
14
|
+
* Implementations must treat the embedding store as a derived projection of git:
|
|
15
|
+
* {@link reindex} rebuilds it from the authoritative records, and a cold/missing
|
|
16
|
+
* embedding index must degrade to the git-backed lexical path, never to wrong
|
|
17
|
+
* or stale results.
|
|
18
|
+
*/
|
|
19
|
+
export interface EmbeddingIndex {
|
|
20
|
+
/** (Re)build the derived vector index from the authoritative git records. */
|
|
21
|
+
reindex(records: readonly StoredRecord[]): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Return the semantically nearest records to `query.text`, honouring the
|
|
24
|
+
* structured/frontmatter clauses of `query` as pre/post filters.
|
|
25
|
+
*/
|
|
26
|
+
semanticSearch(query: RetrievalQuery, k: number): Promise<readonly SemanticMatch[]>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — SEAM ONLY for a future opt-in derived embedding index.
|
|
2
|
+
//
|
|
3
|
+
// The MVP retrieval is git-only: structured/path-scoped queries, frontmatter
|
|
4
|
+
// filtering, and substring text search over the git system-of-record, with a
|
|
5
|
+
// disposable hot cache. Semantic (vector) retrieval — an OpenMemory / mem0-style
|
|
6
|
+
// derived embedding index — is deliberately NOT built here. This file only
|
|
7
|
+
// declares the seam it would plug into, so a later slice can add it without
|
|
8
|
+
// reshaping the retriever or its public API.
|
|
9
|
+
//
|
|
10
|
+
// Like the hot cache, any such embedding index MUST be DERIVED and disposable:
|
|
11
|
+
// rebuildable purely from git, never authoritative. Nothing in this file
|
|
12
|
+
// performs embedding, network I/O, or persistence — it is a type contract.
|
|
13
|
+
export {};
|
|
@@ -1 +1,5 @@
|
|
|
1
|
-
export {};
|
|
1
|
+
export { ContextRetriever, type ContextRetrieverOptions, } from "./retriever.ts";
|
|
2
|
+
export { DerivedRecordIndex } from "./cache.ts";
|
|
3
|
+
export { GitRecordSource, type NamespaceSelector, type RecordSource, RecordSourceError, type ResolvedReadHandle, selectorSubtree, type StoredRecord, } from "./source.ts";
|
|
4
|
+
export { compileQuery, type CompiledQuery, matchesQuery, type OneOrMany, type RetrievalQuery, } from "./query.ts";
|
|
5
|
+
export { type EmbeddingIndex, type SemanticMatch, } from "./embedding.ts";
|
|
@@ -1 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
// @nanobpm/urban/context/retrieval — slice S4 (retrieval over git).
|
|
2
|
+
//
|
|
3
|
+
// Read-side of the Urban context layer: query the git system-of-record that S3
|
|
4
|
+
// writes, using the exact path/namespace layout S3 publishes. MVP is git-only —
|
|
5
|
+
// NO external service:
|
|
6
|
+
//
|
|
7
|
+
// 1. STRUCTURED / PATH-SCOPED queries by scope/namespace, reading just one
|
|
8
|
+
// layout partition when a query pins it (cheap, cold-cache friendly).
|
|
9
|
+
// 2. FRONTMATTER filtering on typed record fields (scope/mode/provenance/
|
|
10
|
+
// authority/id/subject), ANDed clauses with OR-sets per field.
|
|
11
|
+
// 3. TEXT SEARCH across record bodies (case-insensitive substring).
|
|
12
|
+
//
|
|
13
|
+
// Git is the system of record; the hot read cache (DerivedRecordIndex) is a
|
|
14
|
+
// DISPOSABLE projection, rebuildable purely from git and never authoritative —
|
|
15
|
+
// retrieval is correct even with a cold/empty cache by falling back to the
|
|
16
|
+
// source. A future opt-in derived embedding index is left as a documented seam
|
|
17
|
+
// (see embedding.ts / EmbeddingIndex) and is NOT built here.
|
|
18
|
+
export { ContextRetriever, } from "./retriever.js";
|
|
19
|
+
export { DerivedRecordIndex } from "./cache.js";
|
|
20
|
+
export { GitRecordSource, RecordSourceError, selectorSubtree, } from "./source.js";
|
|
21
|
+
export { compileQuery, matchesQuery, } from "./query.js";
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { MemoryAuthority, MemoryMode, MemoryProvenance, MemoryRecord, MemoryScope } from "../schema/index.ts";
|
|
2
|
+
/** A field value that may be given as a single value or an OR-set of values. */
|
|
3
|
+
export type OneOrMany<T> = T | readonly T[];
|
|
4
|
+
/**
|
|
5
|
+
* A retrieval query. All clauses are ANDed together; a clause given as an array
|
|
6
|
+
* is an OR over its members. Absent clauses impose no constraint.
|
|
7
|
+
*/
|
|
8
|
+
export interface RetrievalQuery {
|
|
9
|
+
/** Structured / path-scoped: restrict to one or more scope ladder levels. */
|
|
10
|
+
readonly scope?: OneOrMany<MemoryScope>;
|
|
11
|
+
/** Structured / path-scoped: restrict to one or more `scopeRef` namespaces. */
|
|
12
|
+
readonly scopeRef?: OneOrMany<string>;
|
|
13
|
+
/** Frontmatter filter: normative vs empirical. */
|
|
14
|
+
readonly mode?: OneOrMany<MemoryMode>;
|
|
15
|
+
/** Frontmatter filter: where the record came from. */
|
|
16
|
+
readonly provenance?: OneOrMany<MemoryProvenance>;
|
|
17
|
+
/** Frontmatter filter: hypothesis vs authoritative. */
|
|
18
|
+
readonly authority?: OneOrMany<MemoryAuthority>;
|
|
19
|
+
/** Frontmatter filter: exact record id(s). */
|
|
20
|
+
readonly id?: OneOrMany<string>;
|
|
21
|
+
/** Frontmatter filter: exact subject(s). */
|
|
22
|
+
readonly subject?: OneOrMany<string>;
|
|
23
|
+
/**
|
|
24
|
+
* Free-text search across the record body — case-insensitive substring match
|
|
25
|
+
* over `statement`, `subject` (when present), and `id`. This is the "text
|
|
26
|
+
* search across record bodies" clause; it composes with the structured filters.
|
|
27
|
+
*/
|
|
28
|
+
readonly text?: string;
|
|
29
|
+
/** Cap the number of results returned (applied after ordering). */
|
|
30
|
+
readonly limit?: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* A query compiled to sets for O(1) membership tests. Building it once and
|
|
34
|
+
* reusing it across a candidate list avoids re-normalising the query per record.
|
|
35
|
+
*/
|
|
36
|
+
export interface CompiledQuery {
|
|
37
|
+
readonly scope?: ReadonlySet<MemoryScope>;
|
|
38
|
+
readonly scopeRef?: ReadonlySet<string>;
|
|
39
|
+
readonly mode?: ReadonlySet<MemoryMode>;
|
|
40
|
+
readonly provenance?: ReadonlySet<MemoryProvenance>;
|
|
41
|
+
readonly authority?: ReadonlySet<MemoryAuthority>;
|
|
42
|
+
readonly id?: ReadonlySet<string>;
|
|
43
|
+
readonly subject?: ReadonlySet<string>;
|
|
44
|
+
readonly text?: string;
|
|
45
|
+
readonly limit?: number;
|
|
46
|
+
}
|
|
47
|
+
/** Compile a raw query into membership sets + a normalised (lowercased) needle. */
|
|
48
|
+
export declare function compileQuery(query: RetrievalQuery): CompiledQuery;
|
|
49
|
+
/** `true` iff `record` satisfies every clause of the compiled query. */
|
|
50
|
+
export declare function matchesQuery(record: MemoryRecord, query: CompiledQuery): boolean;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — the query model: frontmatter filtering + text search.
|
|
2
|
+
//
|
|
3
|
+
// A retrieval query is a conjunction of independent predicates: path-scoped
|
|
4
|
+
// namespace narrowing (scope/scopeRef), frontmatter filtering on the typed
|
|
5
|
+
// record fields (mode/provenance/authority/…), and a free-text search across
|
|
6
|
+
// the record body. Every clause is optional; an empty query matches everything.
|
|
7
|
+
// The matching here is pure and cache-agnostic — it runs identically over
|
|
8
|
+
// records read straight from git (cold cache) or served from the hot index.
|
|
9
|
+
function toSet(value) {
|
|
10
|
+
if (value === undefined) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
return new Set(Array.isArray(value) ? value : [value]);
|
|
14
|
+
}
|
|
15
|
+
function matchesOneOf(allowed, actual) {
|
|
16
|
+
if (allowed === undefined) {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
return actual !== undefined && allowed.has(actual);
|
|
20
|
+
}
|
|
21
|
+
/** Compile a raw query into membership sets + a normalised (lowercased) needle. */
|
|
22
|
+
export function compileQuery(query) {
|
|
23
|
+
return {
|
|
24
|
+
scope: toSet(query.scope),
|
|
25
|
+
scopeRef: toSet(query.scopeRef),
|
|
26
|
+
mode: toSet(query.mode),
|
|
27
|
+
provenance: toSet(query.provenance),
|
|
28
|
+
authority: toSet(query.authority),
|
|
29
|
+
id: toSet(query.id),
|
|
30
|
+
subject: toSet(query.subject),
|
|
31
|
+
text: query.text === undefined ? undefined : query.text.toLowerCase(),
|
|
32
|
+
limit: query.limit,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** `true` iff `record` satisfies every clause of the compiled query. */
|
|
36
|
+
export function matchesQuery(record, query) {
|
|
37
|
+
if (!matchesOneOf(query.scope, record.scope))
|
|
38
|
+
return false;
|
|
39
|
+
if (!matchesOneOf(query.scopeRef, record.scopeRef))
|
|
40
|
+
return false;
|
|
41
|
+
if (!matchesOneOf(query.mode, record.mode))
|
|
42
|
+
return false;
|
|
43
|
+
if (!matchesOneOf(query.provenance, record.provenance))
|
|
44
|
+
return false;
|
|
45
|
+
if (!matchesOneOf(query.authority, record.authority))
|
|
46
|
+
return false;
|
|
47
|
+
if (!matchesOneOf(query.id, record.id))
|
|
48
|
+
return false;
|
|
49
|
+
if (!matchesOneOf(query.subject, record.subject))
|
|
50
|
+
return false;
|
|
51
|
+
if (query.text !== undefined && !matchesText(record, query.text))
|
|
52
|
+
return false;
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
/** Case-insensitive substring search across the record's textual body. */
|
|
56
|
+
function matchesText(record, needle) {
|
|
57
|
+
if (needle === "") {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
const haystacks = [record.statement, record.subject ?? "", record.id];
|
|
61
|
+
return haystacks.some((field) => field.toLowerCase().includes(needle));
|
|
62
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { DerivedRecordIndex } from "./cache.ts";
|
|
2
|
+
import type { EmbeddingIndex } from "./embedding.ts";
|
|
3
|
+
import { type RetrievalQuery } from "./query.ts";
|
|
4
|
+
import { type NamespaceSelector, type RecordSource, type ResolvedReadHandle, type StoredRecord } from "./source.ts";
|
|
5
|
+
/** Options for {@link ContextRetriever}. All optional; the defaults are safe. */
|
|
6
|
+
export interface ContextRetrieverOptions {
|
|
7
|
+
/**
|
|
8
|
+
* Override the read source (seam for a future mutable/PII backend). Defaults
|
|
9
|
+
* to a {@link GitRecordSource} over the resolved handle's working copy.
|
|
10
|
+
*/
|
|
11
|
+
readonly source?: RecordSource;
|
|
12
|
+
/**
|
|
13
|
+
* A future opt-in derived embedding index. Purely a forward-compat seam — the
|
|
14
|
+
* MVP does not build or require one, and no method here depends on it.
|
|
15
|
+
*/
|
|
16
|
+
readonly embeddingIndex?: EmbeddingIndex;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Retrieval over the git-backed context store. Bind one to a resolved substrate
|
|
20
|
+
* handle (the local checkout S1 produces) and query it; the derived hot cache is
|
|
21
|
+
* built lazily and can be invalidated/rebuilt at will.
|
|
22
|
+
*/
|
|
23
|
+
export declare class ContextRetriever {
|
|
24
|
+
#private;
|
|
25
|
+
constructor(handle: ResolvedReadHandle | string, options?: ContextRetrieverOptions);
|
|
26
|
+
/** The derived hot cache (for explicit warm/invalidate/rebuild control). */
|
|
27
|
+
get cache(): DerivedRecordIndex;
|
|
28
|
+
/** The read source this retriever reads git through. */
|
|
29
|
+
get source(): RecordSource;
|
|
30
|
+
/**
|
|
31
|
+
* A future opt-in embedding index, if one was supplied. `undefined` in the
|
|
32
|
+
* MVP — retrieval is fully functional without it.
|
|
33
|
+
*/
|
|
34
|
+
get embeddingIndex(): EmbeddingIndex | undefined;
|
|
35
|
+
/**
|
|
36
|
+
* Run a retrieval query. Structured/path-scoped clauses (`scope` + a single
|
|
37
|
+
* `scopeRef`) narrow the read to one layout partition when the cache is cold,
|
|
38
|
+
* so a targeted query is cheap and never forces a full index build; otherwise
|
|
39
|
+
* the frontmatter + text clauses filter the candidate set in memory. Results
|
|
40
|
+
* are ordered by layout path for determinism and capped by `limit`.
|
|
41
|
+
*/
|
|
42
|
+
query(query?: RetrievalQuery): Promise<readonly StoredRecord[]>;
|
|
43
|
+
/**
|
|
44
|
+
* Path-scoped structured query: every record in one namespace partition
|
|
45
|
+
* (`scope` + optional `scopeRef`), read cheaply from just that subtree when
|
|
46
|
+
* the cache is cold. Equivalent to `query({ scope, scopeRef })` but expresses
|
|
47
|
+
* the S3-partition intent directly.
|
|
48
|
+
*/
|
|
49
|
+
byNamespace(scope: NamespaceSelector["scope"], scopeRef?: string): Promise<readonly StoredRecord[]>;
|
|
50
|
+
/**
|
|
51
|
+
* Free-text search across record bodies, optionally combined with structured
|
|
52
|
+
* frontmatter filters. Convenience over {@link query} for the common case.
|
|
53
|
+
*/
|
|
54
|
+
search(text: string, filters?: Omit<RetrievalQuery, "text">): Promise<readonly StoredRecord[]>;
|
|
55
|
+
/** Every record in the store, warming the cache. */
|
|
56
|
+
all(): Promise<readonly StoredRecord[]>;
|
|
57
|
+
/** Read a single record by its S3 layout path (bypasses the cache). */
|
|
58
|
+
get(relPath: string): Promise<StoredRecord | undefined>;
|
|
59
|
+
/** Drop the derived cache so the next query re-reads from git. */
|
|
60
|
+
invalidate(): void;
|
|
61
|
+
/** Eagerly rebuild the derived cache from git. */
|
|
62
|
+
rebuild(): Promise<readonly StoredRecord[]>;
|
|
63
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Slice S4 (retrieval) — the public retriever over the git system-of-record.
|
|
2
|
+
//
|
|
3
|
+
// `ContextRetriever` composes the three pieces of this slice into one API:
|
|
4
|
+
// - a RecordSource (git read seam, default GitRecordSource over a resolved
|
|
5
|
+
// substrate working copy),
|
|
6
|
+
// - a DerivedRecordIndex (disposable hot cache, rebuildable from git),
|
|
7
|
+
// - the pure query model (path-scoped + frontmatter + text search).
|
|
8
|
+
//
|
|
9
|
+
// Correctness does not depend on the cache: every query works from a cold/empty
|
|
10
|
+
// index by reading through to git, and a query narrowed to one namespace reads
|
|
11
|
+
// only that partition (a cheap, path-scoped read) without warming the whole
|
|
12
|
+
// index. Callers keep the cache honest by calling `invalidate()` after a write
|
|
13
|
+
// (git remains the source of truth); nothing here silently serves stale data
|
|
14
|
+
// beyond what the caller has chosen to keep warm.
|
|
15
|
+
import { DerivedRecordIndex } from "./cache.js";
|
|
16
|
+
import { compileQuery, matchesQuery, } from "./query.js";
|
|
17
|
+
import { GitRecordSource, } from "./source.js";
|
|
18
|
+
/**
|
|
19
|
+
* Retrieval over the git-backed context store. Bind one to a resolved substrate
|
|
20
|
+
* handle (the local checkout S1 produces) and query it; the derived hot cache is
|
|
21
|
+
* built lazily and can be invalidated/rebuilt at will.
|
|
22
|
+
*/
|
|
23
|
+
export class ContextRetriever {
|
|
24
|
+
#source;
|
|
25
|
+
#index;
|
|
26
|
+
#embeddingIndex;
|
|
27
|
+
constructor(handle, options = {}) {
|
|
28
|
+
this.#source = options.source ?? new GitRecordSource(handle);
|
|
29
|
+
this.#index = new DerivedRecordIndex(this.#source);
|
|
30
|
+
this.#embeddingIndex = options.embeddingIndex;
|
|
31
|
+
}
|
|
32
|
+
/** The derived hot cache (for explicit warm/invalidate/rebuild control). */
|
|
33
|
+
get cache() {
|
|
34
|
+
return this.#index;
|
|
35
|
+
}
|
|
36
|
+
/** The read source this retriever reads git through. */
|
|
37
|
+
get source() {
|
|
38
|
+
return this.#source;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* A future opt-in embedding index, if one was supplied. `undefined` in the
|
|
42
|
+
* MVP — retrieval is fully functional without it.
|
|
43
|
+
*/
|
|
44
|
+
get embeddingIndex() {
|
|
45
|
+
return this.#embeddingIndex;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Run a retrieval query. Structured/path-scoped clauses (`scope` + a single
|
|
49
|
+
* `scopeRef`) narrow the read to one layout partition when the cache is cold,
|
|
50
|
+
* so a targeted query is cheap and never forces a full index build; otherwise
|
|
51
|
+
* the frontmatter + text clauses filter the candidate set in memory. Results
|
|
52
|
+
* are ordered by layout path for determinism and capped by `limit`.
|
|
53
|
+
*/
|
|
54
|
+
async query(query = {}) {
|
|
55
|
+
const compiled = compileQuery(query);
|
|
56
|
+
const candidates = await this.#candidates(compiled);
|
|
57
|
+
const matched = candidates.filter((stored) => matchesQuery(stored.record, compiled));
|
|
58
|
+
return applyLimit(matched, compiled);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Path-scoped structured query: every record in one namespace partition
|
|
62
|
+
* (`scope` + optional `scopeRef`), read cheaply from just that subtree when
|
|
63
|
+
* the cache is cold. Equivalent to `query({ scope, scopeRef })` but expresses
|
|
64
|
+
* the S3-partition intent directly.
|
|
65
|
+
*/
|
|
66
|
+
async byNamespace(scope, scopeRef) {
|
|
67
|
+
return this.#index.namespace({ scope, scopeRef });
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Free-text search across record bodies, optionally combined with structured
|
|
71
|
+
* frontmatter filters. Convenience over {@link query} for the common case.
|
|
72
|
+
*/
|
|
73
|
+
async search(text, filters = {}) {
|
|
74
|
+
return this.query({ ...filters, text });
|
|
75
|
+
}
|
|
76
|
+
/** Every record in the store, warming the cache. */
|
|
77
|
+
async all() {
|
|
78
|
+
return this.#index.all();
|
|
79
|
+
}
|
|
80
|
+
/** Read a single record by its S3 layout path (bypasses the cache). */
|
|
81
|
+
async get(relPath) {
|
|
82
|
+
return this.#source.read(relPath);
|
|
83
|
+
}
|
|
84
|
+
/** Drop the derived cache so the next query re-reads from git. */
|
|
85
|
+
invalidate() {
|
|
86
|
+
this.#index.invalidate();
|
|
87
|
+
}
|
|
88
|
+
/** Eagerly rebuild the derived cache from git. */
|
|
89
|
+
async rebuild() {
|
|
90
|
+
return this.#index.rebuild();
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The candidate set to filter: when a query pins exactly one scope we read
|
|
94
|
+
* only that scope's subtree (cheap path-scoped read, cold-cache friendly) —
|
|
95
|
+
* pinning a single scopeRef narrows it further to that one namespace
|
|
96
|
+
* partition, otherwise the whole scope subtree (spanning its refs) is read;
|
|
97
|
+
* when no single scope is pinned we read the whole (cached) store.
|
|
98
|
+
*/
|
|
99
|
+
async #candidates(query) {
|
|
100
|
+
const selector = singleNamespace(query);
|
|
101
|
+
if (selector !== undefined) {
|
|
102
|
+
return this.#index.namespace(selector);
|
|
103
|
+
}
|
|
104
|
+
return this.#index.all();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* If a compiled query pins exactly ONE scope (and at most one scopeRef), return
|
|
109
|
+
* the namespace partition it lives in so retrieval can read just that subtree.
|
|
110
|
+
* Returns `undefined` when the query spans multiple scopes/refs (or none), in
|
|
111
|
+
* which case the whole store is the candidate set.
|
|
112
|
+
*/
|
|
113
|
+
function singleNamespace(query) {
|
|
114
|
+
const scope = onlyMember(query.scope);
|
|
115
|
+
if (scope === undefined) {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
const scopeRef = onlyMember(query.scopeRef);
|
|
119
|
+
return { scope, ...(scopeRef === undefined ? {} : { scopeRef }) };
|
|
120
|
+
}
|
|
121
|
+
/** The sole member of a set of exactly one element, else `undefined`. */
|
|
122
|
+
function onlyMember(set) {
|
|
123
|
+
if (set === undefined || set.size !== 1) {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
for (const value of set) {
|
|
127
|
+
return value;
|
|
128
|
+
}
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
function applyLimit(records, query) {
|
|
132
|
+
if (query.limit === undefined || query.limit < 0) {
|
|
133
|
+
return records;
|
|
134
|
+
}
|
|
135
|
+
return records.slice(0, query.limit);
|
|
136
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { type MemoryRecord, type MemoryScope } from "../schema/index.ts";
|
|
2
|
+
/** A record as it is stored in the substrate: its layout path + the record. */
|
|
3
|
+
export interface StoredRecord {
|
|
4
|
+
/** Substrate-root-relative POSIX path the record lives at (S3 layout). */
|
|
5
|
+
readonly path: string;
|
|
6
|
+
/** The validated memory record parsed from that file. */
|
|
7
|
+
readonly record: MemoryRecord;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* A path-scoped selector for a namespace partition in the S3 layout. Reading a
|
|
11
|
+
* single `(scope, scopeRef)` — or a whole `scope` — is a cheap structured query
|
|
12
|
+
* because the layout partitions records into one directory per namespace, so the
|
|
13
|
+
* source can walk just that subtree instead of the whole store.
|
|
14
|
+
*/
|
|
15
|
+
export interface NamespaceSelector {
|
|
16
|
+
readonly scope: MemoryScope;
|
|
17
|
+
/** When present, narrows to one namespace bucket; absent lists the scope. */
|
|
18
|
+
readonly scopeRef?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The read seam retrieval is built on. A future mutable/PII backend can
|
|
22
|
+
* implement this differently (e.g. against a database) without the cache, the
|
|
23
|
+
* query layer, or callers changing.
|
|
24
|
+
*/
|
|
25
|
+
export interface RecordSource {
|
|
26
|
+
/**
|
|
27
|
+
* List every stored record, optionally narrowed to one namespace partition
|
|
28
|
+
* (a cheap, path-scoped read that walks only that subtree of the S3 layout).
|
|
29
|
+
* Invalid / unparseable files under the layout root are skipped, not fatal —
|
|
30
|
+
* a single corrupt file must never make the whole store unreadable.
|
|
31
|
+
*/
|
|
32
|
+
list(selector?: NamespaceSelector): Promise<readonly StoredRecord[]>;
|
|
33
|
+
/**
|
|
34
|
+
* Read a single record by its substrate-relative layout path, or `undefined`
|
|
35
|
+
* when it is absent or not a valid record file.
|
|
36
|
+
*/
|
|
37
|
+
read(relPath: string): Promise<StoredRecord | undefined>;
|
|
38
|
+
}
|
|
39
|
+
/** The minimal resolved-handle shape the read source consumes (from S1). */
|
|
40
|
+
export interface ResolvedReadHandle {
|
|
41
|
+
/** Absolute path to the local substrate working copy (checked out at `ref`). */
|
|
42
|
+
readonly localPath: string;
|
|
43
|
+
/** The git ref the working copy is pinned to (informational for reads). */
|
|
44
|
+
readonly ref?: string;
|
|
45
|
+
}
|
|
46
|
+
/** Thrown when the underlying substrate cannot be read at all. */
|
|
47
|
+
export declare class RecordSourceError extends Error {
|
|
48
|
+
constructor(message: string, options?: {
|
|
49
|
+
cause?: unknown;
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The default git-backed read source: reads records straight off a local
|
|
54
|
+
* substrate working copy. Git is the system of record; this holds no state of
|
|
55
|
+
* its own, so every call reflects the current on-disk (== checked-out ref)
|
|
56
|
+
* content. That is what makes the derived index safe to throw away and rebuild.
|
|
57
|
+
*/
|
|
58
|
+
export declare class GitRecordSource implements RecordSource {
|
|
59
|
+
#private;
|
|
60
|
+
constructor(handle: ResolvedReadHandle | string);
|
|
61
|
+
/** Absolute path of the substrate working copy this source reads from. */
|
|
62
|
+
get root(): string;
|
|
63
|
+
list(selector?: NamespaceSelector): Promise<readonly StoredRecord[]>;
|
|
64
|
+
read(relPath: string): Promise<StoredRecord | undefined>;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* The layout subtree to walk for a selector: one namespace bucket, one whole
|
|
68
|
+
* scope, or the entire layout root. Keeps path-scoping in one place so the
|
|
69
|
+
* cache and query layers never re-derive layout paths themselves.
|
|
70
|
+
*/
|
|
71
|
+
export declare function selectorSubtree(selector?: NamespaceSelector): string;
|