@fgv/ts-agent-memory 5.1.0-49 → 5.1.0-50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/index/memoryIndex.js +23 -16
- package/dist/packlets/index/memoryIndex.js.map +1 -1
- package/dist/packlets/ingest/orchestrator.js +13 -1
- package/dist/packlets/ingest/orchestrator.js.map +1 -1
- package/dist/packlets/retrieve/linkTraversalRetriever.js +12 -26
- package/dist/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/dist/packlets/retrieve/recencyRetriever.js +7 -7
- package/dist/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/dist/packlets/retrieve/retriever.js +91 -10
- package/dist/packlets/retrieve/retriever.js.map +1 -1
- package/dist/packlets/retrieve/semanticRetriever.js +16 -16
- package/dist/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/dist/packlets/retrieve/structuredFilterRetriever.js +7 -7
- package/dist/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/dist/packlets/retrieve/tagRetriever.js +7 -7
- package/dist/packlets/retrieve/tagRetriever.js.map +1 -1
- package/dist/packlets/retrieve/temporalRetrievers.js +23 -20
- package/dist/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/dist/packlets/store/coverage.js +6 -0
- package/dist/packlets/store/coverage.js.map +1 -0
- package/dist/packlets/store/fileTreeMemoryStore.js +221 -79
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/store/index.js +4 -0
- package/dist/packlets/store/index.js.map +1 -1
- package/dist/packlets/store/listSelection.js +36 -0
- package/dist/packlets/store/listSelection.js.map +1 -0
- package/dist/packlets/store/memoryStore.js +6 -0
- package/dist/packlets/store/memoryStore.js.map +1 -0
- package/dist/packlets/store/reconcile.js +6 -0
- package/dist/packlets/store/reconcile.js.map +1 -0
- package/dist/packlets/store/storeCoverage.js +102 -0
- package/dist/packlets/store/storeCoverage.js.map +1 -0
- package/dist/packlets/store/storeReconcile.js +122 -0
- package/dist/packlets/store/storeReconcile.js.map +1 -0
- package/dist/packlets/store/vectorMaintenance.js +116 -8
- package/dist/packlets/store/vectorMaintenance.js.map +1 -1
- package/dist/packlets/store/vectorRecordSource.js +44 -0
- package/dist/packlets/store/vectorRecordSource.js.map +1 -0
- package/dist/packlets/tools/memoryTools.js +25 -2
- package/dist/packlets/tools/memoryTools.js.map +1 -1
- package/dist/packlets/types/envelope.js +25 -0
- package/dist/packlets/types/envelope.js.map +1 -1
- package/dist/packlets/types/index.js +1 -0
- package/dist/packlets/types/index.js.map +1 -1
- package/dist/packlets/types/recordResolver.js +6 -0
- package/dist/packlets/types/recordResolver.js.map +1 -0
- package/dist/packlets/types/temporal.js.map +1 -1
- package/dist/packlets/vector/inMemoryCosineIndex.js +39 -18
- package/dist/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +67 -12
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/dist/packlets/vector/rebuildHelpers.js +38 -0
- package/dist/packlets/vector/rebuildHelpers.js.map +1 -0
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/ts-agent-memory.d.ts +1035 -106
- package/lib/packlets/index/memoryIndex.d.ts +118 -27
- package/lib/packlets/index/memoryIndex.d.ts.map +1 -1
- package/lib/packlets/index/memoryIndex.js +23 -16
- package/lib/packlets/index/memoryIndex.js.map +1 -1
- package/lib/packlets/ingest/orchestrator.d.ts.map +1 -1
- package/lib/packlets/ingest/orchestrator.js +13 -1
- package/lib/packlets/ingest/orchestrator.js.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts +3 -10
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.js +11 -25
- package/lib/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/recencyRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.js +6 -6
- package/lib/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/lib/packlets/retrieve/retriever.d.ts +88 -7
- package/lib/packlets/retrieve/retriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/retriever.js +94 -9
- package/lib/packlets/retrieve/retriever.js.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.d.ts +3 -5
- package/lib/packlets/retrieve/semanticRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.js +15 -15
- package/lib/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.js +6 -6
- package/lib/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/tagRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.js +6 -6
- package/lib/packlets/retrieve/tagRetriever.js.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.d.ts +7 -5
- package/lib/packlets/retrieve/temporalRetrievers.d.ts.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.js +22 -19
- package/lib/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/lib/packlets/store/coverage.d.ts +102 -0
- package/lib/packlets/store/coverage.d.ts.map +1 -0
- package/lib/packlets/store/coverage.js +7 -0
- package/lib/packlets/store/coverage.js.map +1 -0
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +53 -166
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +221 -79
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/store/index.d.ts +4 -0
- package/lib/packlets/store/index.d.ts.map +1 -1
- package/lib/packlets/store/index.js +4 -0
- package/lib/packlets/store/index.js.map +1 -1
- package/lib/packlets/store/listSelection.d.ts +101 -0
- package/lib/packlets/store/listSelection.d.ts.map +1 -0
- package/lib/packlets/store/listSelection.js +40 -0
- package/lib/packlets/store/listSelection.js.map +1 -0
- package/lib/packlets/store/memoryStore.d.ts +237 -0
- package/lib/packlets/store/memoryStore.d.ts.map +1 -0
- package/lib/packlets/store/memoryStore.js +7 -0
- package/lib/packlets/store/memoryStore.js.map +1 -0
- package/lib/packlets/store/reconcile.d.ts +82 -0
- package/lib/packlets/store/reconcile.d.ts.map +1 -0
- package/lib/packlets/store/reconcile.js +7 -0
- package/lib/packlets/store/reconcile.js.map +1 -0
- package/lib/packlets/store/storeCoverage.d.ts +45 -0
- package/lib/packlets/store/storeCoverage.d.ts.map +1 -0
- package/lib/packlets/store/storeCoverage.js +105 -0
- package/lib/packlets/store/storeCoverage.js.map +1 -0
- package/lib/packlets/store/storeReconcile.d.ts +41 -0
- package/lib/packlets/store/storeReconcile.d.ts.map +1 -0
- package/lib/packlets/store/storeReconcile.js +125 -0
- package/lib/packlets/store/storeReconcile.js.map +1 -0
- package/lib/packlets/store/vectorMaintenance.d.ts +74 -0
- package/lib/packlets/store/vectorMaintenance.d.ts.map +1 -1
- package/lib/packlets/store/vectorMaintenance.js +117 -8
- package/lib/packlets/store/vectorMaintenance.js.map +1 -1
- package/lib/packlets/store/vectorRecordSource.d.ts +36 -0
- package/lib/packlets/store/vectorRecordSource.d.ts.map +1 -0
- package/lib/packlets/store/vectorRecordSource.js +47 -0
- package/lib/packlets/store/vectorRecordSource.js.map +1 -0
- package/lib/packlets/tools/memoryTools.d.ts.map +1 -1
- package/lib/packlets/tools/memoryTools.js +25 -2
- package/lib/packlets/tools/memoryTools.js.map +1 -1
- package/lib/packlets/types/envelope.d.ts +24 -2
- package/lib/packlets/types/envelope.d.ts.map +1 -1
- package/lib/packlets/types/envelope.js +26 -0
- package/lib/packlets/types/envelope.js.map +1 -1
- package/lib/packlets/types/index.d.ts +1 -0
- package/lib/packlets/types/index.d.ts.map +1 -1
- package/lib/packlets/types/index.js +1 -0
- package/lib/packlets/types/index.js.map +1 -1
- package/lib/packlets/types/recordResolver.d.ts +39 -0
- package/lib/packlets/types/recordResolver.d.ts.map +1 -0
- package/lib/packlets/types/recordResolver.js +7 -0
- package/lib/packlets/types/recordResolver.js.map +1 -0
- package/lib/packlets/types/temporal.d.ts +26 -6
- package/lib/packlets/types/temporal.d.ts.map +1 -1
- package/lib/packlets/types/temporal.js.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts +9 -2
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.js +40 -19
- package/lib/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +6 -3
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +66 -11
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/lib/packlets/vector/rebuildHelpers.d.ts +30 -0
- package/lib/packlets/vector/rebuildHelpers.d.ts.map +1 -0
- package/lib/packlets/vector/rebuildHelpers.js +42 -0
- package/lib/packlets/vector/rebuildHelpers.js.map +1 -0
- package/lib/packlets/vector/vectorIndex.d.ts +270 -15
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
* Copyright (c) 2026 Erik Fortune
|
|
3
3
|
* SPDX-License-Identifier: MIT
|
|
4
4
|
*/
|
|
5
|
+
export * from './coverage';
|
|
6
|
+
export * from './reconcile';
|
|
7
|
+
export * from './listSelection';
|
|
8
|
+
export * from './memoryStore';
|
|
5
9
|
export * from './fileTreeMemoryStore';
|
|
6
10
|
export * from './scopeEncoding';
|
|
7
11
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/store/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './fileTreeMemoryStore';\nexport * from './scopeEncoding';\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/store/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './coverage';\nexport * from './reconcile';\nexport * from './listSelection';\nexport * from './memoryStore';\nexport * from './fileTreeMemoryStore';\nexport * from './scopeEncoding';\n"]}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Ask {@link IMemoryStore.list} for **every record in the vault**, bodies
|
|
7
|
+
* included.
|
|
8
|
+
*
|
|
9
|
+
* @remarks
|
|
10
|
+
* `list` requires a selection and rejects one that narrows nothing, so this is
|
|
11
|
+
* the only way to get the whole vault — and that is the point. Since the index
|
|
12
|
+
* holds envelopes only, `list` materializes every survivor from storage, so an
|
|
13
|
+
* unnarrowed call reads one file per record. Making it impossible to write by
|
|
14
|
+
* accident is worth more than making it fast.
|
|
15
|
+
*
|
|
16
|
+
* It is named for what it costs rather than for what it returns, and it is
|
|
17
|
+
* greppable: `scanEveryRecord` enumerates every whole-vault read in a codebase in
|
|
18
|
+
* one search. This mirrors `@fgv/ts-extras`' `safer-fetch`, where `addressGuard`
|
|
19
|
+
* is required with no default and `allowAnyAddress()` is the named opt-out.
|
|
20
|
+
*
|
|
21
|
+
* **If you only need to select, you do not need this.** Use
|
|
22
|
+
* {@link IMemoryStore.listEntries}, which returns every entry's scope and
|
|
23
|
+
* envelope, reads no files, and needs no selection.
|
|
24
|
+
* @public
|
|
25
|
+
*/
|
|
26
|
+
export function scanEveryRecord(options) {
|
|
27
|
+
return Object.assign({ scanEveryRecord: true }, ((options === null || options === void 0 ? void 0 : options.asOf) !== undefined ? { asOf: options.asOf } : {}));
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Whether a selection is the explicit whole-vault opt-out.
|
|
31
|
+
* @public
|
|
32
|
+
*/
|
|
33
|
+
export function isWholeVaultScan(selection) {
|
|
34
|
+
return selection.scanEveryRecord === true;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=listSelection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"listSelection.js","sourceRoot":"","sources":["../../../src/packlets/store/listSelection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA6EH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,eAAe,CAAC,OAAoC;IAClE,uBACE,eAAe,EAAE,IAAI,IAClB,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,MAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAC9D;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAA8B;IAC7D,OAAQ,SAA6B,CAAC,eAAe,KAAK,IAAI,CAAC;AACjE,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Kind, MemoryScopeKey, Tag } from '../types';\n\n/**\n * Filter for {@link IMemoryStore.list}. All present fields are ANDed together.\n *\n * @remarks\n * **At least one of `scope` / `kind` / `tag` must be present.** `asOf` is a\n * temporal *projection*, not a narrowing — it collapses versions rather than\n * excluding entities — so it does not satisfy the requirement on its own. A\n * selection that narrows nothing fails with a message naming\n * {@link scanEveryRecord}.\n * @public\n */\nexport interface IMemoryStoreListFilter {\n /**\n * Never present on a narrowing filter.\n *\n * @remarks\n * Exclusivity marker, paired with the `never`s on {@link IWholeVaultScan}.\n * Without it `{ scanEveryRecord: true, kind }` type-checks — TypeScript's\n * excess-property check on a union admits any property declared by *any*\n * member — and `list` then takes the scan branch and silently discards the\n * narrowing. Since the whole point of requiring a selection is that a\n * whole-vault read must be deliberate, a call that reads the whole vault while\n * *looking* narrowed is the one outcome this surface must not permit.\n */\n readonly scanEveryRecord?: never;\n /** Restrict to records in this scope. */\n readonly scope?: MemoryScopeKey;\n /** Restrict to records of this kind. */\n readonly kind?: Kind;\n /** Restrict to records carrying this tag (exact match). */\n readonly tag?: Tag;\n /**\n * For temporal (versioned) kinds: collapse each entity to the single version\n * valid at this epoch ms. Non-temporal records are timeless and pass through\n * unchanged. Absent = no temporal projection (every version is returned).\n */\n readonly asOf?: number;\n}\n\n/**\n * The named, deliberately uncomfortable opt-out from\n * {@link IMemoryStore.list}'s narrowing requirement — build one with\n * {@link scanEveryRecord}.\n * @public\n */\nexport interface IWholeVaultScan {\n /** Discriminator. Always `true`; produced only by {@link scanEveryRecord}. */\n readonly scanEveryRecord: true;\n /**\n * Never present on a whole-vault scan — see\n * {@link IMemoryStoreListFilter.scanEveryRecord} for why these markers exist.\n * A scan that also carried a narrowing axis would have that axis dropped.\n */\n readonly scope?: never;\n /** Never present on a whole-vault scan. See {@link IWholeVaultScan.scope}. */\n readonly kind?: never;\n /** Never present on a whole-vault scan. See {@link IWholeVaultScan.scope}. */\n readonly tag?: never;\n /**\n * Optional temporal projection, exactly as on {@link IMemoryStoreListFilter}.\n * NOT excluded, because `asOf` projects rather than narrows and composes with\n * a whole-vault read exactly as it does with a filtered one.\n */\n readonly asOf?: number;\n}\n\n/**\n * What {@link IMemoryStore.list} accepts: a narrowing filter, or the explicit\n * whole-vault scan.\n * @public\n */\nexport type MemoryListSelection = IMemoryStoreListFilter | IWholeVaultScan;\n\n/**\n * Ask {@link IMemoryStore.list} for **every record in the vault**, bodies\n * included.\n *\n * @remarks\n * `list` requires a selection and rejects one that narrows nothing, so this is\n * the only way to get the whole vault — and that is the point. Since the index\n * holds envelopes only, `list` materializes every survivor from storage, so an\n * unnarrowed call reads one file per record. Making it impossible to write by\n * accident is worth more than making it fast.\n *\n * It is named for what it costs rather than for what it returns, and it is\n * greppable: `scanEveryRecord` enumerates every whole-vault read in a codebase in\n * one search. This mirrors `@fgv/ts-extras`' `safer-fetch`, where `addressGuard`\n * is required with no default and `allowAnyAddress()` is the named opt-out.\n *\n * **If you only need to select, you do not need this.** Use\n * {@link IMemoryStore.listEntries}, which returns every entry's scope and\n * envelope, reads no files, and needs no selection.\n * @public\n */\nexport function scanEveryRecord(options?: { readonly asOf?: number }): IWholeVaultScan {\n return {\n scanEveryRecord: true,\n ...(options?.asOf !== undefined ? { asOf: options.asOf } : {})\n };\n}\n\n/**\n * Whether a selection is the explicit whole-vault opt-out.\n * @public\n */\nexport function isWholeVaultScan(selection: MemoryListSelection): selection is IWholeVaultScan {\n return (selection as IWholeVaultScan).scanEveryRecord === true;\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memoryStore.js","sourceRoot":"","sources":["../../../src/packlets/store/memoryStore.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport {\n DedupScope,\n EntityId,\n IMemoryRecord,\n IMemoryRecordResolver,\n Kind,\n MemoryId,\n MemoryScopeKey\n} from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { IMemoryRecordSource, IScopedMemoryRecord } from '../vector';\nimport { MemoryListSelection } from './listSelection';\nimport { IDerivedStateCoverage } from './coverage';\nimport { DerivedArtifact, ReconcileReport } from './reconcile';\n\n/**\n * The writable, FileTree-backed, content-hash-deduped memory store.\n *\n * @remarks\n * Extends {@link IMemoryRecordResolver} because the retrievers take a resolver\n * and a store is the obvious one to hand them — `{ index, resolver: store }` is\n * the documented wiring, and it has to type-check for the `IMemoryStore` handle\n * a consumer actually holds, not only for the concrete `FileTreeMemoryStore`.\n * @public\n */\nexport interface IMemoryStore extends IMemoryRecordResolver {\n /**\n * Keyed read by entity id. Resolves `entityId` to a storage address via the\n * registered {@link IIdentityCodec} for `kind`. Returns `undefined` when no\n * record exists. For a versioned (temporal) kind the current version is\n * *selected* from the derived in-memory index — the version history is walked\n * over envelopes, never bodies — and then that one version is read from\n * storage. So the selection costs no file reads and the result costs exactly\n * one, rather than one per version.\n */\n get(kind: Kind, entityId: EntityId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * Direct read by `(scope, MemoryId)`. Returns `undefined` when not found.\n */\n getById(scope: MemoryScopeKey, id: MemoryId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * List records: select over the derived index, then materialize the survivors.\n *\n * @remarks\n * **The selection is required and must narrow.** Since the index holds\n * envelopes only, every returned record is read from storage — so a call that\n * narrows nothing reads the whole vault, and that has to be a decision rather\n * than a default. Omitting the argument is a compile error; passing one with no\n * `scope` / `kind` / `tag` fails with a message naming {@link scanEveryRecord},\n * which is how a caller says it meant it.\n *\n * The requirement buys **explicitness, not a cost bound** — `{ kind }` on a\n * vault dominated by that kind still materializes most of it. What it prevents\n * is the whole-vault read nobody chose.\n *\n * If you only need to select, use {@link IMemoryStore.listEntries} instead: no\n * selection, no file reads.\n */\n list(selection: MemoryListSelection): Promise<Result<ReadonlyArray<IMemoryRecord<unknown>>>>;\n\n /**\n * How much of the store's **derived state** exists, resolved by {@link Kind}.\n *\n * @remarks\n * Answers *\"is my derived state consistent with my records, and if not by how\n * much?\"* for every artifact the store derives — `rank`, record vectors, and\n * fragment vectors — in one call.\n *\n * **Cheap and total, by contract rather than by implementation.** Every input is\n * an envelope field or an index-side count: it reads **no record bodies** and\n * calls **no embedder**, and the walk over the vault's own state touches the\n * filesystem not at all. That is why it takes no selection, unlike\n * {@link IMemoryStore.list} — the guard there exists because an unnarrowed list\n * reads the vault, and putting one here would decorate a free operation and make\n * that guard mean less.\n *\n * **The index-side counts are the one exception, and it is the caller's own\n * index that spends it.** A persistent index answers `size` / `recordCount` /\n * `fragmentCount` with a query — `SqliteVecVectorIndex` runs a prepared `COUNT`\n * — so on a durable backend this call does I/O and can **fail**. It is bounded\n * (one count per wired index, never per record) and it is why the return is a\n * `Result` rather than a bare value.\n *\n * *If a future addition to the report would require reading a record body, it\n * does not belong on this report.*\n *\n * The counterpart is {@link IMemoryStore.reconcile}: coverage says **how big**\n * the gap is, cheaply; reconcile says **what it was** and closes what it can, at\n * the cost of re-running the embedder. Neither substitutes for the other.\n *\n * **A lane is reported here whenever its *index* is wired**, which is weaker\n * than what reconcile requires (index **and** embedder). That is intended: an\n * index without an embedder still holds vectors and still answers queries, so\n * its coverage is a real number worth reporting. The consequence to expect is a\n * half-wired store that reports a gap `reconcile` will refuse to close, naming\n * the missing embedder.\n */\n coverage(): Promise<Result<IDerivedStateCoverage>>;\n\n /**\n * Every entry in the vault — scope and envelope, **no bodies**. Reads no files\n * and requires no selection, because there is nothing to be careful about: it\n * returns what the index already holds.\n *\n * @remarks\n * This is the whole-vault read most callers actually want. Selection, grouping,\n * counting, \"which kinds are in here\", \"what links at this\" — all of it is\n * envelope work. Reach for {@link IMemoryStore.list} with\n * {@link scanEveryRecord} only when you genuinely need every body.\n *\n * Synchronous in spirit but `Promise`-returning for consistency with the rest\n * of the store surface.\n */\n listEntries(): Promise<Result<ReadonlyArray<IIndexedMemoryEntry>>>;\n\n /**\n * List EVERY record in the vault, each paired with its scope-qualified\n * `(scope, id)` address — the projection {@link IMemoryRecordSource} requires.\n * Unlike {@link IMemoryStore.list | list}, it takes no filter (whole-vault) and\n * returns {@link IScopedMemoryRecord}s so a re-index keys each entry on the same\n * scoped target the incremental embed-on-write path uses. Two records that share\n * a filename stem across scopes appear as distinct entries.\n */\n listScoped(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n\n /**\n * Adapt this store to the {@link IMemoryRecordSource} seam so it can drive\n * {@link IVectorIndex} rebuilds (e.g. `InMemoryCosineIndex.rebuild`). The\n * returned source's `list()` delegates to {@link IMemoryStore.listScoped},\n * **filtered to the kinds {@link IMemoryStore.embedsKind | embedsKind} reports** —\n * this source exists to feed the record vector index, so a kind excluded from\n * that index is excluded here too, and a reopen does not re-embed records the\n * index will never return. With no\n * {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds} declaration\n * every kind participates and the filter is the identity. `listScoped` itself is\n * **not** filtered and remains the whole-vault surface.\n *\n * The filter also **counts what it drops**, onto\n * {@link IMemoryRecordListing.excluded} — this is the layer where the exclusion\n * decision is made and so the only one that can. The count is always present\n * (empty when nothing was excluded).\n *\n * The store cannot implement {@link IMemoryRecordSource} directly because its\n * `list(filter?)` returns bare records (the ergonomic query surface) while the\n * seam's `list()` returns scope-qualified records.\n */\n asRecordSource(): IMemoryRecordSource;\n\n /**\n * The EFFECTIVE {@link DedupScope} for `kind` — the granularity at which a\n * write for this kind deduplicates against the existing vault.\n *\n * @remarks\n * This is a **read accessor over the store's already-injected write policies**,\n * and it is the single place any caller — the store's own write path included —\n * asks what a kind's dedup granularity is. It resolves the full chain the store\n * applies on write: the kind's registered {@link IWritePolicy}, falling back to\n * the store's default policy, then that policy's\n * {@link IWritePolicy.dedupScope | dedupScope}, falling back to\n * {@link DEFAULT_DEDUP_SCOPE}. Note the store's default policy is a\n * {@link KnowledgeLwwPolicy}, which declares `'content'` — so a kind with NO\n * registered policy resolves to `'content'`, not to `DEFAULT_DEDUP_SCOPE`.\n *\n * It exists so a caller that must agree with the store about dedup granularity\n * — notably the ingest orchestrator's stage-4 layer-1 exact match — can read the\n * declaration through this seam instead of being handed a second copy of the\n * policy map. A second declaration site is precisely the defect this accessor\n * was added to remove.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail (every\n * link in the fallback chain has a total default). It exposes only the scope,\n * never the {@link IWritePolicy} itself, so it can never become a back door for\n * invoking admission or merge logic out of band.\n */\n dedupScopeFor(kind: Kind): DedupScope;\n\n /**\n * Whether records of `kind` participate in the **record-granular** vector index.\n *\n * @remarks\n * A read accessor over the store's injected {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds}\n * declaration, in the same spirit as {@link IMemoryStore.dedupScopeFor} — one\n * place to ask, so the store's write path and any caller reasoning about index\n * coverage cannot disagree. `true` for every kind when no declaration was made.\n *\n * **This is distinct from a {@link MemoryEmbedder} decline, and the difference is\n * cost.** An embedder that returns `undefined` has already been called: the\n * round trip is paid, and on a locally-hosted model that round trip is the\n * expense. A kind excluded here is never handed to the embedder at all. The\n * decline makes the intent *expressible*; this makes it *free*.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail.\n */\n embedsKind(kind: Kind): boolean;\n\n /**\n * Write a record. Validates the body, computes a content hash, deduplicates\n * (scope-wide, before policy), applies the kind's {@link IWritePolicy}, stamps\n * transaction-time metadata (`created` / `updated` / `seq` / `contentHash`),\n * writes the file, and patches the index. Returns the written record — or the\n * existing record unchanged on a dedup no-op.\n */\n put(record: IMemoryRecord<unknown>): Promise<Result<IMemoryRecord<unknown>>>;\n\n /**\n * Repair one derived artifact for one {@link Kind} — **targeted and\n * non-destructive**, unlike `IVectorIndex.rebuild`, which resets the index and\n * re-embeds everything.\n *\n * @remarks\n * The counterpart to {@link IMemoryStore.coverage}: coverage says **how big**\n * the gap is and costs nothing; reconcile says **what it was** and closes what\n * it can, at the cost of reading bodies and re-running the embedder. Neither\n * substitutes for the other — in particular, only reconcile can distinguish a\n * *declined* record from a *failed* one, because learning that requires calling\n * the embedder again.\n *\n * **It only touches what is missing.** For the vector lanes it asks\n * `has(target)` per record and skips the ones already held, so a repair after a\n * brief outage costs a handful of embedder calls rather than a whole vault.\n * That check is also the only way to see a record whose vector the index holds\n * but whose envelope lost its `embeddingRef` — which needs a restamp and no\n * embedder call at all, and which an `embeddingRef`-only repair cannot detect.\n *\n * **`artifact` is required and names one lane.** See {@link DerivedArtifact}\n * for why an operation repairing \"everything wired\" would be the wrong shape.\n *\n * **A vector lane must be wired on BOTH halves — index *and* embedder — or this\n * fails, and it is deliberately stricter than {@link IMemoryStore.coverage}.**\n * The two ask different questions of the same wiring. Coverage asks *what does\n * the index hold*, which an index alone can answer: an index wired without an\n * embedder is a legal store (queries work; writes simply do not embed), so\n * coverage reports that lane rather than pretending it is absent. Reconcile\n * asks to *produce* vectors, which needs the embedder. So a half-wired store\n * legitimately reports a coverage gap it cannot repair, and reconcile names the\n * missing half rather than returning a cheerful success with every record in\n * `failed`.\n *\n * Runs under the store's write lock, like any other mutation.\n */\n reconcile(kind: Kind, artifact: DerivedArtifact): Promise<Result<ReconcileReport>>;\n\n /**\n * Delete a record by `(kind, entityId)`. Non-temporal kinds physically delete\n * the file and return the deleted record's {@link MemoryId}. Temporal\n * (versioned) kinds SOFT-delete: the current version is invalidated\n * (`invalid_at` set), history is retained, and the invalidated version's\n * {@link MemoryId} is returned.\n */\n delete(kind: Kind, entityId: EntityId): Promise<Result<MemoryId>>;\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../../../src/packlets/store/reconcile.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Kind } from '../types';\nimport { ISkippedVectorRecord } from '../vector';\n\n/**\n * Which derived artifact a {@link IMemoryStore.reconcile} call repairs.\n *\n * @remarks\n * **The operation names its artifact rather than repairing everything wired**,\n * and the reasons are structural rather than stylistic: the record and fragment\n * lanes are independently wirable (so an unnamed repair on a fragment-only store\n * would have to no-op or guess), their units are incommensurable (one vector per\n * record vs. N), and their costs differ by orders of magnitude — a measured case\n * put 68 fragments behind a single 56 KB record. An operation whose cost varies\n * that much along a dimension it does not name is the accidental-expense shape\n * `IMemoryStore.list`'s required selection exists to prevent.\n * @public\n */\nexport type DerivedArtifact = 'rank' | 'record-vector' | 'fragment-vector';\n\n/** What every reconcile reports, whatever the artifact. @public */\nexport interface IReconcileReportBase {\n /** The kind reconciled — named by the caller, so every count below is scalar. */\n readonly kind: Kind;\n /** Records of this kind considered. The denominator for everything else. */\n readonly examined: number;\n /** Records whose derived artifact was (re)produced by this call. */\n readonly repaired: number;\n /** Records that failed, with the error. A fault, never a decline. */\n readonly failed: ReadonlyArray<ISkippedVectorRecord>;\n}\n\n/** A `rank` reconcile: re-run the projector and restamp what changed. @public */\nexport interface IRankReconcileReport extends IReconcileReportBase {\n readonly artifact: 'rank';\n}\n\n/**\n * A record-vector reconcile.\n * @public\n */\nexport interface IVectorReconcileReport extends IReconcileReportBase {\n readonly artifact: 'record-vector';\n /**\n * Records the index already held and which needed **no embedder call** — the\n * whole point of a targeted repair, and the number that shows what it saved\n * against a `rebuild`.\n */\n readonly alreadyIndexed: number;\n /**\n * Records whose vector the index held but whose envelope had **lost its\n * `embeddingRef`** — repaired by restamping the reference, with no embedder\n * call.\n *\n * @remarks\n * This case is invisible to an `embeddingRef`-only check, which is one of the\n * two reasons `IVectorIndex.has` is on the contract: a reference-absent record\n * looks identical to a never-embedded one until you ask the index.\n */\n readonly restamped: number;\n /** Records the embedder intentionally declined. Not a failure, not a gap. */\n readonly declined: number;\n}\n\n/**\n * A fragment-vector reconcile.\n *\n * @remarks\n * Note there is no `restamped`: the fragment lane has **no envelope marker**, so\n * there is no reference that can go missing and nothing to restamp. The\n * asymmetry with {@link IVectorReconcileReport} is real rather than an oversight.\n * @public\n */\nexport interface IFragmentReconcileReport extends IReconcileReportBase {\n readonly artifact: 'fragment-vector';\n /** Records already represented, needing no embedder call. */\n readonly alreadyIndexed: number;\n /** Records whose embedder produced no fragments — this lane's decline. */\n readonly declined: number;\n /** Fragments written by this call — the fan-out `repaired` cannot express. */\n readonly fragments: number;\n}\n\n/**\n * What a {@link IMemoryStore.reconcile} established, discriminated by artifact.\n * @public\n */\nexport type ReconcileReport = IRankReconcileReport | IVectorReconcileReport | IFragmentReconcileReport;\n"]}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
import { captureResult, succeed } from '@fgv/ts-utils';
|
|
6
|
+
import { embeddingRefOf } from '../types';
|
|
7
|
+
/** Get or create `kind`'s bucket. */
|
|
8
|
+
function bucketFor(buckets, kind) {
|
|
9
|
+
let bucket = buckets.get(kind);
|
|
10
|
+
if (bucket === undefined) {
|
|
11
|
+
bucket = { expected: 0, covered: 0 };
|
|
12
|
+
buckets.set(kind, bucket);
|
|
13
|
+
}
|
|
14
|
+
return bucket;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* One walk over the projected entries, producing every derived-state count.
|
|
18
|
+
*
|
|
19
|
+
* @remarks
|
|
20
|
+
* **Reads no record bodies and calls no embedder**, and the walk itself touches
|
|
21
|
+
* no filesystem. That is the contract `IMemoryStore.coverage` makes, and this
|
|
22
|
+
* function is where it is kept: every number below comes from an envelope field
|
|
23
|
+
* or an index-side count. The index-side counts are the one place that leaves
|
|
24
|
+
* envelope territory — on a durable backend they run a query, which is why they
|
|
25
|
+
* are captured (see the comment at the `captureResult` below) and why the walk's
|
|
26
|
+
* "no filesystem" claim is scoped to the walk rather than the whole call. A
|
|
27
|
+
* future addition that needs a body does not belong here.
|
|
28
|
+
*
|
|
29
|
+
* Extracted from `fileTreeMemoryStore.ts` because inlining it took that file past
|
|
30
|
+
* the 2000-line `max-lines` cap — the same reason `vectorRecordSource.ts` exists.
|
|
31
|
+
*/
|
|
32
|
+
export function computeCoverage(params) {
|
|
33
|
+
var _a;
|
|
34
|
+
const records = new Map();
|
|
35
|
+
const rank = new Map();
|
|
36
|
+
const vectors = new Map();
|
|
37
|
+
for (const entry of params.entries) {
|
|
38
|
+
const kind = entry.envelope.kind;
|
|
39
|
+
records.set(kind, ((_a = records.get(kind)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
40
|
+
// `rank` is expected only where a projector is registered. A kind with no
|
|
41
|
+
// projector stays absent from the map rather than appearing at 0% — the store
|
|
42
|
+
// is not failing to rank it, it was never asked to.
|
|
43
|
+
if (params.hasRankProjector(kind)) {
|
|
44
|
+
const bucket = bucketFor(rank, kind);
|
|
45
|
+
bucket.expected += 1;
|
|
46
|
+
if (entry.envelope.rank !== undefined) {
|
|
47
|
+
bucket.covered += 1;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// Every kind appears here, INCLUDING excluded ones: `records: 40, expected: 0`
|
|
51
|
+
// is the exclusion story stated rather than inferred, and is strictly more
|
|
52
|
+
// useful than omitting the kind entirely.
|
|
53
|
+
const bucket = bucketFor(vectors, kind);
|
|
54
|
+
if (params.embedsKind(kind)) {
|
|
55
|
+
bucket.expected += 1;
|
|
56
|
+
// Counted INSIDE the exclusion branch so `covered <= expected` holds, which
|
|
57
|
+
// is what `IArtifactCoverage.covered` promises when it says "of those". An
|
|
58
|
+
// excluded kind carrying a residual `embeddingRef` (written before
|
|
59
|
+
// `embedKinds` narrowed and never re-put) would otherwise report
|
|
60
|
+
// `expected: 0, covered: 3` and make `expected - covered` negative for a
|
|
61
|
+
// caller sizing the gap. That residue is not lost: its vector still counts
|
|
62
|
+
// toward `indexSize`, which is exactly the belief-vs-fact disagreement that
|
|
63
|
+
// field exists to surface.
|
|
64
|
+
if (embeddingRefOf(entry.envelope) !== undefined) {
|
|
65
|
+
bucket.covered += 1;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// The index-side counts are the ONE place this walk leaves envelope territory,
|
|
70
|
+
// and on a durable backend they run SQL: `SqliteVecVectorIndex.size` executes a
|
|
71
|
+
// prepared COUNT and throws on a closed connection. Capturing them keeps the
|
|
72
|
+
// `Result` contract honest — coverage FAILS rather than rejecting — and is why
|
|
73
|
+
// the "touches no filesystem" claim is scoped to the walk rather than the whole
|
|
74
|
+
// call: reading a persistent index's count is I/O, and it is the caller's own
|
|
75
|
+
// index doing it.
|
|
76
|
+
return captureResult(() => ({
|
|
77
|
+
recordVectors: params.vectorIndex === undefined ? undefined : { size: params.vectorIndex.size },
|
|
78
|
+
fragmentVectors: params.fragmentIndex === undefined
|
|
79
|
+
? undefined
|
|
80
|
+
: {
|
|
81
|
+
recordCount: params.fragmentIndex.recordCount,
|
|
82
|
+
fragmentCount: params.fragmentIndex.fragmentCount
|
|
83
|
+
}
|
|
84
|
+
}))
|
|
85
|
+
.withErrorFormat((e) => `memory coverage: reading an index count failed: ${e}`)
|
|
86
|
+
.onSuccess((counts) => succeed(Object.assign(Object.assign(Object.assign({ records }, (params.anyRankProjector ? { rank: rank } : {})), (counts.recordVectors !== undefined
|
|
87
|
+
? {
|
|
88
|
+
recordVectors: {
|
|
89
|
+
perKind: vectors,
|
|
90
|
+
indexSize: counts.recordVectors.size
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
: {})), (counts.fragmentVectors !== undefined
|
|
94
|
+
? {
|
|
95
|
+
fragmentVectors: {
|
|
96
|
+
indexRecordCount: counts.fragmentVectors.recordCount,
|
|
97
|
+
indexFragmentCount: counts.fragmentVectors.fragmentCount
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
: {}))));
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=storeCoverage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeCoverage.js","sourceRoot":"","sources":["../../../src/packlets/store/storeCoverage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAU,aAAa,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAQ,cAAc,EAAE,MAAM,UAAU,CAAC;AAkChD,qCAAqC;AACrC,SAAS,SAAS,CAAC,OAA4C,EAAE,IAAU;IACzE,IAAI,MAAM,GAAyC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,eAAe,CAAC,MAA8B;;IAC5D,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAC;IAC3D,MAAM,IAAI,GAAwC,IAAI,GAAG,EAAkC,CAAC;IAC5F,MAAM,OAAO,GAAwC,IAAI,GAAG,EAAkC,CAAC;IAE/F,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,IAAI,GAAS,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAA,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,8EAA8E;QAC9E,oDAAoD;QACpD,IAAI,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,MAAM,GAA6B,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACtC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,2EAA2E;QAC3E,0CAA0C;QAC1C,MAAM,MAAM,GAA6B,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAClE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrB,4EAA4E;YAC5E,2EAA2E;YAC3E,mEAAmE;YACnE,iEAAiE;YACjE,yEAAyE;YACzE,2EAA2E;YAC3E,4EAA4E;YAC5E,2BAA2B;YAC3B,IAAI,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,gFAAgF;IAChF,6EAA6E;IAC7E,+EAA+E;IAC/E,gFAAgF;IAChF,8EAA8E;IAC9E,kBAAkB;IAClB,OAAO,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE;QAC/F,eAAe,EACb,MAAM,CAAC,aAAa,KAAK,SAAS;YAChC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW;gBAC7C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,aAAa;aAClD;KACR,CAAC,CAAC;SACA,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mDAAmD,CAAC,EAAE,CAAC;SAC9E,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CACpB,OAAO,6CACL,OAAO,IAGJ,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAA4C,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACvF,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS;QACpC,CAAC,CAAC;YACE,aAAa,EAAE;gBACb,OAAO,EAAE,OAA+C;gBACxD,SAAS,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;aACrC;SACF;QACH,CAAC,CAAC,EAAE,CAAC,GACJ,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS;QACtC,CAAC,CAAC;YACE,eAAe,EAAE;gBACf,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC,WAAW;gBACpD,kBAAkB,EAAE,MAAM,CAAC,eAAe,CAAC,aAAa;aACzD;SACF;QACH,CAAC,CAAC,EAAE,CAAC,EACP,CACH,CAAC;AACN,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result, captureResult, succeed } from '@fgv/ts-utils';\nimport { Kind, embeddingRefOf } from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { IFragmentVectorIndex, IVectorIndex } from '../vector';\nimport { IArtifactCoverage, IDerivedStateCoverage } from './coverage';\n\n/**\n * Everything {@link computeCoverage} needs, supplied by the store.\n *\n * @remarks\n * Taken as inputs rather than reading a store, so this module has no dependency\n * on `FileTreeMemoryStore` and the walk is testable on its own. Package-internal:\n * `IMemoryStore.coverage` is the published surface.\n */\nexport interface IComputeCoverageParams {\n /** The projected index entries — envelopes only, already free of file reads. */\n readonly entries: ReadonlyArray<IIndexedMemoryEntry>;\n /** Whether `kind` has a registered rank projector. */\n readonly hasRankProjector: (kind: Kind) => boolean;\n /** Whether ANY kind has one — distinguishes an unwired lane from an empty one. */\n readonly anyRankProjector: boolean;\n /** Whether `kind` participates in the record-granular vector index. */\n readonly embedsKind: (kind: Kind) => boolean;\n /** The wired record-vector index, or `undefined` when the lane is unwired. */\n readonly vectorIndex: IVectorIndex | undefined;\n /** The wired fragment index, or `undefined` when the lane is unwired. */\n readonly fragmentIndex: IFragmentVectorIndex | undefined;\n}\n\n/** A coverage bucket under construction; the published shape is readonly. */\ninterface IMutableArtifactCoverage {\n expected: number;\n covered: number;\n}\n\n/** Get or create `kind`'s bucket. */\nfunction bucketFor(buckets: Map<Kind, IMutableArtifactCoverage>, kind: Kind): IMutableArtifactCoverage {\n let bucket: IMutableArtifactCoverage | undefined = buckets.get(kind);\n if (bucket === undefined) {\n bucket = { expected: 0, covered: 0 };\n buckets.set(kind, bucket);\n }\n return bucket;\n}\n\n/**\n * One walk over the projected entries, producing every derived-state count.\n *\n * @remarks\n * **Reads no record bodies and calls no embedder**, and the walk itself touches\n * no filesystem. That is the contract `IMemoryStore.coverage` makes, and this\n * function is where it is kept: every number below comes from an envelope field\n * or an index-side count. The index-side counts are the one place that leaves\n * envelope territory — on a durable backend they run a query, which is why they\n * are captured (see the comment at the `captureResult` below) and why the walk's\n * \"no filesystem\" claim is scoped to the walk rather than the whole call. A\n * future addition that needs a body does not belong here.\n *\n * Extracted from `fileTreeMemoryStore.ts` because inlining it took that file past\n * the 2000-line `max-lines` cap — the same reason `vectorRecordSource.ts` exists.\n */\nexport function computeCoverage(params: IComputeCoverageParams): Result<IDerivedStateCoverage> {\n const records: Map<Kind, number> = new Map<Kind, number>();\n const rank: Map<Kind, IMutableArtifactCoverage> = new Map<Kind, IMutableArtifactCoverage>();\n const vectors: Map<Kind, IMutableArtifactCoverage> = new Map<Kind, IMutableArtifactCoverage>();\n\n for (const entry of params.entries) {\n const kind: Kind = entry.envelope.kind;\n records.set(kind, (records.get(kind) ?? 0) + 1);\n\n // `rank` is expected only where a projector is registered. A kind with no\n // projector stays absent from the map rather than appearing at 0% — the store\n // is not failing to rank it, it was never asked to.\n if (params.hasRankProjector(kind)) {\n const bucket: IMutableArtifactCoverage = bucketFor(rank, kind);\n bucket.expected += 1;\n if (entry.envelope.rank !== undefined) {\n bucket.covered += 1;\n }\n }\n\n // Every kind appears here, INCLUDING excluded ones: `records: 40, expected: 0`\n // is the exclusion story stated rather than inferred, and is strictly more\n // useful than omitting the kind entirely.\n const bucket: IMutableArtifactCoverage = bucketFor(vectors, kind);\n if (params.embedsKind(kind)) {\n bucket.expected += 1;\n // Counted INSIDE the exclusion branch so `covered <= expected` holds, which\n // is what `IArtifactCoverage.covered` promises when it says \"of those\". An\n // excluded kind carrying a residual `embeddingRef` (written before\n // `embedKinds` narrowed and never re-put) would otherwise report\n // `expected: 0, covered: 3` and make `expected - covered` negative for a\n // caller sizing the gap. That residue is not lost: its vector still counts\n // toward `indexSize`, which is exactly the belief-vs-fact disagreement that\n // field exists to surface.\n if (embeddingRefOf(entry.envelope) !== undefined) {\n bucket.covered += 1;\n }\n }\n }\n\n // The index-side counts are the ONE place this walk leaves envelope territory,\n // and on a durable backend they run SQL: `SqliteVecVectorIndex.size` executes a\n // prepared COUNT and throws on a closed connection. Capturing them keeps the\n // `Result` contract honest — coverage FAILS rather than rejecting — and is why\n // the \"touches no filesystem\" claim is scoped to the walk rather than the whole\n // call: reading a persistent index's count is I/O, and it is the caller's own\n // index doing it.\n return captureResult(() => ({\n recordVectors: params.vectorIndex === undefined ? undefined : { size: params.vectorIndex.size },\n fragmentVectors:\n params.fragmentIndex === undefined\n ? undefined\n : {\n recordCount: params.fragmentIndex.recordCount,\n fragmentCount: params.fragmentIndex.fragmentCount\n }\n }))\n .withErrorFormat((e) => `memory coverage: reading an index count failed: ${e}`)\n .onSuccess((counts) =>\n succeed({\n records,\n // Absent, never zero: an unwired lane and an empty one are different facts,\n // and folding them makes a health surface alarm on a feature nobody enabled.\n ...(params.anyRankProjector ? { rank: rank as ReadonlyMap<Kind, IArtifactCoverage> } : {}),\n ...(counts.recordVectors !== undefined\n ? {\n recordVectors: {\n perKind: vectors as ReadonlyMap<Kind, IArtifactCoverage>,\n indexSize: counts.recordVectors.size\n }\n }\n : {}),\n ...(counts.fragmentVectors !== undefined\n ? {\n fragmentVectors: {\n indexRecordCount: counts.fragmentVectors.recordCount,\n indexFragmentCount: counts.fragmentVectors.fragmentCount\n }\n }\n : {})\n })\n );\n}\n"]}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
import { fail, succeed } from '@fgv/ts-utils';
|
|
6
|
+
import { edgeTargetKey, embeddingRefOf } from '../types';
|
|
7
|
+
import { captureVectorHook } from './vectorMaintenance';
|
|
8
|
+
/**
|
|
9
|
+
* Repair one vector lane for one kind, touching only what is missing.
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* **The `has` check is what makes this targeted rather than a rebuild.** A record
|
|
13
|
+
* the index already holds costs one membership query and no embedder call, which
|
|
14
|
+
* is the entire difference between repairing a brief outage and re-embedding a
|
|
15
|
+
* vault. It is also the only way to see the record-lane case where the index
|
|
16
|
+
* holds a vector but the envelope lost its `embeddingRef` — that record needs a
|
|
17
|
+
* restamp and no embedder call, and is indistinguishable from a never-embedded
|
|
18
|
+
* one if you only look at the envelope.
|
|
19
|
+
*
|
|
20
|
+
* Per-record failures are collected rather than fatal: a repair that stops at the
|
|
21
|
+
* first bad record leaves the rest of the gap open, which is the opposite of what
|
|
22
|
+
* a caller asked for. A failure to *materialize* is likewise collected — a record
|
|
23
|
+
* that vanished between the walk and the read is a casualty of this call, not a
|
|
24
|
+
* reason to abandon the others.
|
|
25
|
+
*/
|
|
26
|
+
export async function reconcileVectors(params) {
|
|
27
|
+
const fragment = params.artifact === 'fragment-vector';
|
|
28
|
+
const index = fragment ? params.maintenance.fragmentIndex : params.maintenance.vectorIndex;
|
|
29
|
+
// BOTH halves, not just the index: an index wired without an embedder is a
|
|
30
|
+
// legal store (queries work, writes simply do not embed), and reconciling it
|
|
31
|
+
// would otherwise report a cheerful success with every record in `failed`.
|
|
32
|
+
// "Half the lane is wired" is a wiring mistake and says so.
|
|
33
|
+
const embedder = fragment ? params.maintenance.fragmentEmbedder : params.maintenance.embedder;
|
|
34
|
+
if (index === undefined || embedder === undefined) {
|
|
35
|
+
return fail(`memory reconcile '${params.kind}' ${params.artifact}: the ${fragment ? 'fragment' : 'record-vector'} lane is not fully wired (index: ${index === undefined ? 'absent' : 'present'}, embedder: ${embedder === undefined ? 'absent' : 'present'})`);
|
|
36
|
+
}
|
|
37
|
+
if (!fragment && !params.embedsKind(params.kind)) {
|
|
38
|
+
// Not a failure and not a silent no-op: the caller asked to repair a kind the
|
|
39
|
+
// store is deliberately not embedding, and the honest answer names that
|
|
40
|
+
// rather than reporting a healthy zero.
|
|
41
|
+
return fail(`memory reconcile '${params.kind}' record-vector: this kind is excluded from the record vector index`);
|
|
42
|
+
}
|
|
43
|
+
let repaired = 0;
|
|
44
|
+
let restamped = 0;
|
|
45
|
+
let declined = 0;
|
|
46
|
+
let alreadyIndexed = 0;
|
|
47
|
+
let fragments = 0;
|
|
48
|
+
const failed = [];
|
|
49
|
+
for (const entry of params.targets) {
|
|
50
|
+
const target = { scope: entry.scope, id: entry.envelope.id };
|
|
51
|
+
// Captured like every other consumer hook: `has` belongs to the injected
|
|
52
|
+
// index and may throw rather than fail. Unwrapped it would reject the whole
|
|
53
|
+
// `IMemoryStore.reconcile` call on one bad record, which is both a Result-
|
|
54
|
+
// contract break and the opposite of this loop's collect-and-continue intent.
|
|
55
|
+
const held = await captureVectorHook(() => index.has(target), `membership check for '${entry.envelope.id}'`);
|
|
56
|
+
if (held.isFailure()) {
|
|
57
|
+
failed.push({ target, error: `membership check failed: ${held.message}` });
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (held.value) {
|
|
61
|
+
// The record lane can still be inconsistent while the index holds the
|
|
62
|
+
// vector: a reference lost after the vector was committed. Repairing it
|
|
63
|
+
// costs a write and no embedding.
|
|
64
|
+
if (!fragment && embeddingRefOf(entry.envelope) === undefined) {
|
|
65
|
+
// The one place a synthesized reference is unavoidable: `has` proved the
|
|
66
|
+
// vector exists but there is no contract member that returns the
|
|
67
|
+
// reference the index minted for it, and re-deriving one would cost the
|
|
68
|
+
// embedder call this branch exists to avoid. Sound for both shipped
|
|
69
|
+
// indexes, whose reference IS the scoped key; a third-party index that
|
|
70
|
+
// mints something else gets the scoped key stamped here. Recorded in
|
|
71
|
+
// `docs/FUTURE.md` rather than silently assumed.
|
|
72
|
+
const stamped = params.stampRef(entry.scope, entry.envelope.id, edgeTargetKey(target));
|
|
73
|
+
if (stamped.isFailure()) {
|
|
74
|
+
failed.push({ target, error: `restamping the embedding reference failed: ${stamped.message}` });
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
restamped++;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
alreadyIndexed++;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const resolved = params.resolve(entry.scope, entry.envelope.id);
|
|
84
|
+
if (resolved.isFailure()) {
|
|
85
|
+
failed.push({ target, error: `reading the record failed: ${resolved.message}` });
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (resolved.value === undefined) {
|
|
89
|
+
failed.push({ target, error: 'the index claims this record but the vault has no such file' });
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const outcome = await (fragment
|
|
93
|
+
? params.maintenance.reembedFragments(resolved.value, target)
|
|
94
|
+
: params.maintenance.reembedRecord(resolved.value, target));
|
|
95
|
+
if (outcome.isFailure()) {
|
|
96
|
+
failed.push({ target, error: outcome.message });
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (outcome.value === undefined) {
|
|
100
|
+
declined++;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (fragment) {
|
|
104
|
+
fragments += outcome.value.count;
|
|
105
|
+
repaired++;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
// The reference the INDEX returned, never a synthesized key: the write path
|
|
109
|
+
// persists `add`'s value, so a third-party index whose reference is not the
|
|
110
|
+
// scoped key would otherwise get one stamp from `put` and a different one here.
|
|
111
|
+
const stamped = params.stampRef(entry.scope, entry.envelope.id, outcome.value.ref);
|
|
112
|
+
if (stamped.isFailure()) {
|
|
113
|
+
failed.push({ target, error: `stamping the embedding reference failed: ${stamped.message}` });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
repaired++;
|
|
117
|
+
}
|
|
118
|
+
const base = { kind: params.kind, examined: params.targets.length, repaired, failed };
|
|
119
|
+
return succeed(fragment
|
|
120
|
+
? Object.assign(Object.assign({}, base), { artifact: 'fragment-vector', alreadyIndexed, declined, fragments }) : Object.assign(Object.assign({}, base), { artifact: 'record-vector', alreadyIndexed, restamped, declined }));
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=storeReconcile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeReconcile.js","sourceRoot":"","sources":["../../../src/packlets/store/storeReconcile.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAU,IAAI,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAML,aAAa,EACb,cAAc,EACf,MAAM,UAAU,CAAC;AAIlB,OAAO,EAAsC,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAmB5F;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAA+B;IACpE,MAAM,QAAQ,GAAY,MAAM,CAAC,QAAQ,KAAK,iBAAiB,CAAC;IAChE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC;IAC3F,2EAA2E;IAC3E,6EAA6E;IAC7E,2EAA2E;IAC3E,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC;IAC9F,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,IAAI,CACT,qBAAqB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,QAAQ,SAClD,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAC1B,oCAAoC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,eAC5E,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SACtC,GAAG,CACJ,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,8EAA8E;QAC9E,wEAAwE;QACxE,wCAAwC;QACxC,OAAO,IAAI,CACT,qBAAqB,MAAM,CAAC,IAAI,qEAAqE,CACtG,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAW,CAAC,CAAC;IACzB,IAAI,SAAS,GAAW,CAAC,CAAC;IAC1B,IAAI,QAAQ,GAAW,CAAC,CAAC;IACzB,IAAI,cAAc,GAAW,CAAC,CAAC;IAC/B,IAAI,SAAS,GAAW,CAAC,CAAC;IAC1B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC1E,yEAAyE;QACzE,4EAA4E;QAC5E,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,IAAI,GAAoB,MAAM,iBAAiB,CACnD,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EACvB,yBAAyB,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,CAC9C,CAAC;QACF,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,4BAA4B,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC3E,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,sEAAsE;YACtE,wEAAwE;YACxE,kCAAkC;YAClC,IAAI,CAAC,QAAQ,IAAI,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9D,yEAAyE;gBACzE,iEAAiE;gBACjE,wEAAwE;gBACxE,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,OAAO,GAAoB,MAAM,CAAC,QAAQ,CAC9C,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,CAAC,EAAE,EACjB,aAAa,CAAC,MAAM,CAAC,CACtB,CAAC;gBACF,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;oBACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,8CAA8C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBAChG,SAAS;gBACX,CAAC;gBACD,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAA+C,MAAM,CAAC,OAAO,CACzE,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,CAAC,EAAE,CAClB,CAAC;QACF,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjF,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,6DAA6D,EAAE,CAAC,CAAC;YAC9F,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAwC,MAAM,CAAC,QAAQ;YAClE,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YAC7D,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;YACjC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,4EAA4E;QAC5E,4EAA4E;QAC5E,gFAAgF;QAChF,MAAM,OAAO,GAAoB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpG,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,4CAA4C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC9F,SAAS;QACX,CAAC;QACD,QAAQ,EAAE,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACtF,OAAO,OAAO,CACZ,QAAQ;QACN,CAAC,iCAAM,IAAI,KAAE,QAAQ,EAAE,iBAAiB,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,IAC7E,CAAC,iCAAM,IAAI,KAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,QAAQ,GAAE,CAChF,CAAC;AACJ,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport {\n IEdgeTarget,\n IMemoryRecord,\n Kind,\n MemoryId,\n MemoryScopeKey,\n edgeTargetKey,\n embeddingRefOf\n} from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { ISkippedVectorRecord } from '../vector';\nimport { DerivedArtifact, ReconcileReport } from './reconcile';\nimport { IReembedOutcome, VectorMaintenance, captureVectorHook } from './vectorMaintenance';\n\n/**\n * Everything the vector reconcile branches need, supplied by the store.\n * Package-internal; `IMemoryStore.reconcile` is the published surface.\n */\nexport interface IReconcileVectorsParams {\n readonly kind: Kind;\n readonly artifact: Exclude<DerivedArtifact, 'rank'>;\n /** The kind's index entries — envelopes only, no bodies read yet. */\n readonly targets: ReadonlyArray<IIndexedMemoryEntry>;\n readonly maintenance: VectorMaintenance;\n readonly embedsKind: (kind: Kind) => boolean;\n /** Materialize one record; `undefined` when it has vanished. */\n readonly resolve: (scope: MemoryScopeKey, id: MemoryId) => Result<IMemoryRecord<unknown> | undefined>;\n /** Write an `embeddingRef` onto a record's envelope; `true` when it changed. */\n readonly stampRef: (scope: MemoryScopeKey, id: MemoryId, ref: string) => Result<boolean>;\n}\n\n/**\n * Repair one vector lane for one kind, touching only what is missing.\n *\n * @remarks\n * **The `has` check is what makes this targeted rather than a rebuild.** A record\n * the index already holds costs one membership query and no embedder call, which\n * is the entire difference between repairing a brief outage and re-embedding a\n * vault. It is also the only way to see the record-lane case where the index\n * holds a vector but the envelope lost its `embeddingRef` — that record needs a\n * restamp and no embedder call, and is indistinguishable from a never-embedded\n * one if you only look at the envelope.\n *\n * Per-record failures are collected rather than fatal: a repair that stops at the\n * first bad record leaves the rest of the gap open, which is the opposite of what\n * a caller asked for. A failure to *materialize* is likewise collected — a record\n * that vanished between the walk and the read is a casualty of this call, not a\n * reason to abandon the others.\n */\nexport async function reconcileVectors(params: IReconcileVectorsParams): Promise<Result<ReconcileReport>> {\n const fragment: boolean = params.artifact === 'fragment-vector';\n const index = fragment ? params.maintenance.fragmentIndex : params.maintenance.vectorIndex;\n // BOTH halves, not just the index: an index wired without an embedder is a\n // legal store (queries work, writes simply do not embed), and reconciling it\n // would otherwise report a cheerful success with every record in `failed`.\n // \"Half the lane is wired\" is a wiring mistake and says so.\n const embedder = fragment ? params.maintenance.fragmentEmbedder : params.maintenance.embedder;\n if (index === undefined || embedder === undefined) {\n return fail(\n `memory reconcile '${params.kind}' ${params.artifact}: the ${\n fragment ? 'fragment' : 'record-vector'\n } lane is not fully wired (index: ${index === undefined ? 'absent' : 'present'}, embedder: ${\n embedder === undefined ? 'absent' : 'present'\n })`\n );\n }\n if (!fragment && !params.embedsKind(params.kind)) {\n // Not a failure and not a silent no-op: the caller asked to repair a kind the\n // store is deliberately not embedding, and the honest answer names that\n // rather than reporting a healthy zero.\n return fail(\n `memory reconcile '${params.kind}' record-vector: this kind is excluded from the record vector index`\n );\n }\n\n let repaired: number = 0;\n let restamped: number = 0;\n let declined: number = 0;\n let alreadyIndexed: number = 0;\n let fragments: number = 0;\n const failed: ISkippedVectorRecord[] = [];\n\n for (const entry of params.targets) {\n const target: IEdgeTarget = { scope: entry.scope, id: entry.envelope.id };\n // Captured like every other consumer hook: `has` belongs to the injected\n // index and may throw rather than fail. Unwrapped it would reject the whole\n // `IMemoryStore.reconcile` call on one bad record, which is both a Result-\n // contract break and the opposite of this loop's collect-and-continue intent.\n const held: Result<boolean> = await captureVectorHook(\n () => index.has(target),\n `membership check for '${entry.envelope.id}'`\n );\n if (held.isFailure()) {\n failed.push({ target, error: `membership check failed: ${held.message}` });\n continue;\n }\n if (held.value) {\n // The record lane can still be inconsistent while the index holds the\n // vector: a reference lost after the vector was committed. Repairing it\n // costs a write and no embedding.\n if (!fragment && embeddingRefOf(entry.envelope) === undefined) {\n // The one place a synthesized reference is unavoidable: `has` proved the\n // vector exists but there is no contract member that returns the\n // reference the index minted for it, and re-deriving one would cost the\n // embedder call this branch exists to avoid. Sound for both shipped\n // indexes, whose reference IS the scoped key; a third-party index that\n // mints something else gets the scoped key stamped here. Recorded in\n // `docs/FUTURE.md` rather than silently assumed.\n const stamped: Result<boolean> = params.stampRef(\n entry.scope,\n entry.envelope.id,\n edgeTargetKey(target)\n );\n if (stamped.isFailure()) {\n failed.push({ target, error: `restamping the embedding reference failed: ${stamped.message}` });\n continue;\n }\n restamped++;\n continue;\n }\n alreadyIndexed++;\n continue;\n }\n\n const resolved: Result<IMemoryRecord<unknown> | undefined> = params.resolve(\n entry.scope,\n entry.envelope.id\n );\n if (resolved.isFailure()) {\n failed.push({ target, error: `reading the record failed: ${resolved.message}` });\n continue;\n }\n if (resolved.value === undefined) {\n failed.push({ target, error: 'the index claims this record but the vault has no such file' });\n continue;\n }\n const outcome: Result<IReembedOutcome | undefined> = await (fragment\n ? params.maintenance.reembedFragments(resolved.value, target)\n : params.maintenance.reembedRecord(resolved.value, target));\n if (outcome.isFailure()) {\n failed.push({ target, error: outcome.message });\n continue;\n }\n if (outcome.value === undefined) {\n declined++;\n continue;\n }\n if (fragment) {\n fragments += outcome.value.count;\n repaired++;\n continue;\n }\n // The reference the INDEX returned, never a synthesized key: the write path\n // persists `add`'s value, so a third-party index whose reference is not the\n // scoped key would otherwise get one stamp from `put` and a different one here.\n const stamped: Result<boolean> = params.stampRef(entry.scope, entry.envelope.id, outcome.value.ref);\n if (stamped.isFailure()) {\n failed.push({ target, error: `stamping the embedding reference failed: ${stamped.message}` });\n continue;\n }\n repaired++;\n }\n\n const base = { kind: params.kind, examined: params.targets.length, repaired, failed };\n return succeed(\n fragment\n ? { ...base, artifact: 'fragment-vector', alreadyIndexed, declined, fragments }\n : { ...base, artifact: 'record-vector', alreadyIndexed, restamped, declined }\n );\n}\n"]}
|
|
@@ -14,6 +14,7 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
14
14
|
return t;
|
|
15
15
|
};
|
|
16
16
|
import { fail, succeed } from '@fgv/ts-utils';
|
|
17
|
+
import { embeddingRefOf } from '../types';
|
|
17
18
|
/**
|
|
18
19
|
* Project a record the embedder **declined** into its written form: the same
|
|
19
20
|
* record with no `embeddingRef`, plus the vector (if any) that reference
|
|
@@ -40,7 +41,7 @@ import { fail, succeed } from '@fgv/ts-utils';
|
|
|
40
41
|
* `_persist`.
|
|
41
42
|
*/
|
|
42
43
|
function declineEmbedding(built, index, target, embed) {
|
|
43
|
-
if (built.envelope
|
|
44
|
+
if (embeddingRefOf(built.envelope) === undefined) {
|
|
44
45
|
return { record: built, embed };
|
|
45
46
|
}
|
|
46
47
|
// Rest-spread rather than `embeddingRef: undefined`: the envelope is YAML-
|
|
@@ -50,6 +51,37 @@ function declineEmbedding(built, index, target, embed) {
|
|
|
50
51
|
const _a = built.envelope, { embeddingRef } = _a, envelope = __rest(_a, ["embeddingRef"]);
|
|
51
52
|
return { record: { envelope, body: built.body }, stale: { index, target }, embed };
|
|
52
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Run a consumer-supplied hook, normalizing a synchronous throw or a rejected
|
|
56
|
+
* promise into a `Failure`.
|
|
57
|
+
*
|
|
58
|
+
* @remarks
|
|
59
|
+
* **Every vector-lane hook belongs to the consumer** — the embedder, the fragment
|
|
60
|
+
* embedder, and every member of both index seams — so any of them may throw
|
|
61
|
+
* rather than fail. Left unwrapped, that escapes as a rejected promise out of
|
|
62
|
+
* `IMemoryStore.reconcile`, which the repo's Result contract says cannot happen.
|
|
63
|
+
*
|
|
64
|
+
* **Module-level and exported rather than a private of {@link VectorMaintenance},
|
|
65
|
+
* because the repair loop in `storeReconcile.ts` needs it too.** It calls
|
|
66
|
+
* `index.has` per record, which is a consumer hook like any other; when the four
|
|
67
|
+
* embed/add hooks were wrapped, that fifth one was missed, and a second private
|
|
68
|
+
* copy would have made the next omission just as easy. One hook, one helper.
|
|
69
|
+
*
|
|
70
|
+
* Kept separate from {@link VectorMaintenance._tryVectorOp} (which delegates
|
|
71
|
+
* here and adds a warn) because that warning says "best-effort; derived index
|
|
72
|
+
* left for rebuild" — true of a write, false of a repair. The repair *is* the
|
|
73
|
+
* rebuild, and it returns its failures to the caller who asked for them rather
|
|
74
|
+
* than logging them past a success.
|
|
75
|
+
* @internal
|
|
76
|
+
*/
|
|
77
|
+
export async function captureVectorHook(op, label) {
|
|
78
|
+
try {
|
|
79
|
+
return await op();
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
return fail(`${label} threw: ${String(err)}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
53
85
|
/**
|
|
54
86
|
* The store's record- and fragment-vector maintenance, extracted from
|
|
55
87
|
* {@link FileTreeMemoryStore} as a collaborator.
|
|
@@ -85,6 +117,88 @@ export class VectorMaintenance {
|
|
|
85
117
|
await this._removeVectorBestEffort(target);
|
|
86
118
|
await this._removeFragmentsBestEffort(target);
|
|
87
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* The wired record-vector index, or `undefined`. Read-only, and exposed solely
|
|
122
|
+
* so the store's {@link IMemoryStore.coverage} can report an index-side count
|
|
123
|
+
* without a second copy of the wiring — absent here IS the "lane not wired"
|
|
124
|
+
* answer that coverage reports as `undefined` rather than as zero.
|
|
125
|
+
*/
|
|
126
|
+
get vectorIndex() {
|
|
127
|
+
return this._vectorIndex;
|
|
128
|
+
}
|
|
129
|
+
/** The wired fragment index, or `undefined`. See {@link VectorMaintenance.vectorIndex}. */
|
|
130
|
+
get fragmentIndex() {
|
|
131
|
+
return this._fragmentIndex;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* The wired record embedder, or `undefined`. Exposed alongside the index
|
|
135
|
+
* because a lane is only usable when BOTH halves are present — an index with
|
|
136
|
+
* no embedder is a legal store whose writes simply do not embed, and a repair
|
|
137
|
+
* has to say so rather than failing every record.
|
|
138
|
+
*/
|
|
139
|
+
get embedder() {
|
|
140
|
+
return this._embed;
|
|
141
|
+
}
|
|
142
|
+
/** The wired fragment embedder, or `undefined`. See {@link VectorMaintenance.embedder}. */
|
|
143
|
+
get fragmentEmbedder() {
|
|
144
|
+
return this._fragmentEmbedder;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Re-embed one record into the record-vector index — the repair path.
|
|
148
|
+
*
|
|
149
|
+
* @remarks
|
|
150
|
+
* Distinct from {@link VectorMaintenance.embedOnWrite} in the one way that
|
|
151
|
+
* matters: **this is not best-effort.** Embed-on-write swallows a failure
|
|
152
|
+
* because a vault record is the source of truth and a write must not be
|
|
153
|
+
* rejected over a derived artifact; a repair was *asked for* by a caller who
|
|
154
|
+
* wants to know whether it worked, so a failure is returned.
|
|
155
|
+
*
|
|
156
|
+
* `undefined` means the embedder declined — intentionally not embedded, which
|
|
157
|
+
* is neither a repair nor a fault.
|
|
158
|
+
*/
|
|
159
|
+
async reembedRecord(record, target) {
|
|
160
|
+
if (this._vectorIndex === undefined || this._embed === undefined) {
|
|
161
|
+
return fail('the record-vector lane is not wired');
|
|
162
|
+
}
|
|
163
|
+
const index = this._vectorIndex;
|
|
164
|
+
const embed = this._embed;
|
|
165
|
+
const embedded = await captureVectorHook(() => embed(record), `re-embedding '${record.envelope.id}'`);
|
|
166
|
+
if (embedded.isFailure()) {
|
|
167
|
+
return fail(embedded.message);
|
|
168
|
+
}
|
|
169
|
+
if (embedded.value === undefined) {
|
|
170
|
+
return succeed(undefined);
|
|
171
|
+
}
|
|
172
|
+
// Hoisted: the `undefined` check above does not narrow across the callback
|
|
173
|
+
// boundary below.
|
|
174
|
+
const vector = embedded.value;
|
|
175
|
+
// `add`'s return value IS the reference the store stamps — synthesizing one
|
|
176
|
+
// here would diverge from the write path for any index whose reference is
|
|
177
|
+
// not the scoped key.
|
|
178
|
+
return (await captureVectorHook(() => index.add(target, vector), `vector add for '${record.envelope.id}'`)).onSuccess((ref) => succeed({ count: 1, ref }));
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Re-embed one record's fragments — the repair path, returning the fragment
|
|
182
|
+
* count written. See {@link VectorMaintenance.reembedRecord} for why this is
|
|
183
|
+
* not best-effort. An empty fragment array is this lane's decline and reports
|
|
184
|
+
* `undefined`, though the whole-record-replace still runs so stale fragments
|
|
185
|
+
* are cleared.
|
|
186
|
+
*/
|
|
187
|
+
async reembedFragments(record, target) {
|
|
188
|
+
if (this._fragmentIndex === undefined || this._fragmentEmbedder === undefined) {
|
|
189
|
+
return fail('the fragment lane is not wired');
|
|
190
|
+
}
|
|
191
|
+
const index = this._fragmentIndex;
|
|
192
|
+
const fragmentEmbedder = this._fragmentEmbedder;
|
|
193
|
+
const embedded = await captureVectorHook(() => fragmentEmbedder(record), `re-embedding fragments of '${record.envelope.id}'`);
|
|
194
|
+
if (embedded.isFailure()) {
|
|
195
|
+
return fail(embedded.message);
|
|
196
|
+
}
|
|
197
|
+
const fragments = embedded.value;
|
|
198
|
+
return (await captureVectorHook(() => index.addFragments(target, fragments), `fragment add for '${record.envelope.id}'`)).onSuccess((n) =>
|
|
199
|
+
// The fragment lane has no envelope reference, so `ref` is empty and unused.
|
|
200
|
+
succeed(n === 0 ? undefined : { count: n, ref: '' }));
|
|
201
|
+
}
|
|
88
202
|
/**
|
|
89
203
|
* Best-effort embed-on-write. When a vector index AND an embedder are wired,
|
|
90
204
|
* embeds the built record, `add`s the vector (replace semantics handle a same-id
|
|
@@ -222,13 +336,7 @@ export class VectorMaintenance {
|
|
|
222
336
|
* regardless, since the index is rebuildable.
|
|
223
337
|
*/
|
|
224
338
|
async _tryVectorOp(op, label) {
|
|
225
|
-
|
|
226
|
-
try {
|
|
227
|
-
result = await op();
|
|
228
|
-
}
|
|
229
|
-
catch (err) {
|
|
230
|
-
result = fail(`${label} threw: ${String(err)}`);
|
|
231
|
-
}
|
|
339
|
+
const result = await captureVectorHook(op, label);
|
|
232
340
|
if (result.isFailure()) {
|
|
233
341
|
this._warn(`memory: ${label} failed (best-effort; derived index left for rebuild): ${result.message}`);
|
|
234
342
|
}
|