@fgv/ts-agent-memory 5.1.0-49 → 5.1.0-51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/index/memoryIndex.js +23 -16
- package/dist/packlets/index/memoryIndex.js.map +1 -1
- package/dist/packlets/ingest/orchestrator.js +13 -1
- package/dist/packlets/ingest/orchestrator.js.map +1 -1
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js +94 -5
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -1
- package/dist/packlets/retrieve/linkTraversalRetriever.js +12 -26
- package/dist/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/dist/packlets/retrieve/recencyRetriever.js +7 -7
- package/dist/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/dist/packlets/retrieve/retriever.js +91 -10
- package/dist/packlets/retrieve/retriever.js.map +1 -1
- package/dist/packlets/retrieve/semanticRetriever.js +16 -16
- package/dist/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/dist/packlets/retrieve/structuredFilterRetriever.js +7 -7
- package/dist/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/dist/packlets/retrieve/tagRetriever.js +7 -7
- package/dist/packlets/retrieve/tagRetriever.js.map +1 -1
- package/dist/packlets/retrieve/temporalRetrievers.js +23 -20
- package/dist/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/dist/packlets/store/coverage.js +6 -0
- package/dist/packlets/store/coverage.js.map +1 -0
- package/dist/packlets/store/fileTreeMemoryStore.js +234 -103
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/store/index.js +4 -0
- package/dist/packlets/store/index.js.map +1 -1
- package/dist/packlets/store/listSelection.js +36 -0
- package/dist/packlets/store/listSelection.js.map +1 -0
- package/dist/packlets/store/memoryStore.js +6 -0
- package/dist/packlets/store/memoryStore.js.map +1 -0
- package/dist/packlets/store/reconcile.js +6 -0
- package/dist/packlets/store/reconcile.js.map +1 -0
- package/dist/packlets/store/storeCoverage.js +102 -0
- package/dist/packlets/store/storeCoverage.js.map +1 -0
- package/dist/packlets/store/storeIdentity.js +62 -0
- package/dist/packlets/store/storeIdentity.js.map +1 -0
- package/dist/packlets/store/storeReconcile.js +122 -0
- package/dist/packlets/store/storeReconcile.js.map +1 -0
- package/dist/packlets/store/vectorMaintenance.js +116 -8
- package/dist/packlets/store/vectorMaintenance.js.map +1 -1
- package/dist/packlets/store/vectorRecordSource.js +44 -0
- package/dist/packlets/store/vectorRecordSource.js.map +1 -0
- package/dist/packlets/tools/memoryTools.js +25 -2
- package/dist/packlets/tools/memoryTools.js.map +1 -1
- package/dist/packlets/types/envelope.js +25 -0
- package/dist/packlets/types/envelope.js.map +1 -1
- package/dist/packlets/types/identityResolver.js +6 -0
- package/dist/packlets/types/identityResolver.js.map +1 -0
- package/dist/packlets/types/index.js +2 -0
- package/dist/packlets/types/index.js.map +1 -1
- package/dist/packlets/types/recordResolver.js +6 -0
- package/dist/packlets/types/recordResolver.js.map +1 -0
- package/dist/packlets/types/temporal.js.map +1 -1
- package/dist/packlets/vector/inMemoryCosineIndex.js +39 -18
- package/dist/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +109 -14
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/dist/packlets/vector/rebuildHelpers.js +38 -0
- package/dist/packlets/vector/rebuildHelpers.js.map +1 -0
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/ts-agent-memory.d.ts +1226 -114
- package/lib/packlets/index/memoryIndex.d.ts +118 -27
- package/lib/packlets/index/memoryIndex.d.ts.map +1 -1
- package/lib/packlets/index/memoryIndex.js +23 -16
- package/lib/packlets/index/memoryIndex.js.map +1 -1
- package/lib/packlets/ingest/orchestrator.d.ts.map +1 -1
- package/lib/packlets/ingest/orchestrator.js +13 -1
- package/lib/packlets/ingest/orchestrator.js.map +1 -1
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts +79 -1
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js +94 -5
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts +3 -10
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.js +11 -25
- package/lib/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/recencyRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.js +6 -6
- package/lib/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/lib/packlets/retrieve/retriever.d.ts +88 -7
- package/lib/packlets/retrieve/retriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/retriever.js +94 -9
- package/lib/packlets/retrieve/retriever.js.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.d.ts +3 -5
- package/lib/packlets/retrieve/semanticRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.js +15 -15
- package/lib/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.js +6 -6
- package/lib/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.d.ts +3 -3
- package/lib/packlets/retrieve/tagRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.js +6 -6
- package/lib/packlets/retrieve/tagRetriever.js.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.d.ts +7 -5
- package/lib/packlets/retrieve/temporalRetrievers.d.ts.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.js +22 -19
- package/lib/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/lib/packlets/store/coverage.d.ts +102 -0
- package/lib/packlets/store/coverage.d.ts.map +1 -0
- package/lib/packlets/store/coverage.js +7 -0
- package/lib/packlets/store/coverage.js.map +1 -0
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +56 -167
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +234 -103
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/store/index.d.ts +4 -0
- package/lib/packlets/store/index.d.ts.map +1 -1
- package/lib/packlets/store/index.js +4 -0
- package/lib/packlets/store/index.js.map +1 -1
- package/lib/packlets/store/listSelection.d.ts +101 -0
- package/lib/packlets/store/listSelection.d.ts.map +1 -0
- package/lib/packlets/store/listSelection.js +40 -0
- package/lib/packlets/store/listSelection.js.map +1 -0
- package/lib/packlets/store/memoryStore.d.ts +237 -0
- package/lib/packlets/store/memoryStore.d.ts.map +1 -0
- package/lib/packlets/store/memoryStore.js +7 -0
- package/lib/packlets/store/memoryStore.js.map +1 -0
- package/lib/packlets/store/reconcile.d.ts +82 -0
- package/lib/packlets/store/reconcile.d.ts.map +1 -0
- package/lib/packlets/store/reconcile.js +7 -0
- package/lib/packlets/store/reconcile.js.map +1 -0
- package/lib/packlets/store/storeCoverage.d.ts +45 -0
- package/lib/packlets/store/storeCoverage.d.ts.map +1 -0
- package/lib/packlets/store/storeCoverage.js +105 -0
- package/lib/packlets/store/storeCoverage.js.map +1 -0
- package/lib/packlets/store/storeIdentity.d.ts +38 -0
- package/lib/packlets/store/storeIdentity.d.ts.map +1 -0
- package/lib/packlets/store/storeIdentity.js +67 -0
- package/lib/packlets/store/storeIdentity.js.map +1 -0
- package/lib/packlets/store/storeReconcile.d.ts +41 -0
- package/lib/packlets/store/storeReconcile.d.ts.map +1 -0
- package/lib/packlets/store/storeReconcile.js +125 -0
- package/lib/packlets/store/storeReconcile.js.map +1 -0
- package/lib/packlets/store/vectorMaintenance.d.ts +74 -0
- package/lib/packlets/store/vectorMaintenance.d.ts.map +1 -1
- package/lib/packlets/store/vectorMaintenance.js +117 -8
- package/lib/packlets/store/vectorMaintenance.js.map +1 -1
- package/lib/packlets/store/vectorRecordSource.d.ts +36 -0
- package/lib/packlets/store/vectorRecordSource.d.ts.map +1 -0
- package/lib/packlets/store/vectorRecordSource.js +47 -0
- package/lib/packlets/store/vectorRecordSource.js.map +1 -0
- package/lib/packlets/tools/memoryTools.d.ts.map +1 -1
- package/lib/packlets/tools/memoryTools.js +25 -2
- package/lib/packlets/tools/memoryTools.js.map +1 -1
- package/lib/packlets/types/envelope.d.ts +24 -2
- package/lib/packlets/types/envelope.d.ts.map +1 -1
- package/lib/packlets/types/envelope.js +26 -0
- package/lib/packlets/types/envelope.js.map +1 -1
- package/lib/packlets/types/identityResolver.d.ts +42 -0
- package/lib/packlets/types/identityResolver.d.ts.map +1 -0
- package/lib/packlets/types/identityResolver.js +7 -0
- package/lib/packlets/types/identityResolver.js.map +1 -0
- package/lib/packlets/types/index.d.ts +2 -0
- package/lib/packlets/types/index.d.ts.map +1 -1
- package/lib/packlets/types/index.js +2 -0
- package/lib/packlets/types/index.js.map +1 -1
- package/lib/packlets/types/recordResolver.d.ts +39 -0
- package/lib/packlets/types/recordResolver.d.ts.map +1 -0
- package/lib/packlets/types/recordResolver.js +7 -0
- package/lib/packlets/types/recordResolver.js.map +1 -0
- package/lib/packlets/types/temporal.d.ts +26 -6
- package/lib/packlets/types/temporal.d.ts.map +1 -1
- package/lib/packlets/types/temporal.js.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts +9 -2
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.js +40 -19
- package/lib/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +18 -4
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +108 -13
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/lib/packlets/vector/rebuildHelpers.d.ts +30 -0
- package/lib/packlets/vector/rebuildHelpers.d.ts.map +1 -0
- package/lib/packlets/vector/rebuildHelpers.js +42 -0
- package/lib/packlets/vector/rebuildHelpers.js.map +1 -0
- package/lib/packlets/vector/vectorIndex.d.ts +326 -20
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.computeCoverage = computeCoverage;
|
|
8
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
9
|
+
const types_1 = require("../types");
|
|
10
|
+
/** Get or create `kind`'s bucket. */
|
|
11
|
+
function bucketFor(buckets, kind) {
|
|
12
|
+
let bucket = buckets.get(kind);
|
|
13
|
+
if (bucket === undefined) {
|
|
14
|
+
bucket = { expected: 0, covered: 0 };
|
|
15
|
+
buckets.set(kind, bucket);
|
|
16
|
+
}
|
|
17
|
+
return bucket;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* One walk over the projected entries, producing every derived-state count.
|
|
21
|
+
*
|
|
22
|
+
* @remarks
|
|
23
|
+
* **Reads no record bodies and calls no embedder**, and the walk itself touches
|
|
24
|
+
* no filesystem. That is the contract `IMemoryStore.coverage` makes, and this
|
|
25
|
+
* function is where it is kept: every number below comes from an envelope field
|
|
26
|
+
* or an index-side count. The index-side counts are the one place that leaves
|
|
27
|
+
* envelope territory — on a durable backend they run a query, which is why they
|
|
28
|
+
* are captured (see the comment at the `captureResult` below) and why the walk's
|
|
29
|
+
* "no filesystem" claim is scoped to the walk rather than the whole call. A
|
|
30
|
+
* future addition that needs a body does not belong here.
|
|
31
|
+
*
|
|
32
|
+
* Extracted from `fileTreeMemoryStore.ts` because inlining it took that file past
|
|
33
|
+
* the 2000-line `max-lines` cap — the same reason `vectorRecordSource.ts` exists.
|
|
34
|
+
*/
|
|
35
|
+
function computeCoverage(params) {
|
|
36
|
+
var _a;
|
|
37
|
+
const records = new Map();
|
|
38
|
+
const rank = new Map();
|
|
39
|
+
const vectors = new Map();
|
|
40
|
+
for (const entry of params.entries) {
|
|
41
|
+
const kind = entry.envelope.kind;
|
|
42
|
+
records.set(kind, ((_a = records.get(kind)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
43
|
+
// `rank` is expected only where a projector is registered. A kind with no
|
|
44
|
+
// projector stays absent from the map rather than appearing at 0% — the store
|
|
45
|
+
// is not failing to rank it, it was never asked to.
|
|
46
|
+
if (params.hasRankProjector(kind)) {
|
|
47
|
+
const bucket = bucketFor(rank, kind);
|
|
48
|
+
bucket.expected += 1;
|
|
49
|
+
if (entry.envelope.rank !== undefined) {
|
|
50
|
+
bucket.covered += 1;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Every kind appears here, INCLUDING excluded ones: `records: 40, expected: 0`
|
|
54
|
+
// is the exclusion story stated rather than inferred, and is strictly more
|
|
55
|
+
// useful than omitting the kind entirely.
|
|
56
|
+
const bucket = bucketFor(vectors, kind);
|
|
57
|
+
if (params.embedsKind(kind)) {
|
|
58
|
+
bucket.expected += 1;
|
|
59
|
+
// Counted INSIDE the exclusion branch so `covered <= expected` holds, which
|
|
60
|
+
// is what `IArtifactCoverage.covered` promises when it says "of those". An
|
|
61
|
+
// excluded kind carrying a residual `embeddingRef` (written before
|
|
62
|
+
// `embedKinds` narrowed and never re-put) would otherwise report
|
|
63
|
+
// `expected: 0, covered: 3` and make `expected - covered` negative for a
|
|
64
|
+
// caller sizing the gap. That residue is not lost: its vector still counts
|
|
65
|
+
// toward `indexSize`, which is exactly the belief-vs-fact disagreement that
|
|
66
|
+
// field exists to surface.
|
|
67
|
+
if ((0, types_1.embeddingRefOf)(entry.envelope) !== undefined) {
|
|
68
|
+
bucket.covered += 1;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// The index-side counts are the ONE place this walk leaves envelope territory,
|
|
73
|
+
// and on a durable backend they run SQL: `SqliteVecVectorIndex.size` executes a
|
|
74
|
+
// prepared COUNT and throws on a closed connection. Capturing them keeps the
|
|
75
|
+
// `Result` contract honest — coverage FAILS rather than rejecting — and is why
|
|
76
|
+
// the "touches no filesystem" claim is scoped to the walk rather than the whole
|
|
77
|
+
// call: reading a persistent index's count is I/O, and it is the caller's own
|
|
78
|
+
// index doing it.
|
|
79
|
+
return (0, ts_utils_1.captureResult)(() => ({
|
|
80
|
+
recordVectors: params.vectorIndex === undefined ? undefined : { size: params.vectorIndex.size },
|
|
81
|
+
fragmentVectors: params.fragmentIndex === undefined
|
|
82
|
+
? undefined
|
|
83
|
+
: {
|
|
84
|
+
recordCount: params.fragmentIndex.recordCount,
|
|
85
|
+
fragmentCount: params.fragmentIndex.fragmentCount
|
|
86
|
+
}
|
|
87
|
+
}))
|
|
88
|
+
.withErrorFormat((e) => `memory coverage: reading an index count failed: ${e}`)
|
|
89
|
+
.onSuccess((counts) => (0, ts_utils_1.succeed)(Object.assign(Object.assign(Object.assign({ records }, (params.anyRankProjector ? { rank: rank } : {})), (counts.recordVectors !== undefined
|
|
90
|
+
? {
|
|
91
|
+
recordVectors: {
|
|
92
|
+
perKind: vectors,
|
|
93
|
+
indexSize: counts.recordVectors.size
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
: {})), (counts.fragmentVectors !== undefined
|
|
97
|
+
? {
|
|
98
|
+
fragmentVectors: {
|
|
99
|
+
indexRecordCount: counts.fragmentVectors.recordCount,
|
|
100
|
+
indexFragmentCount: counts.fragmentVectors.fragmentCount
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
: {}))));
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=storeCoverage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeCoverage.js","sourceRoot":"","sources":["../../../src/packlets/store/storeCoverage.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AA+DH,0CAkFC;AA/ID,4CAA+D;AAC/D,oCAAgD;AAkChD,qCAAqC;AACrC,SAAS,SAAS,CAAC,OAA4C,EAAE,IAAU;IACzE,IAAI,MAAM,GAAyC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,eAAe,CAAC,MAA8B;;IAC5D,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAC;IAC3D,MAAM,IAAI,GAAwC,IAAI,GAAG,EAAkC,CAAC;IAC5F,MAAM,OAAO,GAAwC,IAAI,GAAG,EAAkC,CAAC;IAE/F,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,IAAI,GAAS,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAA,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,8EAA8E;QAC9E,oDAAoD;QACpD,IAAI,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,MAAM,GAA6B,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACtC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,2EAA2E;QAC3E,0CAA0C;QAC1C,MAAM,MAAM,GAA6B,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAClE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrB,4EAA4E;YAC5E,2EAA2E;YAC3E,mEAAmE;YACnE,iEAAiE;YACjE,yEAAyE;YACzE,2EAA2E;YAC3E,4EAA4E;YAC5E,2BAA2B;YAC3B,IAAI,IAAA,sBAAc,EAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,gFAAgF;IAChF,6EAA6E;IAC7E,+EAA+E;IAC/E,gFAAgF;IAChF,8EAA8E;IAC9E,kBAAkB;IAClB,OAAO,IAAA,wBAAa,EAAC,GAAG,EAAE,CAAC,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE;QAC/F,eAAe,EACb,MAAM,CAAC,aAAa,KAAK,SAAS;YAChC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW;gBAC7C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,aAAa;aAClD;KACR,CAAC,CAAC;SACA,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mDAAmD,CAAC,EAAE,CAAC;SAC9E,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CACpB,IAAA,kBAAO,8CACL,OAAO,IAGJ,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAA4C,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACvF,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS;QACpC,CAAC,CAAC;YACE,aAAa,EAAE;gBACb,OAAO,EAAE,OAA+C;gBACxD,SAAS,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;aACrC;SACF;QACH,CAAC,CAAC,EAAE,CAAC,GACJ,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS;QACtC,CAAC,CAAC;YACE,eAAe,EAAE;gBACf,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC,WAAW;gBACpD,kBAAkB,EAAE,MAAM,CAAC,eAAe,CAAC,aAAa;aACzD;SACF;QACH,CAAC,CAAC,EAAE,CAAC,EACP,CACH,CAAC;AACN,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result, captureResult, succeed } from '@fgv/ts-utils';\nimport { Kind, embeddingRefOf } from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { IFragmentVectorIndex, IVectorIndex } from '../vector';\nimport { IArtifactCoverage, IDerivedStateCoverage } from './coverage';\n\n/**\n * Everything {@link computeCoverage} needs, supplied by the store.\n *\n * @remarks\n * Taken as inputs rather than reading a store, so this module has no dependency\n * on `FileTreeMemoryStore` and the walk is testable on its own. Package-internal:\n * `IMemoryStore.coverage` is the published surface.\n */\nexport interface IComputeCoverageParams {\n /** The projected index entries — envelopes only, already free of file reads. */\n readonly entries: ReadonlyArray<IIndexedMemoryEntry>;\n /** Whether `kind` has a registered rank projector. */\n readonly hasRankProjector: (kind: Kind) => boolean;\n /** Whether ANY kind has one — distinguishes an unwired lane from an empty one. */\n readonly anyRankProjector: boolean;\n /** Whether `kind` participates in the record-granular vector index. */\n readonly embedsKind: (kind: Kind) => boolean;\n /** The wired record-vector index, or `undefined` when the lane is unwired. */\n readonly vectorIndex: IVectorIndex | undefined;\n /** The wired fragment index, or `undefined` when the lane is unwired. */\n readonly fragmentIndex: IFragmentVectorIndex | undefined;\n}\n\n/** A coverage bucket under construction; the published shape is readonly. */\ninterface IMutableArtifactCoverage {\n expected: number;\n covered: number;\n}\n\n/** Get or create `kind`'s bucket. */\nfunction bucketFor(buckets: Map<Kind, IMutableArtifactCoverage>, kind: Kind): IMutableArtifactCoverage {\n let bucket: IMutableArtifactCoverage | undefined = buckets.get(kind);\n if (bucket === undefined) {\n bucket = { expected: 0, covered: 0 };\n buckets.set(kind, bucket);\n }\n return bucket;\n}\n\n/**\n * One walk over the projected entries, producing every derived-state count.\n *\n * @remarks\n * **Reads no record bodies and calls no embedder**, and the walk itself touches\n * no filesystem. That is the contract `IMemoryStore.coverage` makes, and this\n * function is where it is kept: every number below comes from an envelope field\n * or an index-side count. The index-side counts are the one place that leaves\n * envelope territory — on a durable backend they run a query, which is why they\n * are captured (see the comment at the `captureResult` below) and why the walk's\n * \"no filesystem\" claim is scoped to the walk rather than the whole call. A\n * future addition that needs a body does not belong here.\n *\n * Extracted from `fileTreeMemoryStore.ts` because inlining it took that file past\n * the 2000-line `max-lines` cap — the same reason `vectorRecordSource.ts` exists.\n */\nexport function computeCoverage(params: IComputeCoverageParams): Result<IDerivedStateCoverage> {\n const records: Map<Kind, number> = new Map<Kind, number>();\n const rank: Map<Kind, IMutableArtifactCoverage> = new Map<Kind, IMutableArtifactCoverage>();\n const vectors: Map<Kind, IMutableArtifactCoverage> = new Map<Kind, IMutableArtifactCoverage>();\n\n for (const entry of params.entries) {\n const kind: Kind = entry.envelope.kind;\n records.set(kind, (records.get(kind) ?? 0) + 1);\n\n // `rank` is expected only where a projector is registered. A kind with no\n // projector stays absent from the map rather than appearing at 0% — the store\n // is not failing to rank it, it was never asked to.\n if (params.hasRankProjector(kind)) {\n const bucket: IMutableArtifactCoverage = bucketFor(rank, kind);\n bucket.expected += 1;\n if (entry.envelope.rank !== undefined) {\n bucket.covered += 1;\n }\n }\n\n // Every kind appears here, INCLUDING excluded ones: `records: 40, expected: 0`\n // is the exclusion story stated rather than inferred, and is strictly more\n // useful than omitting the kind entirely.\n const bucket: IMutableArtifactCoverage = bucketFor(vectors, kind);\n if (params.embedsKind(kind)) {\n bucket.expected += 1;\n // Counted INSIDE the exclusion branch so `covered <= expected` holds, which\n // is what `IArtifactCoverage.covered` promises when it says \"of those\". An\n // excluded kind carrying a residual `embeddingRef` (written before\n // `embedKinds` narrowed and never re-put) would otherwise report\n // `expected: 0, covered: 3` and make `expected - covered` negative for a\n // caller sizing the gap. That residue is not lost: its vector still counts\n // toward `indexSize`, which is exactly the belief-vs-fact disagreement that\n // field exists to surface.\n if (embeddingRefOf(entry.envelope) !== undefined) {\n bucket.covered += 1;\n }\n }\n }\n\n // The index-side counts are the ONE place this walk leaves envelope territory,\n // and on a durable backend they run SQL: `SqliteVecVectorIndex.size` executes a\n // prepared COUNT and throws on a closed connection. Capturing them keeps the\n // `Result` contract honest — coverage FAILS rather than rejecting — and is why\n // the \"touches no filesystem\" claim is scoped to the walk rather than the whole\n // call: reading a persistent index's count is I/O, and it is the caller's own\n // index doing it.\n return captureResult(() => ({\n recordVectors: params.vectorIndex === undefined ? undefined : { size: params.vectorIndex.size },\n fragmentVectors:\n params.fragmentIndex === undefined\n ? undefined\n : {\n recordCount: params.fragmentIndex.recordCount,\n fragmentCount: params.fragmentIndex.fragmentCount\n }\n }))\n .withErrorFormat((e) => `memory coverage: reading an index count failed: ${e}`)\n .onSuccess((counts) =>\n succeed({\n records,\n // Absent, never zero: an unwired lane and an empty one are different facts,\n // and folding them makes a health surface alarm on a feature nobody enabled.\n ...(params.anyRankProjector ? { rank: rank as ReadonlyMap<Kind, IArtifactCoverage> } : {}),\n ...(counts.recordVectors !== undefined\n ? {\n recordVectors: {\n perKind: vectors as ReadonlyMap<Kind, IArtifactCoverage>,\n indexSize: counts.recordVectors.size\n }\n }\n : {}),\n ...(counts.fragmentVectors !== undefined\n ? {\n fragmentVectors: {\n indexRecordCount: counts.fragmentVectors.recordCount,\n indexFragmentCount: counts.fragmentVectors.fragmentCount\n }\n }\n : {})\n })\n );\n}\n"]}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { FileTree } from '@fgv/ts-json-base';
|
|
2
|
+
import { Result } from '@fgv/ts-utils';
|
|
3
|
+
import { EntityId, IIdentityCodec, IIdentityCodecResult, IMemoryRecord, Kind, MemoryScopeKey } from '../types';
|
|
4
|
+
/**
|
|
5
|
+
* The identity codec registered for `kind`, or the default.
|
|
6
|
+
*
|
|
7
|
+
* @remarks
|
|
8
|
+
* Package-internal. Extracted from `FileTreeMemoryStore` rather than living on it
|
|
9
|
+
* because the file is at its `max-lines` cap — the fourth consecutive stream to pay
|
|
10
|
+
* that toll, which `TECH_DEBT.md` names as the trigger to promote the split to P1.
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
13
|
+
export declare function codecFor(codecs: ReadonlyMap<Kind, IIdentityCodec>, defaultCodec: IIdentityCodec | undefined, kind: Kind): Result<IIdentityCodec>;
|
|
14
|
+
/**
|
|
15
|
+
* Resolve `(kind, entityId)` to the storage address the vault files it under,
|
|
16
|
+
* without reading the record.
|
|
17
|
+
*
|
|
18
|
+
* @remarks
|
|
19
|
+
* `kind` selects the codec and the codec computes the address, so this is a
|
|
20
|
+
* function rather than a search — which is what makes an `EntityId` that collides
|
|
21
|
+
* across kinds a non-issue instead of an ambiguity to disambiguate.
|
|
22
|
+
* @internal
|
|
23
|
+
*/
|
|
24
|
+
export declare function resolveIdentity(codecs: ReadonlyMap<Kind, IIdentityCodec>, defaultCodec: IIdentityCodec | undefined, kind: Kind, entityId: EntityId): Result<IIdentityCodecResult>;
|
|
25
|
+
/**
|
|
26
|
+
* Cross-check a loaded record's declared identity against the address it was read
|
|
27
|
+
* from.
|
|
28
|
+
*
|
|
29
|
+
* @remarks
|
|
30
|
+
* The filename stem and the scope are the storage-side identity; the envelope's
|
|
31
|
+
* `id` / `entityId` are what downstream code trusts verbatim (merge-into
|
|
32
|
+
* re-addressing, for one). A tampered or corrupt file declaring a foreign
|
|
33
|
+
* `entityId` would otherwise load undetected, so the two are reconciled here
|
|
34
|
+
* through the codec's own round-trip.
|
|
35
|
+
* @internal
|
|
36
|
+
*/
|
|
37
|
+
export declare function verifyLoadedIdentity(codec: Result<IIdentityCodec>, scope: MemoryScopeKey, file: FileTree.IFileTreeFileItem, record: IMemoryRecord<unknown>): Result<IMemoryRecord<unknown>>;
|
|
38
|
+
//# sourceMappingURL=storeIdentity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeIdentity.d.ts","sourceRoot":"","sources":["../../../src/packlets/store/storeIdentity.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAiB,MAAM,eAAe,CAAC;AACtD,OAAO,EACL,QAAQ,EACR,cAAc,EACd,oBAAoB,EACpB,aAAa,EACb,IAAI,EACJ,cAAc,EACf,MAAM,UAAU,CAAC;AAElB;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CACtB,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,cAAc,CAAC,EACzC,YAAY,EAAE,cAAc,GAAG,SAAS,EACxC,IAAI,EAAE,IAAI,GACT,MAAM,CAAC,cAAc,CAAC,CAMxB;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,cAAc,CAAC,EACzC,YAAY,EAAE,cAAc,GAAG,SAAS,EACxC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAAC,oBAAoB,CAAC,CAE9B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC,EAC7B,KAAK,EAAE,cAAc,EACrB,IAAI,EAAE,QAAQ,CAAC,iBAAiB,EAChC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAC7B,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAiBhC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.codecFor = codecFor;
|
|
8
|
+
exports.resolveIdentity = resolveIdentity;
|
|
9
|
+
exports.verifyLoadedIdentity = verifyLoadedIdentity;
|
|
10
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
11
|
+
/**
|
|
12
|
+
* The identity codec registered for `kind`, or the default.
|
|
13
|
+
*
|
|
14
|
+
* @remarks
|
|
15
|
+
* Package-internal. Extracted from `FileTreeMemoryStore` rather than living on it
|
|
16
|
+
* because the file is at its `max-lines` cap — the fourth consecutive stream to pay
|
|
17
|
+
* that toll, which `TECH_DEBT.md` names as the trigger to promote the split to P1.
|
|
18
|
+
* @internal
|
|
19
|
+
*/
|
|
20
|
+
function codecFor(codecs, defaultCodec, kind) {
|
|
21
|
+
var _a;
|
|
22
|
+
const codec = (_a = codecs.get(kind)) !== null && _a !== void 0 ? _a : defaultCodec;
|
|
23
|
+
if (codec === undefined) {
|
|
24
|
+
return (0, ts_utils_1.fail)(`no identity codec registered for kind '${kind}'`);
|
|
25
|
+
}
|
|
26
|
+
return (0, ts_utils_1.succeed)(codec);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Resolve `(kind, entityId)` to the storage address the vault files it under,
|
|
30
|
+
* without reading the record.
|
|
31
|
+
*
|
|
32
|
+
* @remarks
|
|
33
|
+
* `kind` selects the codec and the codec computes the address, so this is a
|
|
34
|
+
* function rather than a search — which is what makes an `EntityId` that collides
|
|
35
|
+
* across kinds a non-issue instead of an ambiguity to disambiguate.
|
|
36
|
+
* @internal
|
|
37
|
+
*/
|
|
38
|
+
function resolveIdentity(codecs, defaultCodec, kind, entityId) {
|
|
39
|
+
return codecFor(codecs, defaultCodec, kind).onSuccess((codec) => codec.encode(entityId));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Cross-check a loaded record's declared identity against the address it was read
|
|
43
|
+
* from.
|
|
44
|
+
*
|
|
45
|
+
* @remarks
|
|
46
|
+
* The filename stem and the scope are the storage-side identity; the envelope's
|
|
47
|
+
* `id` / `entityId` are what downstream code trusts verbatim (merge-into
|
|
48
|
+
* re-addressing, for one). A tampered or corrupt file declaring a foreign
|
|
49
|
+
* `entityId` would otherwise load undetected, so the two are reconciled here
|
|
50
|
+
* through the codec's own round-trip.
|
|
51
|
+
* @internal
|
|
52
|
+
*/
|
|
53
|
+
function verifyLoadedIdentity(codec, scope, file, record) {
|
|
54
|
+
if (record.envelope.id !== file.baseName) {
|
|
55
|
+
return (0, ts_utils_1.fail)(`memory file '${file.absolutePath}': envelope id '${record.envelope.id}' does not match filename stem '${file.baseName}'`);
|
|
56
|
+
}
|
|
57
|
+
return codec
|
|
58
|
+
.onSuccess((c) => c.verifyRoundTrip(scope, file.baseName).onSuccess(() => c.decode(scope, file.baseName)))
|
|
59
|
+
.withErrorFormat((msg) => `memory file '${file.absolutePath}': ${msg}`)
|
|
60
|
+
.onSuccess((decodedEntityId) => {
|
|
61
|
+
if (decodedEntityId !== record.envelope.entityId) {
|
|
62
|
+
return (0, ts_utils_1.fail)(`memory file '${file.absolutePath}': envelope entityId '${record.envelope.entityId}' does not match scope-derived entityId '${decodedEntityId}'`);
|
|
63
|
+
}
|
|
64
|
+
return (0, ts_utils_1.succeed)(record);
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=storeIdentity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeIdentity.js","sourceRoot":"","sources":["../../../src/packlets/store/storeIdentity.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAsBH,4BAUC;AAYD,0CAOC;AAcD,oDAsBC;AApFD,4CAAsD;AAUtD;;;;;;;;GAQG;AACH,SAAgB,QAAQ,CACtB,MAAyC,EACzC,YAAwC,EACxC,IAAU;;IAEV,MAAM,KAAK,GAA+B,MAAA,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,YAAY,CAAC;IAC3E,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,IAAA,eAAI,EAAC,0CAA0C,IAAI,GAAG,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,IAAA,kBAAO,EAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAgB,eAAe,CAC7B,MAAyC,EACzC,YAAwC,EACxC,IAAU,EACV,QAAkB;IAElB,OAAO,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,oBAAoB,CAClC,KAA6B,EAC7B,KAAqB,EACrB,IAAgC,EAChC,MAA8B;IAE9B,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,OAAO,IAAA,eAAI,EACT,gBAAgB,IAAI,CAAC,YAAY,mBAAmB,MAAM,CAAC,QAAQ,CAAC,EAAE,mCAAmC,IAAI,CAAC,QAAQ,GAAG,CAC1H,CAAC;IACJ,CAAC;IACD,OAAO,KAAK;SACT,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;SACzG,eAAe,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,IAAI,CAAC,YAAY,MAAM,GAAG,EAAE,CAAC;SACtE,SAAS,CAAC,CAAC,eAAe,EAAE,EAAE;QAC7B,IAAI,eAAe,KAAK,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACjD,OAAO,IAAA,eAAI,EACT,gBAAgB,IAAI,CAAC,YAAY,yBAAyB,MAAM,CAAC,QAAQ,CAAC,QAAQ,4CAA4C,eAAe,GAAG,CACjJ,CAAC;QACJ,CAAC;QACD,OAAO,IAAA,kBAAO,EAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { FileTree } from '@fgv/ts-json-base';\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport {\n EntityId,\n IIdentityCodec,\n IIdentityCodecResult,\n IMemoryRecord,\n Kind,\n MemoryScopeKey\n} from '../types';\n\n/**\n * The identity codec registered for `kind`, or the default.\n *\n * @remarks\n * Package-internal. Extracted from `FileTreeMemoryStore` rather than living on it\n * because the file is at its `max-lines` cap — the fourth consecutive stream to pay\n * that toll, which `TECH_DEBT.md` names as the trigger to promote the split to P1.\n * @internal\n */\nexport function codecFor(\n codecs: ReadonlyMap<Kind, IIdentityCodec>,\n defaultCodec: IIdentityCodec | undefined,\n kind: Kind\n): Result<IIdentityCodec> {\n const codec: IIdentityCodec | undefined = codecs.get(kind) ?? defaultCodec;\n if (codec === undefined) {\n return fail(`no identity codec registered for kind '${kind}'`);\n }\n return succeed(codec);\n}\n\n/**\n * Resolve `(kind, entityId)` to the storage address the vault files it under,\n * without reading the record.\n *\n * @remarks\n * `kind` selects the codec and the codec computes the address, so this is a\n * function rather than a search — which is what makes an `EntityId` that collides\n * across kinds a non-issue instead of an ambiguity to disambiguate.\n * @internal\n */\nexport function resolveIdentity(\n codecs: ReadonlyMap<Kind, IIdentityCodec>,\n defaultCodec: IIdentityCodec | undefined,\n kind: Kind,\n entityId: EntityId\n): Result<IIdentityCodecResult> {\n return codecFor(codecs, defaultCodec, kind).onSuccess((codec) => codec.encode(entityId));\n}\n\n/**\n * Cross-check a loaded record's declared identity against the address it was read\n * from.\n *\n * @remarks\n * The filename stem and the scope are the storage-side identity; the envelope's\n * `id` / `entityId` are what downstream code trusts verbatim (merge-into\n * re-addressing, for one). A tampered or corrupt file declaring a foreign\n * `entityId` would otherwise load undetected, so the two are reconciled here\n * through the codec's own round-trip.\n * @internal\n */\nexport function verifyLoadedIdentity(\n codec: Result<IIdentityCodec>,\n scope: MemoryScopeKey,\n file: FileTree.IFileTreeFileItem,\n record: IMemoryRecord<unknown>\n): Result<IMemoryRecord<unknown>> {\n if (record.envelope.id !== file.baseName) {\n return fail(\n `memory file '${file.absolutePath}': envelope id '${record.envelope.id}' does not match filename stem '${file.baseName}'`\n );\n }\n return codec\n .onSuccess((c) => c.verifyRoundTrip(scope, file.baseName).onSuccess(() => c.decode(scope, file.baseName)))\n .withErrorFormat((msg) => `memory file '${file.absolutePath}': ${msg}`)\n .onSuccess((decodedEntityId) => {\n if (decodedEntityId !== record.envelope.entityId) {\n return fail(\n `memory file '${file.absolutePath}': envelope entityId '${record.envelope.entityId}' does not match scope-derived entityId '${decodedEntityId}'`\n );\n }\n return succeed(record);\n });\n}\n"]}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Result } from '@fgv/ts-utils';
|
|
2
|
+
import { IMemoryRecord, Kind, MemoryId, MemoryScopeKey } from '../types';
|
|
3
|
+
import { IIndexedMemoryEntry } from '../index';
|
|
4
|
+
import { DerivedArtifact, ReconcileReport } from './reconcile';
|
|
5
|
+
import { VectorMaintenance } from './vectorMaintenance';
|
|
6
|
+
/**
|
|
7
|
+
* Everything the vector reconcile branches need, supplied by the store.
|
|
8
|
+
* Package-internal; `IMemoryStore.reconcile` is the published surface.
|
|
9
|
+
*/
|
|
10
|
+
export interface IReconcileVectorsParams {
|
|
11
|
+
readonly kind: Kind;
|
|
12
|
+
readonly artifact: Exclude<DerivedArtifact, 'rank'>;
|
|
13
|
+
/** The kind's index entries — envelopes only, no bodies read yet. */
|
|
14
|
+
readonly targets: ReadonlyArray<IIndexedMemoryEntry>;
|
|
15
|
+
readonly maintenance: VectorMaintenance;
|
|
16
|
+
readonly embedsKind: (kind: Kind) => boolean;
|
|
17
|
+
/** Materialize one record; `undefined` when it has vanished. */
|
|
18
|
+
readonly resolve: (scope: MemoryScopeKey, id: MemoryId) => Result<IMemoryRecord<unknown> | undefined>;
|
|
19
|
+
/** Write an `embeddingRef` onto a record's envelope; `true` when it changed. */
|
|
20
|
+
readonly stampRef: (scope: MemoryScopeKey, id: MemoryId, ref: string) => Result<boolean>;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Repair one vector lane for one kind, touching only what is missing.
|
|
24
|
+
*
|
|
25
|
+
* @remarks
|
|
26
|
+
* **The `has` check is what makes this targeted rather than a rebuild.** A record
|
|
27
|
+
* the index already holds costs one membership query and no embedder call, which
|
|
28
|
+
* is the entire difference between repairing a brief outage and re-embedding a
|
|
29
|
+
* vault. It is also the only way to see the record-lane case where the index
|
|
30
|
+
* holds a vector but the envelope lost its `embeddingRef` — that record needs a
|
|
31
|
+
* restamp and no embedder call, and is indistinguishable from a never-embedded
|
|
32
|
+
* one if you only look at the envelope.
|
|
33
|
+
*
|
|
34
|
+
* Per-record failures are collected rather than fatal: a repair that stops at the
|
|
35
|
+
* first bad record leaves the rest of the gap open, which is the opposite of what
|
|
36
|
+
* a caller asked for. A failure to *materialize* is likewise collected — a record
|
|
37
|
+
* that vanished between the walk and the read is a casualty of this call, not a
|
|
38
|
+
* reason to abandon the others.
|
|
39
|
+
*/
|
|
40
|
+
export declare function reconcileVectors(params: IReconcileVectorsParams): Promise<Result<ReconcileReport>>;
|
|
41
|
+
//# sourceMappingURL=storeReconcile.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeReconcile.d.ts","sourceRoot":"","sources":["../../../src/packlets/store/storeReconcile.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAiB,MAAM,eAAe,CAAC;AACtD,OAAO,EAEL,aAAa,EACb,IAAI,EACJ,QAAQ,EACR,cAAc,EAGf,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAmB,iBAAiB,EAAqB,MAAM,qBAAqB,CAAC;AAE5F;;;GAGG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACpD,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,mBAAmB,CAAC,CAAC;IACrD,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,QAAQ,KAAK,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IACtG,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC;CAC1F;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAwHxG"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.reconcileVectors = reconcileVectors;
|
|
8
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
9
|
+
const types_1 = require("../types");
|
|
10
|
+
const vectorMaintenance_1 = require("./vectorMaintenance");
|
|
11
|
+
/**
|
|
12
|
+
* Repair one vector lane for one kind, touching only what is missing.
|
|
13
|
+
*
|
|
14
|
+
* @remarks
|
|
15
|
+
* **The `has` check is what makes this targeted rather than a rebuild.** A record
|
|
16
|
+
* the index already holds costs one membership query and no embedder call, which
|
|
17
|
+
* is the entire difference between repairing a brief outage and re-embedding a
|
|
18
|
+
* vault. It is also the only way to see the record-lane case where the index
|
|
19
|
+
* holds a vector but the envelope lost its `embeddingRef` — that record needs a
|
|
20
|
+
* restamp and no embedder call, and is indistinguishable from a never-embedded
|
|
21
|
+
* one if you only look at the envelope.
|
|
22
|
+
*
|
|
23
|
+
* Per-record failures are collected rather than fatal: a repair that stops at the
|
|
24
|
+
* first bad record leaves the rest of the gap open, which is the opposite of what
|
|
25
|
+
* a caller asked for. A failure to *materialize* is likewise collected — a record
|
|
26
|
+
* that vanished between the walk and the read is a casualty of this call, not a
|
|
27
|
+
* reason to abandon the others.
|
|
28
|
+
*/
|
|
29
|
+
async function reconcileVectors(params) {
|
|
30
|
+
const fragment = params.artifact === 'fragment-vector';
|
|
31
|
+
const index = fragment ? params.maintenance.fragmentIndex : params.maintenance.vectorIndex;
|
|
32
|
+
// BOTH halves, not just the index: an index wired without an embedder is a
|
|
33
|
+
// legal store (queries work, writes simply do not embed), and reconciling it
|
|
34
|
+
// would otherwise report a cheerful success with every record in `failed`.
|
|
35
|
+
// "Half the lane is wired" is a wiring mistake and says so.
|
|
36
|
+
const embedder = fragment ? params.maintenance.fragmentEmbedder : params.maintenance.embedder;
|
|
37
|
+
if (index === undefined || embedder === undefined) {
|
|
38
|
+
return (0, ts_utils_1.fail)(`memory reconcile '${params.kind}' ${params.artifact}: the ${fragment ? 'fragment' : 'record-vector'} lane is not fully wired (index: ${index === undefined ? 'absent' : 'present'}, embedder: ${embedder === undefined ? 'absent' : 'present'})`);
|
|
39
|
+
}
|
|
40
|
+
if (!fragment && !params.embedsKind(params.kind)) {
|
|
41
|
+
// Not a failure and not a silent no-op: the caller asked to repair a kind the
|
|
42
|
+
// store is deliberately not embedding, and the honest answer names that
|
|
43
|
+
// rather than reporting a healthy zero.
|
|
44
|
+
return (0, ts_utils_1.fail)(`memory reconcile '${params.kind}' record-vector: this kind is excluded from the record vector index`);
|
|
45
|
+
}
|
|
46
|
+
let repaired = 0;
|
|
47
|
+
let restamped = 0;
|
|
48
|
+
let declined = 0;
|
|
49
|
+
let alreadyIndexed = 0;
|
|
50
|
+
let fragments = 0;
|
|
51
|
+
const failed = [];
|
|
52
|
+
for (const entry of params.targets) {
|
|
53
|
+
const target = { scope: entry.scope, id: entry.envelope.id };
|
|
54
|
+
// Captured like every other consumer hook: `has` belongs to the injected
|
|
55
|
+
// index and may throw rather than fail. Unwrapped it would reject the whole
|
|
56
|
+
// `IMemoryStore.reconcile` call on one bad record, which is both a Result-
|
|
57
|
+
// contract break and the opposite of this loop's collect-and-continue intent.
|
|
58
|
+
const held = await (0, vectorMaintenance_1.captureVectorHook)(() => index.has(target), `membership check for '${entry.envelope.id}'`);
|
|
59
|
+
if (held.isFailure()) {
|
|
60
|
+
failed.push({ target, error: `membership check failed: ${held.message}` });
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (held.value) {
|
|
64
|
+
// The record lane can still be inconsistent while the index holds the
|
|
65
|
+
// vector: a reference lost after the vector was committed. Repairing it
|
|
66
|
+
// costs a write and no embedding.
|
|
67
|
+
if (!fragment && (0, types_1.embeddingRefOf)(entry.envelope) === undefined) {
|
|
68
|
+
// The one place a synthesized reference is unavoidable: `has` proved the
|
|
69
|
+
// vector exists but there is no contract member that returns the
|
|
70
|
+
// reference the index minted for it, and re-deriving one would cost the
|
|
71
|
+
// embedder call this branch exists to avoid. Sound for both shipped
|
|
72
|
+
// indexes, whose reference IS the scoped key; a third-party index that
|
|
73
|
+
// mints something else gets the scoped key stamped here. Recorded in
|
|
74
|
+
// `docs/FUTURE.md` rather than silently assumed.
|
|
75
|
+
const stamped = params.stampRef(entry.scope, entry.envelope.id, (0, types_1.edgeTargetKey)(target));
|
|
76
|
+
if (stamped.isFailure()) {
|
|
77
|
+
failed.push({ target, error: `restamping the embedding reference failed: ${stamped.message}` });
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
restamped++;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
alreadyIndexed++;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const resolved = params.resolve(entry.scope, entry.envelope.id);
|
|
87
|
+
if (resolved.isFailure()) {
|
|
88
|
+
failed.push({ target, error: `reading the record failed: ${resolved.message}` });
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (resolved.value === undefined) {
|
|
92
|
+
failed.push({ target, error: 'the index claims this record but the vault has no such file' });
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const outcome = await (fragment
|
|
96
|
+
? params.maintenance.reembedFragments(resolved.value, target)
|
|
97
|
+
: params.maintenance.reembedRecord(resolved.value, target));
|
|
98
|
+
if (outcome.isFailure()) {
|
|
99
|
+
failed.push({ target, error: outcome.message });
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (outcome.value === undefined) {
|
|
103
|
+
declined++;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (fragment) {
|
|
107
|
+
fragments += outcome.value.count;
|
|
108
|
+
repaired++;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
// The reference the INDEX returned, never a synthesized key: the write path
|
|
112
|
+
// persists `add`'s value, so a third-party index whose reference is not the
|
|
113
|
+
// scoped key would otherwise get one stamp from `put` and a different one here.
|
|
114
|
+
const stamped = params.stampRef(entry.scope, entry.envelope.id, outcome.value.ref);
|
|
115
|
+
if (stamped.isFailure()) {
|
|
116
|
+
failed.push({ target, error: `stamping the embedding reference failed: ${stamped.message}` });
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
repaired++;
|
|
120
|
+
}
|
|
121
|
+
const base = { kind: params.kind, examined: params.targets.length, repaired, failed };
|
|
122
|
+
return (0, ts_utils_1.succeed)(fragment
|
|
123
|
+
? Object.assign(Object.assign({}, base), { artifact: 'fragment-vector', alreadyIndexed, declined, fragments }) : Object.assign(Object.assign({}, base), { artifact: 'record-vector', alreadyIndexed, restamped, declined }));
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=storeReconcile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeReconcile.js","sourceRoot":"","sources":["../../../src/packlets/store/storeReconcile.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAoDH,4CAwHC;AA1KD,4CAAsD;AACtD,oCAQkB;AAIlB,2DAA4F;AAmB5F;;;;;;;;;;;;;;;;;GAiBG;AACI,KAAK,UAAU,gBAAgB,CAAC,MAA+B;IACpE,MAAM,QAAQ,GAAY,MAAM,CAAC,QAAQ,KAAK,iBAAiB,CAAC;IAChE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC;IAC3F,2EAA2E;IAC3E,6EAA6E;IAC7E,2EAA2E;IAC3E,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC;IAC9F,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,IAAA,eAAI,EACT,qBAAqB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,QAAQ,SAClD,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAC1B,oCAAoC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,eAC5E,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SACtC,GAAG,CACJ,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,8EAA8E;QAC9E,wEAAwE;QACxE,wCAAwC;QACxC,OAAO,IAAA,eAAI,EACT,qBAAqB,MAAM,CAAC,IAAI,qEAAqE,CACtG,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAW,CAAC,CAAC;IACzB,IAAI,SAAS,GAAW,CAAC,CAAC;IAC1B,IAAI,QAAQ,GAAW,CAAC,CAAC;IACzB,IAAI,cAAc,GAAW,CAAC,CAAC;IAC/B,IAAI,SAAS,GAAW,CAAC,CAAC;IAC1B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC1E,yEAAyE;QACzE,4EAA4E;QAC5E,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,IAAI,GAAoB,MAAM,IAAA,qCAAiB,EACnD,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EACvB,yBAAyB,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,CAC9C,CAAC;QACF,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,4BAA4B,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC3E,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,sEAAsE;YACtE,wEAAwE;YACxE,kCAAkC;YAClC,IAAI,CAAC,QAAQ,IAAI,IAAA,sBAAc,EAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9D,yEAAyE;gBACzE,iEAAiE;gBACjE,wEAAwE;gBACxE,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,OAAO,GAAoB,MAAM,CAAC,QAAQ,CAC9C,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,CAAC,EAAE,EACjB,IAAA,qBAAa,EAAC,MAAM,CAAC,CACtB,CAAC;gBACF,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;oBACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,8CAA8C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBAChG,SAAS;gBACX,CAAC;gBACD,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAA+C,MAAM,CAAC,OAAO,CACzE,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,CAAC,EAAE,CAClB,CAAC;QACF,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjF,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,6DAA6D,EAAE,CAAC,CAAC;YAC9F,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAwC,MAAM,CAAC,QAAQ;YAClE,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YAC7D,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;YACjC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,4EAA4E;QAC5E,4EAA4E;QAC5E,gFAAgF;QAChF,MAAM,OAAO,GAAoB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpG,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,4CAA4C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC9F,SAAS;QACX,CAAC;QACD,QAAQ,EAAE,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACtF,OAAO,IAAA,kBAAO,EACZ,QAAQ;QACN,CAAC,iCAAM,IAAI,KAAE,QAAQ,EAAE,iBAAiB,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,IAC7E,CAAC,iCAAM,IAAI,KAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,QAAQ,GAAE,CAChF,CAAC;AACJ,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport {\n IEdgeTarget,\n IMemoryRecord,\n Kind,\n MemoryId,\n MemoryScopeKey,\n edgeTargetKey,\n embeddingRefOf\n} from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { ISkippedVectorRecord } from '../vector';\nimport { DerivedArtifact, ReconcileReport } from './reconcile';\nimport { IReembedOutcome, VectorMaintenance, captureVectorHook } from './vectorMaintenance';\n\n/**\n * Everything the vector reconcile branches need, supplied by the store.\n * Package-internal; `IMemoryStore.reconcile` is the published surface.\n */\nexport interface IReconcileVectorsParams {\n readonly kind: Kind;\n readonly artifact: Exclude<DerivedArtifact, 'rank'>;\n /** The kind's index entries — envelopes only, no bodies read yet. */\n readonly targets: ReadonlyArray<IIndexedMemoryEntry>;\n readonly maintenance: VectorMaintenance;\n readonly embedsKind: (kind: Kind) => boolean;\n /** Materialize one record; `undefined` when it has vanished. */\n readonly resolve: (scope: MemoryScopeKey, id: MemoryId) => Result<IMemoryRecord<unknown> | undefined>;\n /** Write an `embeddingRef` onto a record's envelope; `true` when it changed. */\n readonly stampRef: (scope: MemoryScopeKey, id: MemoryId, ref: string) => Result<boolean>;\n}\n\n/**\n * Repair one vector lane for one kind, touching only what is missing.\n *\n * @remarks\n * **The `has` check is what makes this targeted rather than a rebuild.** A record\n * the index already holds costs one membership query and no embedder call, which\n * is the entire difference between repairing a brief outage and re-embedding a\n * vault. It is also the only way to see the record-lane case where the index\n * holds a vector but the envelope lost its `embeddingRef` — that record needs a\n * restamp and no embedder call, and is indistinguishable from a never-embedded\n * one if you only look at the envelope.\n *\n * Per-record failures are collected rather than fatal: a repair that stops at the\n * first bad record leaves the rest of the gap open, which is the opposite of what\n * a caller asked for. A failure to *materialize* is likewise collected — a record\n * that vanished between the walk and the read is a casualty of this call, not a\n * reason to abandon the others.\n */\nexport async function reconcileVectors(params: IReconcileVectorsParams): Promise<Result<ReconcileReport>> {\n const fragment: boolean = params.artifact === 'fragment-vector';\n const index = fragment ? params.maintenance.fragmentIndex : params.maintenance.vectorIndex;\n // BOTH halves, not just the index: an index wired without an embedder is a\n // legal store (queries work, writes simply do not embed), and reconciling it\n // would otherwise report a cheerful success with every record in `failed`.\n // \"Half the lane is wired\" is a wiring mistake and says so.\n const embedder = fragment ? params.maintenance.fragmentEmbedder : params.maintenance.embedder;\n if (index === undefined || embedder === undefined) {\n return fail(\n `memory reconcile '${params.kind}' ${params.artifact}: the ${\n fragment ? 'fragment' : 'record-vector'\n } lane is not fully wired (index: ${index === undefined ? 'absent' : 'present'}, embedder: ${\n embedder === undefined ? 'absent' : 'present'\n })`\n );\n }\n if (!fragment && !params.embedsKind(params.kind)) {\n // Not a failure and not a silent no-op: the caller asked to repair a kind the\n // store is deliberately not embedding, and the honest answer names that\n // rather than reporting a healthy zero.\n return fail(\n `memory reconcile '${params.kind}' record-vector: this kind is excluded from the record vector index`\n );\n }\n\n let repaired: number = 0;\n let restamped: number = 0;\n let declined: number = 0;\n let alreadyIndexed: number = 0;\n let fragments: number = 0;\n const failed: ISkippedVectorRecord[] = [];\n\n for (const entry of params.targets) {\n const target: IEdgeTarget = { scope: entry.scope, id: entry.envelope.id };\n // Captured like every other consumer hook: `has` belongs to the injected\n // index and may throw rather than fail. Unwrapped it would reject the whole\n // `IMemoryStore.reconcile` call on one bad record, which is both a Result-\n // contract break and the opposite of this loop's collect-and-continue intent.\n const held: Result<boolean> = await captureVectorHook(\n () => index.has(target),\n `membership check for '${entry.envelope.id}'`\n );\n if (held.isFailure()) {\n failed.push({ target, error: `membership check failed: ${held.message}` });\n continue;\n }\n if (held.value) {\n // The record lane can still be inconsistent while the index holds the\n // vector: a reference lost after the vector was committed. Repairing it\n // costs a write and no embedding.\n if (!fragment && embeddingRefOf(entry.envelope) === undefined) {\n // The one place a synthesized reference is unavoidable: `has` proved the\n // vector exists but there is no contract member that returns the\n // reference the index minted for it, and re-deriving one would cost the\n // embedder call this branch exists to avoid. Sound for both shipped\n // indexes, whose reference IS the scoped key; a third-party index that\n // mints something else gets the scoped key stamped here. Recorded in\n // `docs/FUTURE.md` rather than silently assumed.\n const stamped: Result<boolean> = params.stampRef(\n entry.scope,\n entry.envelope.id,\n edgeTargetKey(target)\n );\n if (stamped.isFailure()) {\n failed.push({ target, error: `restamping the embedding reference failed: ${stamped.message}` });\n continue;\n }\n restamped++;\n continue;\n }\n alreadyIndexed++;\n continue;\n }\n\n const resolved: Result<IMemoryRecord<unknown> | undefined> = params.resolve(\n entry.scope,\n entry.envelope.id\n );\n if (resolved.isFailure()) {\n failed.push({ target, error: `reading the record failed: ${resolved.message}` });\n continue;\n }\n if (resolved.value === undefined) {\n failed.push({ target, error: 'the index claims this record but the vault has no such file' });\n continue;\n }\n const outcome: Result<IReembedOutcome | undefined> = await (fragment\n ? params.maintenance.reembedFragments(resolved.value, target)\n : params.maintenance.reembedRecord(resolved.value, target));\n if (outcome.isFailure()) {\n failed.push({ target, error: outcome.message });\n continue;\n }\n if (outcome.value === undefined) {\n declined++;\n continue;\n }\n if (fragment) {\n fragments += outcome.value.count;\n repaired++;\n continue;\n }\n // The reference the INDEX returned, never a synthesized key: the write path\n // persists `add`'s value, so a third-party index whose reference is not the\n // scoped key would otherwise get one stamp from `put` and a different one here.\n const stamped: Result<boolean> = params.stampRef(entry.scope, entry.envelope.id, outcome.value.ref);\n if (stamped.isFailure()) {\n failed.push({ target, error: `stamping the embedding reference failed: ${stamped.message}` });\n continue;\n }\n repaired++;\n }\n\n const base = { kind: params.kind, examined: params.targets.length, repaired, failed };\n return succeed(\n fragment\n ? { ...base, artifact: 'fragment-vector', alreadyIndexed, declined, fragments }\n : { ...base, artifact: 'record-vector', alreadyIndexed, restamped, declined }\n );\n}\n"]}
|
|
@@ -2,6 +2,16 @@ import { Result } from '@fgv/ts-utils';
|
|
|
2
2
|
import { MemoryEmbedOutcome } from '../observe';
|
|
3
3
|
import { IEdgeTarget, IMemoryRecord, Kind, MemoryId, MemoryScopeKey } from '../types';
|
|
4
4
|
import { FragmentEmbedder, IFragmentVectorIndex, IVectorIndex, MemoryEmbedder } from '../vector';
|
|
5
|
+
/**
|
|
6
|
+
* What a repair-path re-embed established: how many vectors/fragments were
|
|
7
|
+
* written, and — record lane only — the reference the index returned, which is
|
|
8
|
+
* what the store stamps onto the envelope.
|
|
9
|
+
*/
|
|
10
|
+
export interface IReembedOutcome {
|
|
11
|
+
readonly count: number;
|
|
12
|
+
/** The index-supplied reference. Empty on the fragment lane, which has none. */
|
|
13
|
+
readonly ref: string;
|
|
14
|
+
}
|
|
5
15
|
/**
|
|
6
16
|
* The internal outcome of record-level embed-on-write: the record to persist,
|
|
7
17
|
* plus — only when the embedder declined a record that already carried an
|
|
@@ -25,6 +35,30 @@ export interface IEmbedOnWriteOutcome {
|
|
|
25
35
|
*/
|
|
26
36
|
readonly embed?: MemoryEmbedOutcome;
|
|
27
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Run a consumer-supplied hook, normalizing a synchronous throw or a rejected
|
|
40
|
+
* promise into a `Failure`.
|
|
41
|
+
*
|
|
42
|
+
* @remarks
|
|
43
|
+
* **Every vector-lane hook belongs to the consumer** — the embedder, the fragment
|
|
44
|
+
* embedder, and every member of both index seams — so any of them may throw
|
|
45
|
+
* rather than fail. Left unwrapped, that escapes as a rejected promise out of
|
|
46
|
+
* `IMemoryStore.reconcile`, which the repo's Result contract says cannot happen.
|
|
47
|
+
*
|
|
48
|
+
* **Module-level and exported rather than a private of {@link VectorMaintenance},
|
|
49
|
+
* because the repair loop in `storeReconcile.ts` needs it too.** It calls
|
|
50
|
+
* `index.has` per record, which is a consumer hook like any other; when the four
|
|
51
|
+
* embed/add hooks were wrapped, that fifth one was missed, and a second private
|
|
52
|
+
* copy would have made the next omission just as easy. One hook, one helper.
|
|
53
|
+
*
|
|
54
|
+
* Kept separate from {@link VectorMaintenance._tryVectorOp} (which delegates
|
|
55
|
+
* here and adds a warn) because that warning says "best-effort; derived index
|
|
56
|
+
* left for rebuild" — true of a write, false of a repair. The repair *is* the
|
|
57
|
+
* rebuild, and it returns its failures to the caller who asked for them rather
|
|
58
|
+
* than logging them past a success.
|
|
59
|
+
* @internal
|
|
60
|
+
*/
|
|
61
|
+
export declare function captureVectorHook<T>(op: () => Promise<Result<T>>, label: string): Promise<Result<T>>;
|
|
28
62
|
/**
|
|
29
63
|
* What {@link VectorMaintenance} needs from the store it serves. Every field is
|
|
30
64
|
* the store's own, passed in rather than reached for, so this collaborator holds
|
|
@@ -76,6 +110,46 @@ export declare class VectorMaintenance {
|
|
|
76
110
|
* committed by the time they call it.
|
|
77
111
|
*/
|
|
78
112
|
removeAll(target: IEdgeTarget): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* The wired record-vector index, or `undefined`. Read-only, and exposed solely
|
|
115
|
+
* so the store's {@link IMemoryStore.coverage} can report an index-side count
|
|
116
|
+
* without a second copy of the wiring — absent here IS the "lane not wired"
|
|
117
|
+
* answer that coverage reports as `undefined` rather than as zero.
|
|
118
|
+
*/
|
|
119
|
+
get vectorIndex(): IVectorIndex | undefined;
|
|
120
|
+
/** The wired fragment index, or `undefined`. See {@link VectorMaintenance.vectorIndex}. */
|
|
121
|
+
get fragmentIndex(): IFragmentVectorIndex | undefined;
|
|
122
|
+
/**
|
|
123
|
+
* The wired record embedder, or `undefined`. Exposed alongside the index
|
|
124
|
+
* because a lane is only usable when BOTH halves are present — an index with
|
|
125
|
+
* no embedder is a legal store whose writes simply do not embed, and a repair
|
|
126
|
+
* has to say so rather than failing every record.
|
|
127
|
+
*/
|
|
128
|
+
get embedder(): MemoryEmbedder | undefined;
|
|
129
|
+
/** The wired fragment embedder, or `undefined`. See {@link VectorMaintenance.embedder}. */
|
|
130
|
+
get fragmentEmbedder(): FragmentEmbedder | undefined;
|
|
131
|
+
/**
|
|
132
|
+
* Re-embed one record into the record-vector index — the repair path.
|
|
133
|
+
*
|
|
134
|
+
* @remarks
|
|
135
|
+
* Distinct from {@link VectorMaintenance.embedOnWrite} in the one way that
|
|
136
|
+
* matters: **this is not best-effort.** Embed-on-write swallows a failure
|
|
137
|
+
* because a vault record is the source of truth and a write must not be
|
|
138
|
+
* rejected over a derived artifact; a repair was *asked for* by a caller who
|
|
139
|
+
* wants to know whether it worked, so a failure is returned.
|
|
140
|
+
*
|
|
141
|
+
* `undefined` means the embedder declined — intentionally not embedded, which
|
|
142
|
+
* is neither a repair nor a fault.
|
|
143
|
+
*/
|
|
144
|
+
reembedRecord(record: IMemoryRecord<unknown>, target: IEdgeTarget): Promise<Result<IReembedOutcome | undefined>>;
|
|
145
|
+
/**
|
|
146
|
+
* Re-embed one record's fragments — the repair path, returning the fragment
|
|
147
|
+
* count written. See {@link VectorMaintenance.reembedRecord} for why this is
|
|
148
|
+
* not best-effort. An empty fragment array is this lane's decline and reports
|
|
149
|
+
* `undefined`, though the whole-record-replace still runs so stale fragments
|
|
150
|
+
* are cleared.
|
|
151
|
+
*/
|
|
152
|
+
reembedFragments(record: IMemoryRecord<unknown>, target: IEdgeTarget): Promise<Result<IReembedOutcome | undefined>>;
|
|
79
153
|
/**
|
|
80
154
|
* Best-effort embed-on-write. When a vector index AND an embedder are wired,
|
|
81
155
|
* embeds the built record, `add`s the vector (replace semantics handle a same-id
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vectorMaintenance.d.ts","sourceRoot":"","sources":["../../../src/packlets/store/vectorMaintenance.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAiB,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"vectorMaintenance.d.ts","sourceRoot":"","sources":["../../../src/packlets/store/vectorMaintenance.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAiB,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAkB,MAAM,UAAU,CAAC;AACtG,OAAO,EACL,gBAAgB,EAEhB,oBAAoB,EACpB,YAAY,EACZ,cAAc,EACf,MAAM,WAAW,CAAC;AAEnB;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gFAAgF;IAChF,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;KAAE,CAAC;IAChF;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,kBAAkB,CAAC;CACrC;AA4CD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAM1G;AAED;;;;;GAKG;AACH,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,YAAY,CAAC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAC9C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAC7C,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC;;;;OAIG;IACH,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC;CAC9C;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA2B;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAmC;IAClE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA+B;IACjE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA0B;gBAEnC,MAAM,EAAE,wBAAwB;IASnD;;;;OAIG;IACU,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAK1D;;;;;OAKG;IACH,IAAW,WAAW,IAAI,YAAY,GAAG,SAAS,CAEjD;IAED,2FAA2F;IAC3F,IAAW,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAE3D;IAED;;;;;OAKG;IACH,IAAW,QAAQ,IAAI,cAAc,GAAG,SAAS,CAEhD;IAED,2FAA2F;IAC3F,IAAW,gBAAgB,IAAI,gBAAgB,GAAG,SAAS,CAE1D;IAED;;;;;;;;;;;;OAYG;IACU,aAAa,CACxB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IA2B/C;;;;;;OAMG;IACU,gBAAgB,CAC3B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IAyB/C;;;;;;;;;;;;;;OAcG;IACU,YAAY,CACvB,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAC5B,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAuDxC;;;;;;;;;;;OAWG;IACU,gBAAgB,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAUlF;;;;;;;;;;OAUG;IACU,qBAAqB,CAChC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAC5B,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAqBzC;;;;;;OAMG;YACW,0BAA0B;IAQxC;;;;;OAKG;IACU,oBAAoB,CAAC,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzG;;;;OAIG;YACW,YAAY;IAQ1B;;;;;OAKG;YACW,uBAAuB;CAOtC"}
|