@fgv/ts-agent-memory 5.1.0-48 → 5.1.0-50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/index/memoryIndex.js +23 -16
- package/dist/packlets/index/memoryIndex.js.map +1 -1
- package/dist/packlets/ingest/orchestrator.js +13 -1
- package/dist/packlets/ingest/orchestrator.js.map +1 -1
- package/dist/packlets/retrieve/linkTraversalRetriever.js +13 -27
- package/dist/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/dist/packlets/retrieve/recencyRetriever.js +8 -8
- package/dist/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/dist/packlets/retrieve/retriever.js +98 -11
- package/dist/packlets/retrieve/retriever.js.map +1 -1
- package/dist/packlets/retrieve/semanticRetriever.js +16 -16
- package/dist/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/dist/packlets/retrieve/structuredFilterRetriever.js +34 -12
- package/dist/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/dist/packlets/retrieve/tagRetriever.js +8 -8
- package/dist/packlets/retrieve/tagRetriever.js.map +1 -1
- package/dist/packlets/retrieve/temporalRetrievers.js +24 -21
- package/dist/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/dist/packlets/store/coverage.js +6 -0
- package/dist/packlets/store/coverage.js.map +1 -0
- package/dist/packlets/store/fileTreeMemoryStore.js +303 -62
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/store/index.js +4 -0
- package/dist/packlets/store/index.js.map +1 -1
- package/dist/packlets/store/listSelection.js +36 -0
- package/dist/packlets/store/listSelection.js.map +1 -0
- package/dist/packlets/store/memoryStore.js +6 -0
- package/dist/packlets/store/memoryStore.js.map +1 -0
- package/dist/packlets/store/reconcile.js +6 -0
- package/dist/packlets/store/reconcile.js.map +1 -0
- package/dist/packlets/store/storeCoverage.js +102 -0
- package/dist/packlets/store/storeCoverage.js.map +1 -0
- package/dist/packlets/store/storeReconcile.js +122 -0
- package/dist/packlets/store/storeReconcile.js.map +1 -0
- package/dist/packlets/store/vectorMaintenance.js +116 -8
- package/dist/packlets/store/vectorMaintenance.js.map +1 -1
- package/dist/packlets/store/vectorRecordSource.js +44 -0
- package/dist/packlets/store/vectorRecordSource.js.map +1 -0
- package/dist/packlets/tools/memoryTools.js +25 -2
- package/dist/packlets/tools/memoryTools.js.map +1 -1
- package/dist/packlets/types/envelope.js +25 -0
- package/dist/packlets/types/envelope.js.map +1 -1
- package/dist/packlets/types/index.js +1 -0
- package/dist/packlets/types/index.js.map +1 -1
- package/dist/packlets/types/recordResolver.js +6 -0
- package/dist/packlets/types/recordResolver.js.map +1 -0
- package/dist/packlets/types/temporal.js.map +1 -1
- package/dist/packlets/vector/inMemoryCosineIndex.js +39 -18
- package/dist/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +67 -12
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/dist/packlets/vector/rebuildHelpers.js +38 -0
- package/dist/packlets/vector/rebuildHelpers.js.map +1 -0
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/ts-agent-memory.d.ts +1140 -90
- package/lib/packlets/index/memoryIndex.d.ts +118 -27
- package/lib/packlets/index/memoryIndex.d.ts.map +1 -1
- package/lib/packlets/index/memoryIndex.js +23 -16
- package/lib/packlets/index/memoryIndex.js.map +1 -1
- package/lib/packlets/ingest/orchestrator.d.ts.map +1 -1
- package/lib/packlets/ingest/orchestrator.js +13 -1
- package/lib/packlets/ingest/orchestrator.js.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts +4 -11
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.js +12 -26
- package/lib/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.d.ts +4 -4
- package/lib/packlets/retrieve/recencyRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.js +7 -7
- package/lib/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/lib/packlets/retrieve/retriever.d.ts +113 -13
- package/lib/packlets/retrieve/retriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/retriever.js +101 -10
- package/lib/packlets/retrieve/retriever.js.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.d.ts +3 -5
- package/lib/packlets/retrieve/semanticRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/semanticRetriever.js +15 -15
- package/lib/packlets/retrieve/semanticRetriever.js.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts +29 -7
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.js +33 -11
- package/lib/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.d.ts +4 -4
- package/lib/packlets/retrieve/tagRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.js +7 -7
- package/lib/packlets/retrieve/tagRetriever.js.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.d.ts +7 -5
- package/lib/packlets/retrieve/temporalRetrievers.d.ts.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.js +23 -20
- package/lib/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/lib/packlets/store/coverage.d.ts +102 -0
- package/lib/packlets/store/coverage.d.ts.map +1 -0
- package/lib/packlets/store/coverage.js +7 -0
- package/lib/packlets/store/coverage.js.map +1 -0
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +88 -132
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +302 -61
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/store/index.d.ts +4 -0
- package/lib/packlets/store/index.d.ts.map +1 -1
- package/lib/packlets/store/index.js +4 -0
- package/lib/packlets/store/index.js.map +1 -1
- package/lib/packlets/store/listSelection.d.ts +101 -0
- package/lib/packlets/store/listSelection.d.ts.map +1 -0
- package/lib/packlets/store/listSelection.js +40 -0
- package/lib/packlets/store/listSelection.js.map +1 -0
- package/lib/packlets/store/memoryStore.d.ts +237 -0
- package/lib/packlets/store/memoryStore.d.ts.map +1 -0
- package/lib/packlets/store/memoryStore.js +7 -0
- package/lib/packlets/store/memoryStore.js.map +1 -0
- package/lib/packlets/store/reconcile.d.ts +82 -0
- package/lib/packlets/store/reconcile.d.ts.map +1 -0
- package/lib/packlets/store/reconcile.js +7 -0
- package/lib/packlets/store/reconcile.js.map +1 -0
- package/lib/packlets/store/storeCoverage.d.ts +45 -0
- package/lib/packlets/store/storeCoverage.d.ts.map +1 -0
- package/lib/packlets/store/storeCoverage.js +105 -0
- package/lib/packlets/store/storeCoverage.js.map +1 -0
- package/lib/packlets/store/storeReconcile.d.ts +41 -0
- package/lib/packlets/store/storeReconcile.d.ts.map +1 -0
- package/lib/packlets/store/storeReconcile.js +125 -0
- package/lib/packlets/store/storeReconcile.js.map +1 -0
- package/lib/packlets/store/vectorMaintenance.d.ts +74 -0
- package/lib/packlets/store/vectorMaintenance.d.ts.map +1 -1
- package/lib/packlets/store/vectorMaintenance.js +117 -8
- package/lib/packlets/store/vectorMaintenance.js.map +1 -1
- package/lib/packlets/store/vectorRecordSource.d.ts +36 -0
- package/lib/packlets/store/vectorRecordSource.d.ts.map +1 -0
- package/lib/packlets/store/vectorRecordSource.js +47 -0
- package/lib/packlets/store/vectorRecordSource.js.map +1 -0
- package/lib/packlets/tools/memoryTools.d.ts.map +1 -1
- package/lib/packlets/tools/memoryTools.js +25 -2
- package/lib/packlets/tools/memoryTools.js.map +1 -1
- package/lib/packlets/types/envelope.d.ts +33 -0
- package/lib/packlets/types/envelope.d.ts.map +1 -1
- package/lib/packlets/types/envelope.js +26 -0
- package/lib/packlets/types/envelope.js.map +1 -1
- package/lib/packlets/types/index.d.ts +1 -0
- package/lib/packlets/types/index.d.ts.map +1 -1
- package/lib/packlets/types/index.js +1 -0
- package/lib/packlets/types/index.js.map +1 -1
- package/lib/packlets/types/recordResolver.d.ts +39 -0
- package/lib/packlets/types/recordResolver.d.ts.map +1 -0
- package/lib/packlets/types/recordResolver.js +7 -0
- package/lib/packlets/types/recordResolver.js.map +1 -0
- package/lib/packlets/types/temporal.d.ts +26 -6
- package/lib/packlets/types/temporal.d.ts.map +1 -1
- package/lib/packlets/types/temporal.js.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts +9 -2
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.js +40 -19
- package/lib/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +6 -3
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +66 -11
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/lib/packlets/vector/rebuildHelpers.d.ts +30 -0
- package/lib/packlets/vector/rebuildHelpers.d.ts.map +1 -0
- package/lib/packlets/vector/rebuildHelpers.js +42 -0
- package/lib/packlets/vector/rebuildHelpers.js.map +1 -0
- package/lib/packlets/vector/vectorIndex.d.ts +270 -15
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -10,8 +10,12 @@ const ts_json_base_1 = require("@fgv/ts-json-base");
|
|
|
10
10
|
const types_1 = require("../types");
|
|
11
11
|
const converters_1 = require("../converters");
|
|
12
12
|
const vectorMaintenance_1 = require("./vectorMaintenance");
|
|
13
|
+
const storeCoverage_1 = require("./storeCoverage");
|
|
14
|
+
const storeReconcile_1 = require("./storeReconcile");
|
|
13
15
|
const index_1 = require("../index");
|
|
14
16
|
const scopeEncoding_1 = require("./scopeEncoding");
|
|
17
|
+
const listSelection_1 = require("./listSelection");
|
|
18
|
+
const vectorRecordSource_1 = require("./vectorRecordSource");
|
|
15
19
|
/** The on-disk extension for a memory record file. */
|
|
16
20
|
const MEMORY_FILE_EXTENSION = '.md';
|
|
17
21
|
/**
|
|
@@ -127,7 +131,7 @@ class FileTreeMemoryStore {
|
|
|
127
131
|
// version whose `invalid_at` is null/absent) from the entity subtree,
|
|
128
132
|
// read off the derived index. `asOf` resolution is via `list({ asOf })`
|
|
129
133
|
// and the temporal retrievers.
|
|
130
|
-
return
|
|
134
|
+
return this._readVersionedCurrent(addr.scope);
|
|
131
135
|
}
|
|
132
136
|
return this._readRecord(addr.scope, addr.idStem);
|
|
133
137
|
}));
|
|
@@ -143,50 +147,130 @@ class FileTreeMemoryStore {
|
|
|
143
147
|
return this._readRecord(scope, id);
|
|
144
148
|
}
|
|
145
149
|
/** {@inheritDoc IMemoryStore.list} */
|
|
146
|
-
async list(
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
150
|
+
async list(selection) {
|
|
151
|
+
// Deliberately NOT annotated `: boolean`. An explicit annotation discards the
|
|
152
|
+
// type predicate, and TypeScript's aliased-condition narrowing is what lets
|
|
153
|
+
// the ternary below see `selection` as a filter on the false branch. With the
|
|
154
|
+
// annotation this line compiles and the next one does not.
|
|
155
|
+
const scan = (0, listSelection_1.isWholeVaultScan)(selection);
|
|
156
|
+
const filter = scan ? {} : selection;
|
|
157
|
+
if (!scan && filter.scope === undefined && filter.kind === undefined && filter.tag === undefined) {
|
|
158
|
+
// `asOf` alone does not narrow — it collapses versions, it does not exclude
|
|
159
|
+
// entities — so a selection carrying only `asOf` lands here too.
|
|
160
|
+
return (0, ts_utils_1.fail)('memory list: a selection must narrow by scope, kind or tag; ' +
|
|
161
|
+
'pass scanEveryRecord() to read every record in the vault deliberately');
|
|
162
|
+
}
|
|
163
|
+
// Select over envelopes, project temporally, and materialize LAST — so a
|
|
164
|
+
// narrowed list reads only the files it is going to return.
|
|
165
|
+
const selected = this._index.entries().filter((entry) => {
|
|
166
|
+
if (filter.scope !== undefined && entry.scope !== filter.scope) {
|
|
151
167
|
return false;
|
|
152
168
|
}
|
|
153
|
-
if (
|
|
169
|
+
if (filter.kind !== undefined && entry.envelope.kind !== filter.kind) {
|
|
154
170
|
return false;
|
|
155
171
|
}
|
|
156
|
-
if (
|
|
172
|
+
if (filter.tag !== undefined && !entry.envelope.tags.includes(filter.tag)) {
|
|
157
173
|
return false;
|
|
158
174
|
}
|
|
159
175
|
return true;
|
|
160
|
-
})
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
176
|
+
});
|
|
177
|
+
const projected = selection.asOf === undefined
|
|
178
|
+
? // No temporal projection requested: the flat-path guarantee — every
|
|
179
|
+
// version is returned.
|
|
180
|
+
selected
|
|
181
|
+
: FileTreeMemoryStore._projectAsOf(selected, selection.asOf);
|
|
182
|
+
return this._materialize(projected);
|
|
183
|
+
}
|
|
184
|
+
/** {@inheritDoc IMemoryStore.coverage} */
|
|
185
|
+
coverage() {
|
|
186
|
+
return Promise.resolve((0, storeCoverage_1.computeCoverage)({
|
|
187
|
+
entries: this._index.entries(),
|
|
188
|
+
hasRankProjector: (kind) => this._rankProjectors.has(kind),
|
|
189
|
+
anyRankProjector: this._rankProjectors.size > 0,
|
|
190
|
+
embedsKind: (kind) => this.embedsKind(kind),
|
|
191
|
+
vectorIndex: this._vectors.vectorIndex,
|
|
192
|
+
fragmentIndex: this._vectors.fragmentIndex
|
|
193
|
+
}));
|
|
194
|
+
}
|
|
195
|
+
/** {@inheritDoc IMemoryStore.listEntries} */
|
|
196
|
+
async listEntries() {
|
|
197
|
+
return (0, ts_utils_1.succeed)(this._index.entries());
|
|
168
198
|
}
|
|
169
199
|
/** {@inheritDoc IMemoryStore.listScoped} */
|
|
170
200
|
async listScoped() {
|
|
171
|
-
// The derived index already carries each record's scope
|
|
172
|
-
// (
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
201
|
+
// The derived index already carries each record's scope, so the scoped
|
|
202
|
+
// projection keys straight off it — `(scope, id)` is exactly the address the
|
|
203
|
+
// vector index uses. Bodies ARE materialized here: this seam feeds an
|
|
204
|
+
// embedder, which is the one consumer that genuinely needs every body. No
|
|
205
|
+
// filter/temporal projection — it re-embeds the whole vault by contract.
|
|
206
|
+
//
|
|
207
|
+
// **Complete or failed — never quietly short.** This call is the sole feed
|
|
208
|
+
// for `asRecordSource()`, and therefore for `IVectorIndex.rebuild`'s coverage
|
|
209
|
+
// report. A record dropped here would land in none of `records` / `excluded`
|
|
210
|
+
// / `indexed` / `declined` / `skipped`, so a caller computing coverage would
|
|
211
|
+
// undercount *in the direction of looking healthier* — which is precisely the
|
|
212
|
+
// failure `vectorRecordSource`'s own tally exists to prevent, reintroduced one
|
|
213
|
+
// layer down. A loud failure is recoverable; a silent undercount is not.
|
|
214
|
+
//
|
|
215
|
+
// Two consequences worth stating rather than discovering. Before the index
|
|
216
|
+
// was projected this method read nothing and could not fail, because the
|
|
217
|
+
// index held the records themselves; it now costs a full-vault read and is
|
|
218
|
+
// fallible. And it is NOT write-locked, so a concurrent `put` whose cap-cull
|
|
219
|
+
// physically evicts a record between the snapshot and the read will fail it.
|
|
220
|
+
// That window is real and is tracked in `docs/FUTURE.md` — the answer is to
|
|
221
|
+
// make the loss *countable*, not to swallow it.
|
|
222
|
+
return (0, ts_utils_1.mapResults)(this._index.entries().map((entry) => {
|
|
223
|
+
const target = { scope: entry.scope, id: entry.envelope.id };
|
|
224
|
+
return this._resolveRequired(entry).onSuccess((record) => (0, ts_utils_1.succeed)({ target, record }));
|
|
225
|
+
}));
|
|
226
|
+
}
|
|
227
|
+
/** {@inheritDoc IMemoryRecordResolver.resolveRecord} */
|
|
228
|
+
resolveRecord(scope, id) {
|
|
229
|
+
return this._readRecord(scope, id);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Materialize a selected set of entries into records, dropping any that have
|
|
233
|
+
* vanished since selection.
|
|
234
|
+
*
|
|
235
|
+
* @remarks
|
|
236
|
+
* A miss is not a failure. Selection reads the in-memory index and
|
|
237
|
+
* materialization reads storage, so a record deleted in between is a legitimate
|
|
238
|
+
* race and yields a shorter list rather than an error. A read that FAILS is a
|
|
239
|
+
* real fault and propagates.
|
|
240
|
+
*/
|
|
241
|
+
_materialize(entries) {
|
|
242
|
+
return (0, ts_utils_1.mapResults)(entries.map((entry) => this._readRecord(entry.scope, entry.envelope.id))).onSuccess((records) => (0, ts_utils_1.succeed)(records.filter((r) => r !== undefined)));
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Materialize one entry, treating "gone" as a fault rather than a miss.
|
|
246
|
+
*
|
|
247
|
+
* @remarks
|
|
248
|
+
* For paths where a vanished record really does mean the index and the vault
|
|
249
|
+
* disagree, rather than that something legitimately removed it in between.
|
|
250
|
+
*
|
|
251
|
+
* Three of the four callers hold the write lock, so nothing can have removed
|
|
252
|
+
* the record since the entry was read. `get()`'s versioned path does not, and
|
|
253
|
+
* is safe only because temporal kinds never physically delete a version — they
|
|
254
|
+
* invalidate in place, and cap-cull does not apply to them. **If eviction is
|
|
255
|
+
* ever added to the temporal path, that caller must change**, or it
|
|
256
|
+
* reintroduces the race this method exists to detect.
|
|
257
|
+
*
|
|
258
|
+
* `listScoped` also does not hold the lock, and uses this deliberately anyway:
|
|
259
|
+
* it feeds a coverage report, so a silent drop there is worse than a loud
|
|
260
|
+
* failure. See its comment, and `docs/FUTURE.md` for the eviction window.
|
|
261
|
+
*
|
|
262
|
+
* The drop-tolerant counterpart is {@link FileTreeMemoryStore._materialize},
|
|
263
|
+
* for readers where a record that vanished between selection and
|
|
264
|
+
* materialization is a miss rather than a fault.
|
|
265
|
+
*/
|
|
266
|
+
_resolveRequired(entry) {
|
|
267
|
+
return this._readRecord(entry.scope, entry.envelope.id).onSuccess((record) => record === undefined
|
|
268
|
+
? (0, ts_utils_1.fail)(`memory: index entry '${entry.scope}/${entry.envelope.id}' has no record in the vault`)
|
|
269
|
+
: (0, ts_utils_1.succeed)(record));
|
|
179
270
|
}
|
|
180
271
|
/** {@inheritDoc IMemoryStore.asRecordSource} */
|
|
181
272
|
asRecordSource() {
|
|
182
|
-
return
|
|
183
|
-
// Filtered to the kinds that participate in the record vector index. This
|
|
184
|
-
// source exists to drive `IVectorIndex` rebuilds, so a kind excluded from the
|
|
185
|
-
// index has no business being re-embedded on open — which is where the cost
|
|
186
|
-
// is worst, since a rebuild embeds the whole vault serially. With no
|
|
187
|
-
// `embedKinds` declaration every kind passes and this is the identity filter.
|
|
188
|
-
list: async () => (await this.listScoped()).onSuccess((scoped) => (0, ts_utils_1.succeed)(scoped.filter((s) => this.embedsKind(s.record.envelope.kind))))
|
|
189
|
-
};
|
|
273
|
+
return (0, vectorRecordSource_1.vectorRecordSource)(this);
|
|
190
274
|
}
|
|
191
275
|
/**
|
|
192
276
|
* Collapse temporal records to the single version valid at `asOf` per entity;
|
|
@@ -195,21 +279,21 @@ class FileTreeMemoryStore {
|
|
|
195
279
|
* filtering is deferred — OQ-9). An entity with no version valid at `asOf`
|
|
196
280
|
* contributes nothing.
|
|
197
281
|
*/
|
|
198
|
-
static _projectAsOf(
|
|
282
|
+
static _projectAsOf(entries, asOf) {
|
|
199
283
|
const passthrough = [];
|
|
200
284
|
const groups = new Map();
|
|
201
|
-
for (const
|
|
202
|
-
if (!(0, types_1.isTemporalRecord)(
|
|
203
|
-
passthrough.push(
|
|
285
|
+
for (const entry of entries) {
|
|
286
|
+
if (!(0, types_1.isTemporalRecord)(entry)) {
|
|
287
|
+
passthrough.push(entry);
|
|
204
288
|
continue;
|
|
205
289
|
}
|
|
206
|
-
const key = `${
|
|
290
|
+
const key = `${entry.envelope.kind}\0${entry.envelope.entityId}`;
|
|
207
291
|
const existing = groups.get(key);
|
|
208
292
|
if (existing === undefined) {
|
|
209
|
-
groups.set(key, [
|
|
293
|
+
groups.set(key, [entry]);
|
|
210
294
|
}
|
|
211
295
|
else {
|
|
212
|
-
existing.push(
|
|
296
|
+
existing.push(entry);
|
|
213
297
|
}
|
|
214
298
|
}
|
|
215
299
|
const result = [...passthrough];
|
|
@@ -221,6 +305,10 @@ class FileTreeMemoryStore {
|
|
|
221
305
|
}
|
|
222
306
|
return result;
|
|
223
307
|
}
|
|
308
|
+
/** {@inheritDoc IMemoryStore.reconcile} */
|
|
309
|
+
async reconcile(kind, artifact) {
|
|
310
|
+
return this._enqueue(() => this._reconcileLocked(kind, artifact));
|
|
311
|
+
}
|
|
224
312
|
/** {@inheritDoc IMemoryStore.put} */
|
|
225
313
|
async put(record) {
|
|
226
314
|
const result = await this._enqueue(() => this._putLocked(record));
|
|
@@ -390,8 +478,11 @@ class FileTreeMemoryStore {
|
|
|
390
478
|
// mutableFields) actually applies. Content-dedup must never shadow LWW for
|
|
391
479
|
// the same entity.
|
|
392
480
|
const duplicate = this._findByContentHash(scope, hash, record.envelope.id);
|
|
393
|
-
if (duplicate
|
|
394
|
-
return (0, ts_utils_1.
|
|
481
|
+
if (duplicate.isFailure()) {
|
|
482
|
+
return (0, ts_utils_1.fail)(duplicate.message);
|
|
483
|
+
}
|
|
484
|
+
if (duplicate.value !== undefined) {
|
|
485
|
+
return (0, ts_utils_1.succeed)({ record: duplicate.value, evicted: [] });
|
|
395
486
|
}
|
|
396
487
|
}
|
|
397
488
|
return this._readRecord(scope, idStem).thenOnSuccess((existing) => {
|
|
@@ -412,9 +503,8 @@ class FileTreeMemoryStore {
|
|
|
412
503
|
// (a replace, not a grow). The same-id `existing` record is threaded
|
|
413
504
|
// separately into `_buildRecord` for the merge-patch. Knowledge LWW ignores
|
|
414
505
|
// this argument, so its behavior is unchanged by the wider cohort.
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
.admit(record, cohort)
|
|
506
|
+
return this._admissionCohort(scope, record.envelope.kind, idStem)
|
|
507
|
+
.onSuccess((cohort) => policy.admit(record, cohort))
|
|
418
508
|
.thenOnSuccess((decision) => this._admitWrite(record, body, scope, idStem, hash, policy, existing, decision));
|
|
419
509
|
});
|
|
420
510
|
}
|
|
@@ -564,7 +654,16 @@ class FileTreeMemoryStore {
|
|
|
564
654
|
* written, or fully invalidated / soft-deleted).
|
|
565
655
|
*/
|
|
566
656
|
_readVersionedCurrent(scope) {
|
|
567
|
-
|
|
657
|
+
// Select over ENVELOPES, then materialize the one winner — which is what
|
|
658
|
+
// `IMemoryStore.get`'s docstring promises and what `selectCurrentVersion`
|
|
659
|
+
// being generic over `IEnvelopeCarrier` exists for. Materializing every
|
|
660
|
+
// version first (as this did) cost N file reads and N body validations to
|
|
661
|
+
// return one record, and made that docstring false.
|
|
662
|
+
const current = (0, types_1.selectCurrentVersion)(this._index.entries().filter((entry) => entry.scope === scope));
|
|
663
|
+
if (current === undefined) {
|
|
664
|
+
return (0, ts_utils_1.succeed)(undefined);
|
|
665
|
+
}
|
|
666
|
+
return this._resolveRequired(current);
|
|
568
667
|
}
|
|
569
668
|
/**
|
|
570
669
|
* Every persisted version of the entity whose subtree is `scope`. All version
|
|
@@ -572,10 +671,13 @@ class FileTreeMemoryStore {
|
|
|
572
671
|
* entityId), so a scope filter over the index isolates one entity's versions.
|
|
573
672
|
*/
|
|
574
673
|
_versionsForEntity(scope) {
|
|
575
|
-
|
|
674
|
+
// Selected on the envelope (scope), materialized after — so a versioned write
|
|
675
|
+
// reads only that entity's versions, never the vault. Bounded by the entity's
|
|
676
|
+
// version count.
|
|
677
|
+
return (0, ts_utils_1.mapResults)(this._index
|
|
576
678
|
.entries()
|
|
577
679
|
.filter((entry) => entry.scope === scope)
|
|
578
|
-
.map((entry) => entry
|
|
680
|
+
.map((entry) => this._resolveRequired(entry)));
|
|
579
681
|
}
|
|
580
682
|
/**
|
|
581
683
|
* Versioned write (invalidate-don't-delete). Builds the new version's content
|
|
@@ -597,7 +699,11 @@ class FileTreeMemoryStore {
|
|
|
597
699
|
// still-current version at snapshot time — normally one, but two-or-more if a
|
|
598
700
|
// prior invalidation partially failed; invalidating all of them lets the write
|
|
599
701
|
// self-heal a stuck state (P2-7).
|
|
600
|
-
const
|
|
702
|
+
const snapshot = this._versionsForEntity(scope);
|
|
703
|
+
if (snapshot.isFailure()) {
|
|
704
|
+
return (0, ts_utils_1.fail)(snapshot.message);
|
|
705
|
+
}
|
|
706
|
+
const versions = snapshot.value;
|
|
601
707
|
const priorCurrents = versions.filter(types_1.isVersionCurrent);
|
|
602
708
|
const current = (0, types_1.selectCurrentVersion)(versions);
|
|
603
709
|
const policy = this._policyFor(kind);
|
|
@@ -723,7 +829,11 @@ class FileTreeMemoryStore {
|
|
|
723
829
|
* builds on it), so a hard delete would defeat the purpose.
|
|
724
830
|
*/
|
|
725
831
|
async _deleteVersioned(entityId, scope) {
|
|
726
|
-
const
|
|
832
|
+
const snapshot = this._versionsForEntity(scope);
|
|
833
|
+
if (snapshot.isFailure()) {
|
|
834
|
+
return (0, ts_utils_1.fail)(snapshot.message);
|
|
835
|
+
}
|
|
836
|
+
const versions = snapshot.value;
|
|
727
837
|
const currents = versions.filter(types_1.isVersionCurrent);
|
|
728
838
|
const current = (0, types_1.selectCurrentVersion)(versions);
|
|
729
839
|
if (current === undefined) {
|
|
@@ -770,10 +880,12 @@ class FileTreeMemoryStore {
|
|
|
770
880
|
* across first-writes and updates.
|
|
771
881
|
*/
|
|
772
882
|
_admissionCohort(scope, kind, idStem) {
|
|
773
|
-
|
|
883
|
+
// Envelope-only selection; the cohort is per-(scope, kind) and bounded by the
|
|
884
|
+
// cull cap, so materializing it is small by construction.
|
|
885
|
+
return (0, ts_utils_1.mapResults)(this._index
|
|
774
886
|
.entries()
|
|
775
|
-
.filter((entry) => entry.scope === scope && entry.
|
|
776
|
-
.map((entry) => entry
|
|
887
|
+
.filter((entry) => entry.scope === scope && entry.envelope.kind === kind && entry.envelope.id !== idStem)
|
|
888
|
+
.map((entry) => this._resolveRequired(entry)));
|
|
777
889
|
}
|
|
778
890
|
/**
|
|
779
891
|
* Find a record in `scope` whose `contentHash` equals `hash`, if any,
|
|
@@ -781,12 +893,13 @@ class FileTreeMemoryStore {
|
|
|
781
893
|
* dedup skip the same-id record so it does not shadow the LWW update path.
|
|
782
894
|
*/
|
|
783
895
|
_findByContentHash(scope, hash, excludeId) {
|
|
896
|
+
// The clearest win of the projection: the hash lives on the envelope, so this
|
|
897
|
+
// scans envelopes and reads exactly ONE file — the match — where it used to
|
|
898
|
+
// hold every body in the scope to look at one field.
|
|
784
899
|
const match = this._index
|
|
785
900
|
.entries()
|
|
786
|
-
.find((entry) => entry.scope === scope &&
|
|
787
|
-
|
|
788
|
-
entry.record.envelope.id !== excludeId);
|
|
789
|
-
return match === null || match === void 0 ? void 0 : match.record;
|
|
901
|
+
.find((entry) => entry.scope === scope && entry.envelope.contentHash === hash && entry.envelope.id !== excludeId);
|
|
902
|
+
return match === undefined ? (0, ts_utils_1.succeed)(undefined) : this._resolveRequired(match);
|
|
790
903
|
}
|
|
791
904
|
/**
|
|
792
905
|
* True when `incoming`'s caller-authored mutable metadata (`tags` / `provenance`)
|
|
@@ -811,6 +924,128 @@ class FileTreeMemoryStore {
|
|
|
811
924
|
_contentHash(kind, body, links) {
|
|
812
925
|
return this._hasher.computeHash({ kind, body, links });
|
|
813
926
|
}
|
|
927
|
+
/**
|
|
928
|
+
* The rank branch of {@link FileTreeMemoryStore.reconcile}, under the write lock.
|
|
929
|
+
*
|
|
930
|
+
* @remarks
|
|
931
|
+
* Re-reads each record's file rather than trusting the in-memory index, for
|
|
932
|
+
* two reasons: the index holds converted bodies on some paths and raw ones on
|
|
933
|
+
* others, and re-serializing a *converted* body could change the bytes on disk
|
|
934
|
+
* — a reconcile of an ordering field has no business rewriting content.
|
|
935
|
+
* `splitFrontmatter` hands back the body text unconverted, so the round trip
|
|
936
|
+
* carries the authored characters through untouched and only the envelope
|
|
937
|
+
* moves. The parsed record is additionally put through `_verifyLoaded`, the
|
|
938
|
+
* same id-vs-filename and scope-derived-entityId check the load paths apply —
|
|
939
|
+
* without it, reconcile would be the one path that accepts and rewrites a file
|
|
940
|
+
* the store would refuse to load.
|
|
941
|
+
*
|
|
942
|
+
* "Untouched" is not quite "byte-identical", and the exception is line
|
|
943
|
+
* endings: `splitFrontmatter` strips a trailing `\r` per line and
|
|
944
|
+
* `joinFrontmatter` writes `\n` delimiters, so a CRLF-authored file comes back
|
|
945
|
+
* LF-normalized. That is **the store's behavior on every write path, not
|
|
946
|
+
* something reconcile introduces** — an ordinary `put` normalizes the same way
|
|
947
|
+
* — so reconcile does not rewrite content that a subsequent write would have
|
|
948
|
+
* left alone. The property being claimed here is the narrower and load-bearing
|
|
949
|
+
* one: no body is round-tripped through its registered Converter.
|
|
950
|
+
*
|
|
951
|
+
* The projector is fed an `IMemoryRecord<string>` carrying that raw body,
|
|
952
|
+
* which is exactly the shape {@link FileTreeMemoryStore._stampRank} hands it
|
|
953
|
+
* on the write path — so a projector cannot see one thing on a write and
|
|
954
|
+
* another on a reconcile. `_stampRank` itself is reused verbatim, which also
|
|
955
|
+
* inherits its throw semantics (logged at `warn`, `rank` cleared).
|
|
956
|
+
*/
|
|
957
|
+
async _reconcileLocked(kind, artifact) {
|
|
958
|
+
// Envelope-only selection: the walk needs `(scope, id)` and the kind, and
|
|
959
|
+
// each branch materializes only the records it decides to repair.
|
|
960
|
+
const targets = this._index
|
|
961
|
+
.entries()
|
|
962
|
+
.filter((entry) => entry.envelope.kind === kind);
|
|
963
|
+
if (artifact === 'rank') {
|
|
964
|
+
return this._reconcileRankLocked(kind, targets);
|
|
965
|
+
}
|
|
966
|
+
return (0, storeReconcile_1.reconcileVectors)({
|
|
967
|
+
kind,
|
|
968
|
+
artifact,
|
|
969
|
+
targets,
|
|
970
|
+
maintenance: this._vectors,
|
|
971
|
+
embedsKind: (k) => this.embedsKind(k),
|
|
972
|
+
resolve: (scope, id) => this._readRecord(scope, id),
|
|
973
|
+
stampRef: (scope, id, ref) => this._rewriteEnvelope(scope, id, (r) => r.envelope.embeddingRef === ref
|
|
974
|
+
? undefined
|
|
975
|
+
: { envelope: Object.assign(Object.assign({}, r.envelope), { embeddingRef: ref }), body: r.body })
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
/** The rank branch of {@link FileTreeMemoryStore._reconcileLocked}. */
|
|
979
|
+
async _reconcileRankLocked(kind, targets) {
|
|
980
|
+
if (!this._rankProjectors.has(kind)) {
|
|
981
|
+
return (0, ts_utils_1.fail)(`memory reconcile '${kind}' rank: no rank projector is registered for this kind`);
|
|
982
|
+
}
|
|
983
|
+
// No materialization at all: `_rewriteEnvelope` re-reads each record itself.
|
|
984
|
+
let repaired = 0;
|
|
985
|
+
for (const target of targets) {
|
|
986
|
+
const applied = this._rewriteEnvelope(target.scope, target.envelope.id, (r) => {
|
|
987
|
+
const stamped = this._stampRank(r);
|
|
988
|
+
return stamped.envelope.rank === r.envelope.rank ? undefined : stamped;
|
|
989
|
+
});
|
|
990
|
+
if (applied.isFailure()) {
|
|
991
|
+
return (0, ts_utils_1.fail)(`memory reconcile '${kind}' rank: ${applied.message}`);
|
|
992
|
+
}
|
|
993
|
+
if (applied.value) {
|
|
994
|
+
repaired++;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
return (0, ts_utils_1.succeed)({ artifact: 'rank', kind, examined: targets.length, repaired, failed: [] });
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* Re-apply the rank projector to one record on disk. Returns whether `rank`
|
|
1001
|
+
* actually changed — an unchanged rank writes nothing, so a reconcile over an
|
|
1002
|
+
* already-consistent store touches no files.
|
|
1003
|
+
*/
|
|
1004
|
+
_rewriteEnvelope(scope, id, mutate) {
|
|
1005
|
+
return this._resolveScopeDir(scope).onSuccess((scopeDir) => {
|
|
1006
|
+
/* c8 ignore next 3 - defensive: the scope dir exists for any indexed record */
|
|
1007
|
+
if (scopeDir === undefined) {
|
|
1008
|
+
return (0, ts_utils_1.fail)(`'${id}': scope '${scope}' not found`);
|
|
1009
|
+
}
|
|
1010
|
+
return scopeDir.getChildren().onSuccess((children) => {
|
|
1011
|
+
const targetName = `${id}${MEMORY_FILE_EXTENSION}`;
|
|
1012
|
+
const file = children.find((c) => c.type === 'file' && c.name === targetName);
|
|
1013
|
+
/* c8 ignore next 3 - defensive: the file exists for any indexed record */
|
|
1014
|
+
if (file === undefined) {
|
|
1015
|
+
return (0, ts_utils_1.fail)(`'${id}': file not found`);
|
|
1016
|
+
}
|
|
1017
|
+
return file.getRawContents().onSuccess((raw) =>
|
|
1018
|
+
// `parseMemoryFile` validates the body through the registered Converter,
|
|
1019
|
+
// so a corrupt record is refused rather than silently rewritten, and
|
|
1020
|
+
// `_verifyLoaded` re-applies the same id-vs-filename and
|
|
1021
|
+
// scope-derived-entityId checks the two load paths apply. Reconcile is a
|
|
1022
|
+
// read-then-write, and a file can change on disk after the index was
|
|
1023
|
+
// built, so skipping them would make this the one path that accepts —
|
|
1024
|
+
// and rewrites — a record the store would otherwise refuse to load.
|
|
1025
|
+
// `splitFrontmatter` then supplies the body text for the write.
|
|
1026
|
+
(0, converters_1.parseMemoryFile)(raw, this._registry)
|
|
1027
|
+
.withErrorFormat((msg) => `'${id}': ${msg}`)
|
|
1028
|
+
.onSuccess((parsedRecord) => this._verifyLoaded(scope, file, parsedRecord))
|
|
1029
|
+
.onSuccess((parsed) => (0, converters_1.splitFrontmatter)(raw)
|
|
1030
|
+
.withErrorFormat((msg) => `'${id}': ${msg}`)
|
|
1031
|
+
.onSuccess((parts) => {
|
|
1032
|
+
// The mutator says "nothing to change" with `undefined` rather
|
|
1033
|
+
// than by returning an equal record: persisting unconditionally
|
|
1034
|
+
// would bump `updated` on every record of the kind, trading a
|
|
1035
|
+
// wrong value for a wrong timestamp, and a deep comparison here
|
|
1036
|
+
// would have to know which fields each caller touches.
|
|
1037
|
+
const stamped = mutate({
|
|
1038
|
+
envelope: parsed.envelope,
|
|
1039
|
+
body: parts.body
|
|
1040
|
+
});
|
|
1041
|
+
if (stamped === undefined) {
|
|
1042
|
+
return (0, ts_utils_1.succeed)(false);
|
|
1043
|
+
}
|
|
1044
|
+
return this._persist(stamped, scope, id).onSuccess(() => (0, ts_utils_1.succeed)(true));
|
|
1045
|
+
})));
|
|
1046
|
+
});
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
814
1049
|
/**
|
|
815
1050
|
* Stamp the store-computed {@link IMemoryEnvelope.rank} onto a fully-built,
|
|
816
1051
|
* fully-stamped record by running the kind's registered {@link RankProjector}.
|
|
@@ -999,8 +1234,8 @@ class FileTreeMemoryStore {
|
|
|
999
1234
|
_initialIndex(onRecordError) {
|
|
1000
1235
|
return this._collectEntries(this._root, [], onRecordError).onSuccess((entries) => this._index.rebuild(entries).onSuccess(() => {
|
|
1001
1236
|
for (const entry of entries) {
|
|
1002
|
-
if (entry.
|
|
1003
|
-
this._seq = entry.
|
|
1237
|
+
if (entry.envelope.seq > this._seq) {
|
|
1238
|
+
this._seq = entry.envelope.seq;
|
|
1004
1239
|
}
|
|
1005
1240
|
}
|
|
1006
1241
|
return (0, ts_utils_1.succeed)(true);
|
|
@@ -1041,11 +1276,17 @@ class FileTreeMemoryStore {
|
|
|
1041
1276
|
* failure passes through untouched so the historical error is byte-identical.
|
|
1042
1277
|
*/
|
|
1043
1278
|
_loadRecordFile(scope, child, onRecordError) {
|
|
1044
|
-
return child
|
|
1279
|
+
return (child
|
|
1045
1280
|
.getRawContents()
|
|
1046
1281
|
.onSuccess((raw) => (0, converters_1.parseMemoryFile)(raw, this._registry))
|
|
1047
1282
|
.onSuccess((parsedRecord) => this._verifyLoaded(scope, child, parsedRecord))
|
|
1048
|
-
|
|
1283
|
+
// Parse → validate → PROJECT → discard, per file. The body is read and
|
|
1284
|
+
// fully validated (which is what gives `onRecordError` its meaning), then
|
|
1285
|
+
// dropped here rather than carried into the index. Peak body residency
|
|
1286
|
+
// across the whole open is therefore ONE record, not N — which is the
|
|
1287
|
+
// resident-memory moment this whole surface exists to fix, and it is why
|
|
1288
|
+
// the projection had to reach `rebuild` and not just the read methods.
|
|
1289
|
+
.onSuccess((verified) => (0, ts_utils_1.succeed)([{ scope, envelope: verified.envelope }]))
|
|
1049
1290
|
.onFailure((message) => {
|
|
1050
1291
|
if (onRecordError === 'skip') {
|
|
1051
1292
|
const path = `${scope}/${child.name}`;
|
|
@@ -1054,7 +1295,7 @@ class FileTreeMemoryStore {
|
|
|
1054
1295
|
this._warnSwallowed(error);
|
|
1055
1296
|
}
|
|
1056
1297
|
return (0, ts_utils_1.fail)(message);
|
|
1057
|
-
});
|
|
1298
|
+
}));
|
|
1058
1299
|
}
|
|
1059
1300
|
}
|
|
1060
1301
|
exports.FileTreeMemoryStore = FileTreeMemoryStore;
|