@remnic/core 9.6.18 → 9.6.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.js +9 -9
- package/dist/access-boundary.js +10 -10
- package/dist/access-cli.js +37 -36
- package/dist/access-cli.js.map +1 -1
- package/dist/access-http.js +16 -16
- package/dist/access-identity-continuity-surface.js +9 -9
- package/dist/access-lcm-surface.js +9 -9
- package/dist/access-mcp.js +15 -15
- package/dist/access-observe-write-surface.js +9 -9
- package/dist/access-operations-batch.js +12 -12
- package/dist/access-operations.js +14 -14
- package/dist/access-recall-surface.js +9 -9
- package/dist/access-schema.js +5 -5
- package/dist/access-service.js +9 -9
- package/dist/{capsule-crypto-RAARDY3M.js → capsule-crypto-CZJSLEFG.js} +2 -2
- package/dist/{chunk-EEIROWEG.js → chunk-2BFLDH5F.js} +2 -2
- package/dist/{chunk-UPSAT4PY.js → chunk-2MR3MFQB.js} +2 -2
- package/dist/{chunk-LNQUAJNC.js → chunk-437S3G37.js} +1 -1
- package/dist/{chunk-JZKR6ZAV.js → chunk-4Q2XWAHZ.js} +2 -2
- package/dist/{chunk-HR4NACR7.js → chunk-A2OFRQ5B.js} +5 -5
- package/dist/{chunk-D3YEDDCC.js → chunk-ARDRBJHD.js} +4 -4
- package/dist/{chunk-U4CWMKN6.js → chunk-B4LPRAWL.js} +2 -2
- package/dist/{chunk-L6NW2XSU.js → chunk-D3D677BV.js} +4 -4
- package/dist/{chunk-7WSLR2CU.js → chunk-DE6ENYLG.js} +2 -2
- package/dist/{chunk-DEG5ULFJ.js → chunk-DQ2CJYUM.js} +2 -2
- package/dist/{chunk-KTULX4YO.js → chunk-FLVL4L3L.js} +4 -4
- package/dist/{chunk-KJ3QGI63.js → chunk-GXLFDLX2.js} +6 -6
- package/dist/{chunk-MGN7VHWQ.js → chunk-HHAA35A3.js} +120 -7
- package/dist/chunk-HHAA35A3.js.map +1 -0
- package/dist/chunk-I6WYQSX4.js +99 -0
- package/dist/chunk-I6WYQSX4.js.map +1 -0
- package/dist/{chunk-JCEFF346.js → chunk-KEY3FEMP.js} +18 -18
- package/dist/{chunk-PYHOAXTI.js → chunk-LRGSLV7M.js} +4 -4
- package/dist/{chunk-CYWCKFCL.js → chunk-M5QKGHCR.js} +5 -5
- package/dist/{chunk-RSS3CZV7.js → chunk-MRQN5Q3I.js} +2 -2
- package/dist/{chunk-HLN7RROI.js → chunk-SFHI7JBE.js} +4 -4
- package/dist/{chunk-5TYS6NTK.js → chunk-XMNVHYI6.js} +84 -35
- package/dist/chunk-XMNVHYI6.js.map +1 -0
- package/dist/{chunk-ZZYLM6KQ.js → chunk-YDF4HL3W.js} +4 -4
- package/dist/cli.js +25 -25
- package/dist/contradiction/index.js +4 -4
- package/dist/{first-start-migration-VBJQL6OI.js → first-start-migration-4I3VDSYS.js} +4 -4
- package/dist/index.js +46 -45
- package/dist/lcm/index.js +3 -3
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/search.js +3 -3
- package/dist/operator-toolkit.js +5 -5
- package/dist/orchestrator.js +39 -38
- package/dist/retrieval-agents.js +2 -2
- package/dist/search/factory.js +2 -2
- package/dist/search/index.js +4 -4
- package/dist/temporal-index.d.ts +42 -7
- package/dist/temporal-index.js +3 -1
- package/dist/temporal-timeline-recall.d.ts +24 -0
- package/dist/temporal-timeline-recall.js +8 -0
- package/dist/temporal-timeline-recall.js.map +1 -0
- package/dist/transfer/backup.js +3 -3
- package/dist/transfer/capsule-export.js +4 -4
- package/dist/transfer/capsule-import.js +3 -3
- package/dist/transfer/import-sqlite.js +2 -2
- package/package.json +2 -2
- package/src/lifecycle/tombstones.test.ts +78 -5
- package/src/orchestration/persistence-index.ts +15 -2
- package/src/orchestration/recall-internal.ts +77 -35
- package/src/temporal-index.test.ts +157 -0
- package/src/temporal-index.ts +211 -7
- package/src/temporal-timeline-recall.test.ts +70 -0
- package/src/temporal-timeline-recall.ts +111 -0
- package/dist/chunk-5TYS6NTK.js.map +0 -1
- package/dist/chunk-MGN7VHWQ.js.map +0 -1
- /package/dist/{capsule-crypto-RAARDY3M.js.map → capsule-crypto-CZJSLEFG.js.map} +0 -0
- /package/dist/{chunk-EEIROWEG.js.map → chunk-2BFLDH5F.js.map} +0 -0
- /package/dist/{chunk-UPSAT4PY.js.map → chunk-2MR3MFQB.js.map} +0 -0
- /package/dist/{chunk-LNQUAJNC.js.map → chunk-437S3G37.js.map} +0 -0
- /package/dist/{chunk-JZKR6ZAV.js.map → chunk-4Q2XWAHZ.js.map} +0 -0
- /package/dist/{chunk-HR4NACR7.js.map → chunk-A2OFRQ5B.js.map} +0 -0
- /package/dist/{chunk-D3YEDDCC.js.map → chunk-ARDRBJHD.js.map} +0 -0
- /package/dist/{chunk-U4CWMKN6.js.map → chunk-B4LPRAWL.js.map} +0 -0
- /package/dist/{chunk-L6NW2XSU.js.map → chunk-D3D677BV.js.map} +0 -0
- /package/dist/{chunk-7WSLR2CU.js.map → chunk-DE6ENYLG.js.map} +0 -0
- /package/dist/{chunk-DEG5ULFJ.js.map → chunk-DQ2CJYUM.js.map} +0 -0
- /package/dist/{chunk-KTULX4YO.js.map → chunk-FLVL4L3L.js.map} +0 -0
- /package/dist/{chunk-KJ3QGI63.js.map → chunk-GXLFDLX2.js.map} +0 -0
- /package/dist/{chunk-JCEFF346.js.map → chunk-KEY3FEMP.js.map} +0 -0
- /package/dist/{chunk-PYHOAXTI.js.map → chunk-LRGSLV7M.js.map} +0 -0
- /package/dist/{chunk-CYWCKFCL.js.map → chunk-M5QKGHCR.js.map} +0 -0
- /package/dist/{chunk-RSS3CZV7.js.map → chunk-MRQN5Q3I.js.map} +0 -0
- /package/dist/{chunk-HLN7RROI.js.map → chunk-SFHI7JBE.js.map} +0 -0
- /package/dist/{chunk-ZZYLM6KQ.js.map → chunk-YDF4HL3W.js.map} +0 -0
- /package/dist/{first-start-migration-VBJQL6OI.js.map → first-start-migration-4I3VDSYS.js.map} +0 -0
|
@@ -36,6 +36,7 @@ import { StorageManager } from "../index.js";
|
|
|
36
36
|
import { inferIntentFromText, planRecallMode } from "../intent.js";
|
|
37
37
|
import { LcmEngine } from "../lcm/index.js";
|
|
38
38
|
import { log } from "../logger.js";
|
|
39
|
+
import { isActiveMemoryStatus } from "../memory-lifecycle-ledger-utils.js";
|
|
39
40
|
import { buildRetrievedMemoryProvenance } from "../memory-provenance.js";
|
|
40
41
|
import { NamespaceCatalog } from "../namespaces/catalog.js";
|
|
41
42
|
import { canReadNamespace, resolvePrincipal } from "../namespaces/principal.js";
|
|
@@ -64,6 +65,8 @@ import { isDisagreementPrompt } from "../signal.js";
|
|
|
64
65
|
import { HourlySummarizer } from "../summarizer.js";
|
|
65
66
|
import { buildTargetedFactRecallSection, shouldRecallTargetedFactEvidence } from "../targeted-fact-recall.js";
|
|
66
67
|
import { shouldFilterSupersededFromRecall } from "../temporal-supersession.js";
|
|
68
|
+
import { queryTemporalTimelineAsync } from "../temporal-index.js";
|
|
69
|
+
import { buildTemporalTimelineRecallSection, type TemporalTimelineRecallItem } from "../temporal-timeline-recall.js";
|
|
67
70
|
import { isValidAsOf } from "../temporal-validity.js";
|
|
68
71
|
import { TmtBuilder } from "../tmt.js";
|
|
69
72
|
import { TranscriptManager } from "../transcript.js";
|
|
@@ -3659,54 +3662,93 @@ export class RecallInternalCoordinator {
|
|
|
3659
3662
|
}
|
|
3660
3663
|
}
|
|
3661
3664
|
|
|
3662
|
-
// 0e. Chronological event-order evidence.
|
|
3663
|
-
//
|
|
3665
|
+
// 0e. Chronological event-order evidence. Prefer the ingest-time temporal
|
|
3666
|
+
// index, which can safely merge event times across source sessions. Fall
|
|
3667
|
+
// back to the legacy per-session LCM turn order when the index is absent.
|
|
3664
3668
|
const eventOrderMaxChars =
|
|
3665
3669
|
this.deps.getRecallSectionMaxChars("event-order") ??
|
|
3666
3670
|
this.deps.config.eventOrderRecallMaxChars;
|
|
3671
|
+
const eventOrderMaxItems =
|
|
3672
|
+
this.deps.getRecallSectionNumber("event-order", "maxResults") ??
|
|
3673
|
+
this.deps.config.eventOrderRecallMaxResults;
|
|
3667
3674
|
if (
|
|
3668
3675
|
this.deps.isSpecializedRecallSectionEnabled(
|
|
3669
3676
|
"event-order",
|
|
3670
3677
|
resolveRecallEnhancementCapabilities(this.deps.config).eventOrderRecall,
|
|
3671
3678
|
) &&
|
|
3672
3679
|
eventOrderMaxChars !== 0 &&
|
|
3673
|
-
|
|
3680
|
+
eventOrderMaxItems > 0 &&
|
|
3674
3681
|
(recallMode as RecallPlanMode) !== "no_recall" &&
|
|
3675
3682
|
shouldRecallEventOrderEvidence(retrievalQuery)
|
|
3676
3683
|
) {
|
|
3677
3684
|
try {
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
this.deps.
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3685
|
+
const maxItems = eventOrderMaxItems;
|
|
3686
|
+
let eventOrderSection = "";
|
|
3687
|
+
const timelineCandidateLimit = Math.min(256, Math.max(48, maxItems * 12));
|
|
3688
|
+
const timeline = await queryTemporalTimelineAsync(this.deps.config.memoryDir, {
|
|
3689
|
+
query: retrievalQuery,
|
|
3690
|
+
limit: timelineCandidateLimit,
|
|
3691
|
+
});
|
|
3692
|
+
if (timeline) {
|
|
3693
|
+
const timelineItems: TemporalTimelineRecallItem[] = [];
|
|
3694
|
+
// Sequential reads keep file-descriptor pressure bounded. The index
|
|
3695
|
+
// query above has already capped total reads deterministically.
|
|
3696
|
+
for (const event of timeline) {
|
|
3697
|
+
const namespace = this.deps.namespaceFromPath(event.path);
|
|
3698
|
+
if (
|
|
3699
|
+
resolveNamespaceCapabilities(this.deps.config).namespaces &&
|
|
3700
|
+
!recallNamespaces.includes(namespace)
|
|
3701
|
+
) {
|
|
3702
|
+
continue;
|
|
3703
|
+
}
|
|
3704
|
+
try {
|
|
3705
|
+
const storage = resolveNamespaceCapabilities(this.deps.config).namespaces
|
|
3706
|
+
? await this.deps.storageRouter.storageFor(namespace)
|
|
3707
|
+
: this.deps.storage;
|
|
3708
|
+
const memory = await storage.readMemoryByPath(event.path);
|
|
3709
|
+
if (!memory || !isActiveMemoryStatus(memory.frontmatter.status)) continue;
|
|
3710
|
+
if (!isValidAsOf(memory.frontmatter, asOfMs ?? Date.now())) continue;
|
|
3711
|
+
timelineItems.push({
|
|
3712
|
+
memory,
|
|
3713
|
+
eventAt: event.eventAt,
|
|
3714
|
+
...(event.observedAt ? { observedAt: event.observedAt } : {}),
|
|
3715
|
+
...(event.sessionKey ? { sessionKey: event.sessionKey } : {}),
|
|
3716
|
+
...(event.validUntil ? { validUntil: event.validUntil } : {}),
|
|
3717
|
+
});
|
|
3718
|
+
} catch {
|
|
3719
|
+
continue;
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
eventOrderSection = buildTemporalTimelineRecallSection({
|
|
3723
|
+
items: timelineItems,
|
|
3724
|
+
query: retrievalQuery,
|
|
3725
|
+
maxChars: eventOrderMaxChars,
|
|
3726
|
+
maxItems,
|
|
3727
|
+
});
|
|
3728
|
+
}
|
|
3729
|
+
|
|
3730
|
+
// Legacy fallback: LCM turn indexes are local to one session, so keep
|
|
3731
|
+
// first-non-empty semantics and never interleave them across sessions.
|
|
3732
|
+
if (!eventOrderSection && this.deps.lcmEngine?.enabled) {
|
|
3733
|
+
eventOrderSection = await firstNonEmptyLcmRead(
|
|
3734
|
+
(lcmSessionId) =>
|
|
3735
|
+
buildEventOrderRecallSection({
|
|
3736
|
+
engine: this.deps.lcmEngine,
|
|
3737
|
+
sessionId: lcmSessionId,
|
|
3738
|
+
query: retrievalQuery,
|
|
3739
|
+
maxChars: eventOrderMaxChars,
|
|
3740
|
+
maxItems,
|
|
3741
|
+
maxScanWindowTurns:
|
|
3742
|
+
this.deps.getRecallSectionNumber("event-order", "maxTurns") ??
|
|
3743
|
+
this.deps.config.eventOrderRecallScanWindowTurns,
|
|
3744
|
+
maxScanWindowTokens:
|
|
3745
|
+
this.deps.getRecallSectionNumber("event-order", "maxTokens") ??
|
|
3746
|
+
this.deps.config.eventOrderRecallScanWindowTokens,
|
|
3747
|
+
}),
|
|
3748
|
+
(s) => !s,
|
|
3749
|
+
"",
|
|
3750
|
+
);
|
|
3751
|
+
}
|
|
3710
3752
|
if (eventOrderSection) {
|
|
3711
3753
|
this.deps.appendRecallSection(
|
|
3712
3754
|
sectionBuckets,
|
|
@@ -7,9 +7,12 @@ import test from "node:test";
|
|
|
7
7
|
import { setTimeout as delay } from "node:timers/promises";
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
+
deindexMemory,
|
|
10
11
|
indexMemory,
|
|
12
|
+
indexesExist,
|
|
11
13
|
queryByDateRangeAsync,
|
|
12
14
|
queryByTagsAsync,
|
|
15
|
+
queryTemporalTimelineAsync,
|
|
13
16
|
resolvePromptTagPrefilterAsync,
|
|
14
17
|
} from "./temporal-index.js";
|
|
15
18
|
|
|
@@ -236,3 +239,157 @@ test("indexMemory replaces stale date and tag memberships for an existing path",
|
|
|
236
239
|
assert.deepEqual(alphaMatches, new Set());
|
|
237
240
|
assert.deepEqual(betaMatches, new Set([memoryPath]));
|
|
238
241
|
});
|
|
242
|
+
|
|
243
|
+
test("temporal timeline orders event time across sessions, not ingest order", async () => {
|
|
244
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-order-"));
|
|
245
|
+
indexMemory(memoryDir, "/tmp/rome.md", "2026-07-02T00:00:00.000Z", [], {
|
|
246
|
+
validAt: "2026-05-10T00:00:00.000Z",
|
|
247
|
+
observedAt: "2026-07-02T00:00:00.000Z",
|
|
248
|
+
sessionKey: "session-b",
|
|
249
|
+
});
|
|
250
|
+
indexMemory(memoryDir, "/tmp/paris.md", "2026-07-03T00:00:00.000Z", [], {
|
|
251
|
+
validAt: "2026-03-04T00:00:00.000Z",
|
|
252
|
+
observedAt: "2026-07-03T00:00:00.000Z",
|
|
253
|
+
sessionKey: "session-a",
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
const timeline = await queryTemporalTimelineAsync(memoryDir);
|
|
257
|
+
assert.deepEqual(
|
|
258
|
+
timeline?.map(({ path, eventAt, sessionKey }) => ({ path, eventAt, sessionKey })),
|
|
259
|
+
[
|
|
260
|
+
{ path: "/tmp/paris.md", eventAt: "2026-03-04T00:00:00.000Z", sessionKey: "session-a" },
|
|
261
|
+
{ path: "/tmp/rome.md", eventAt: "2026-05-10T00:00:00.000Z", sessionKey: "session-b" },
|
|
262
|
+
],
|
|
263
|
+
);
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
test("temporal timeline uses stable observation/path ordering for event-time ties", async () => {
|
|
267
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-tie-"));
|
|
268
|
+
const eventAt = "2026-03-04T00:00:00.000Z";
|
|
269
|
+
indexMemory(memoryDir, "/tmp/z.md", eventAt, [], { validAt: eventAt, observedAt: "2026-07-03T00:00:00.000Z" });
|
|
270
|
+
indexMemory(memoryDir, "/tmp/a.md", eventAt, [], { validAt: eventAt, observedAt: "2026-07-02T00:00:00.000Z" });
|
|
271
|
+
indexMemory(memoryDir, "/tmp/b.md", eventAt, [], { validAt: eventAt, observedAt: "2026-07-02T00:00:00.000Z" });
|
|
272
|
+
|
|
273
|
+
const first = await queryTemporalTimelineAsync(memoryDir);
|
|
274
|
+
const second = await queryTemporalTimelineAsync(memoryDir);
|
|
275
|
+
assert.deepEqual(first?.map((entry) => entry.path), ["/tmp/a.md", "/tmp/b.md", "/tmp/z.md"]);
|
|
276
|
+
assert.deepEqual(second, first);
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
test("old or malformed temporal index shape is unavailable and requests rebuild", async () => {
|
|
280
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-malformed-"));
|
|
281
|
+
await mkdir(join(memoryDir, "state"), { recursive: true });
|
|
282
|
+
await writeFile(
|
|
283
|
+
join(memoryDir, "state", "index_time.json"),
|
|
284
|
+
JSON.stringify({ version: 2, dates: {}, events: [] }),
|
|
285
|
+
"utf8",
|
|
286
|
+
);
|
|
287
|
+
await writeFile(
|
|
288
|
+
join(memoryDir, "state", "index_tags.json"),
|
|
289
|
+
JSON.stringify({ version: 2, tags: {}, aliases: {} }),
|
|
290
|
+
"utf8",
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
assert.equal(indexesExist(memoryDir), false);
|
|
294
|
+
assert.equal(await queryTemporalTimelineAsync(memoryDir), null);
|
|
295
|
+
|
|
296
|
+
await writeFile(
|
|
297
|
+
join(memoryDir, "state", "index_time.json"),
|
|
298
|
+
JSON.stringify({ version: 1, dates: {} }),
|
|
299
|
+
"utf8",
|
|
300
|
+
);
|
|
301
|
+
assert.equal(indexesExist(memoryDir), false);
|
|
302
|
+
assert.equal(await queryTemporalTimelineAsync(memoryDir), null);
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
test("deindex removes timeline rows so stale events cannot be recalled", async () => {
|
|
306
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-delete-"));
|
|
307
|
+
const memoryPath = "/tmp/deleted.md";
|
|
308
|
+
const createdAt = "2026-03-04T00:00:00.000Z";
|
|
309
|
+
indexMemory(memoryDir, memoryPath, createdAt, ["trip"], { sessionKey: "session-a" });
|
|
310
|
+
deindexMemory(memoryDir, memoryPath, createdAt, ["trip"]);
|
|
311
|
+
assert.deepEqual(await queryTemporalTimelineAsync(memoryDir), []);
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
test("large temporal indexes return a bounded relevant oversample before memory reads", async () => {
|
|
315
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-bounded-"));
|
|
316
|
+
for (let i = 0; i < 600; i += 1) {
|
|
317
|
+
const day = String((i % 28) + 1).padStart(2, "0");
|
|
318
|
+
indexMemory(
|
|
319
|
+
memoryDir,
|
|
320
|
+
`/tmp/event-${String(i).padStart(4, "0")}.md`,
|
|
321
|
+
`2026-03-${day}T00:00:00.000Z`,
|
|
322
|
+
[],
|
|
323
|
+
{
|
|
324
|
+
validAt: `2026-03-${day}T00:00:00.000Z`,
|
|
325
|
+
searchText: i === 317 ? "unique-marzipan-trip to Lisbon" : `routine event ${i}`,
|
|
326
|
+
},
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const relevant = await queryTemporalTimelineAsync(memoryDir, {
|
|
331
|
+
query: "When was the marzipan trip to Lisbon?",
|
|
332
|
+
limit: 48,
|
|
333
|
+
});
|
|
334
|
+
assert.ok(relevant);
|
|
335
|
+
assert.ok(relevant.length <= 48);
|
|
336
|
+
assert.ok(relevant.some((entry) => entry.path.endsWith("event-0317.md")));
|
|
337
|
+
|
|
338
|
+
const generic = await queryTemporalTimelineAsync(memoryDir, {
|
|
339
|
+
query: "What happened first or last?",
|
|
340
|
+
limit: 48,
|
|
341
|
+
});
|
|
342
|
+
assert.equal(generic?.length, 48);
|
|
343
|
+
assert.equal(generic?.[0]?.eventAt, "2026-03-01T00:00:00.000Z");
|
|
344
|
+
assert.equal(generic?.at(-1)?.eventAt, "2026-03-28T00:00:00.000Z");
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
test("temporal timeline generic edge selection honors small and zero limits", async () => {
|
|
348
|
+
const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-timeline-small-limit-"));
|
|
349
|
+
for (let day = 1; day <= 4; day += 1) {
|
|
350
|
+
const timestamp = `2026-04-0${day}T00:00:00.000Z`;
|
|
351
|
+
indexMemory(memoryDir, `/tmp/edge-${day}.md`, timestamp, [], {
|
|
352
|
+
validAt: timestamp,
|
|
353
|
+
searchText: `unrelated event ${day}`,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const genericQuery = "What happened first or last?";
|
|
358
|
+
const limitOne = await queryTemporalTimelineAsync(memoryDir, {
|
|
359
|
+
query: genericQuery,
|
|
360
|
+
limit: 1,
|
|
361
|
+
});
|
|
362
|
+
assert.deepEqual(limitOne?.map((entry) => entry.path), ["/tmp/edge-1.md"]);
|
|
363
|
+
|
|
364
|
+
const limitTwo = await queryTemporalTimelineAsync(memoryDir, {
|
|
365
|
+
query: genericQuery,
|
|
366
|
+
limit: 2,
|
|
367
|
+
});
|
|
368
|
+
assert.deepEqual(limitTwo?.map((entry) => entry.path), [
|
|
369
|
+
"/tmp/edge-1.md",
|
|
370
|
+
"/tmp/edge-4.md",
|
|
371
|
+
]);
|
|
372
|
+
|
|
373
|
+
assert.deepEqual(
|
|
374
|
+
await queryTemporalTimelineAsync(memoryDir, { query: genericQuery, limit: 0 }),
|
|
375
|
+
[],
|
|
376
|
+
);
|
|
377
|
+
assert.deepEqual(
|
|
378
|
+
await queryTemporalTimelineAsync(memoryDir, { query: genericQuery, limit: -3 }),
|
|
379
|
+
[],
|
|
380
|
+
);
|
|
381
|
+
assert.deepEqual(
|
|
382
|
+
(await queryTemporalTimelineAsync(memoryDir, {
|
|
383
|
+
query: genericQuery,
|
|
384
|
+
limit: 1.9,
|
|
385
|
+
}))?.map((entry) => entry.path),
|
|
386
|
+
["/tmp/edge-1.md"],
|
|
387
|
+
);
|
|
388
|
+
assert.equal(
|
|
389
|
+
(await queryTemporalTimelineAsync(memoryDir, {
|
|
390
|
+
query: genericQuery,
|
|
391
|
+
limit: Number.POSITIVE_INFINITY,
|
|
392
|
+
}))?.length,
|
|
393
|
+
4,
|
|
394
|
+
);
|
|
395
|
+
});
|
package/src/temporal-index.ts
CHANGED
|
@@ -27,6 +27,37 @@ export interface TemporalIndex {
|
|
|
27
27
|
lastRebuildAt?: string;
|
|
28
28
|
/** Map from YYYY-MM-DD → array of memory paths */
|
|
29
29
|
dates: Record<string, string[]>;
|
|
30
|
+
/**
|
|
31
|
+
* Per-memory event-time rows used to reconstruct one chronology across
|
|
32
|
+
* source sessions. Keyed by path so an update replaces the prior row.
|
|
33
|
+
*/
|
|
34
|
+
events: Record<string, TemporalIndexEvent>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface TemporalIndexEvent {
|
|
38
|
+
path: string;
|
|
39
|
+
/** Event/valid time used for chronology. */
|
|
40
|
+
eventAt: string;
|
|
41
|
+
/** Ingest/source-observation time, when known. */
|
|
42
|
+
observedAt?: string;
|
|
43
|
+
/** Source session provenance, when the extractor recorded it. */
|
|
44
|
+
sessionKey?: string;
|
|
45
|
+
/** Exclusive validity end, when the event is bounded. */
|
|
46
|
+
validUntil?: string;
|
|
47
|
+
/** SHA-256 lexical fingerprints for bounded query-aware candidate selection. */
|
|
48
|
+
searchTokenHashes?: string[];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface TemporalIndexEntry {
|
|
52
|
+
path: string;
|
|
53
|
+
createdAt: string;
|
|
54
|
+
tags: string[];
|
|
55
|
+
validAt?: string;
|
|
56
|
+
observedAt?: string;
|
|
57
|
+
sessionKey?: string;
|
|
58
|
+
validUntil?: string;
|
|
59
|
+
/** Used only to derive token hashes; raw text is never persisted in the index. */
|
|
60
|
+
searchText?: string;
|
|
30
61
|
}
|
|
31
62
|
|
|
32
63
|
export interface TagIndex {
|
|
@@ -44,7 +75,7 @@ export interface TagNode {
|
|
|
44
75
|
parents?: string[];
|
|
45
76
|
}
|
|
46
77
|
|
|
47
|
-
const INDEX_VERSION =
|
|
78
|
+
const INDEX_VERSION = 2;
|
|
48
79
|
const TEMPORAL_INDEX_FILE = "index_time.json";
|
|
49
80
|
const TAG_INDEX_FILE = "index_tags.json";
|
|
50
81
|
const TAG_INDEX_VERSION = 2;
|
|
@@ -275,10 +306,28 @@ function writeJsonAtomic(filePath: string, data: unknown): void {
|
|
|
275
306
|
}
|
|
276
307
|
}
|
|
277
308
|
|
|
309
|
+
function emptyTemporalIndex(): TemporalIndex {
|
|
310
|
+
return { version: INDEX_VERSION, dates: {}, events: {} };
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
314
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function normalizeTemporalIndex(raw: TemporalIndex | null | undefined): TemporalIndex {
|
|
318
|
+
if (!isRecord(raw)) return emptyTemporalIndex();
|
|
319
|
+
return {
|
|
320
|
+
version: typeof raw.version === "number" ? raw.version : 0,
|
|
321
|
+
dates: isRecord(raw.dates) ? raw.dates as Record<string, string[]> : {},
|
|
322
|
+
events: isRecord(raw.events) ? raw.events as Record<string, TemporalIndexEvent> : {},
|
|
323
|
+
...(typeof raw.lastRebuildAt === "string" ? { lastRebuildAt: raw.lastRebuildAt } : {}),
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
278
327
|
function updateTemporalIndex(memoryDir: string, update: (index: TemporalIndex) => void): void {
|
|
279
328
|
const indexPath = temporalIndexPath(memoryDir);
|
|
280
329
|
withIndexFileLock(indexPath, () => {
|
|
281
|
-
const index = readJsonSafe<TemporalIndex>(indexPath,
|
|
330
|
+
const index = normalizeTemporalIndex(readJsonSafe<TemporalIndex>(indexPath, emptyTemporalIndex()));
|
|
282
331
|
update(index);
|
|
283
332
|
writeJsonAtomic(indexPath, index);
|
|
284
333
|
});
|
|
@@ -305,6 +354,62 @@ function isoDateFromTimestamp(isoString: string): string {
|
|
|
305
354
|
return isoString.slice(0, 10); // YYYY-MM-DD
|
|
306
355
|
}
|
|
307
356
|
|
|
357
|
+
function normalizedIsoTimestamp(value: string | undefined): string | undefined {
|
|
358
|
+
if (typeof value !== "string" || value.trim().length === 0) return undefined;
|
|
359
|
+
const ms = Date.parse(value);
|
|
360
|
+
if (!Number.isFinite(ms)) return undefined;
|
|
361
|
+
try {
|
|
362
|
+
return new Date(ms).toISOString();
|
|
363
|
+
} catch {
|
|
364
|
+
return undefined;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function temporalEventFromEntry(entry: TemporalIndexEntry): TemporalIndexEvent {
|
|
369
|
+
const createdAt = normalizedIsoTimestamp(entry.createdAt) ?? new Date(0).toISOString();
|
|
370
|
+
const eventAt = normalizedIsoTimestamp(entry.validAt) ?? createdAt;
|
|
371
|
+
const observedAt = normalizedIsoTimestamp(entry.observedAt);
|
|
372
|
+
const validUntil = normalizedIsoTimestamp(entry.validUntil);
|
|
373
|
+
const sessionKey = typeof entry.sessionKey === "string" && entry.sessionKey.trim().length > 0
|
|
374
|
+
? entry.sessionKey.replace(/[\u0000-\u001f\u007f]+/g, " ").trim()
|
|
375
|
+
: undefined;
|
|
376
|
+
const searchTokenHashes = temporalSearchTokenHashes(entry.searchText);
|
|
377
|
+
return {
|
|
378
|
+
path: entry.path,
|
|
379
|
+
eventAt,
|
|
380
|
+
...(observedAt ? { observedAt } : {}),
|
|
381
|
+
...(sessionKey ? { sessionKey } : {}),
|
|
382
|
+
...(validUntil ? { validUntil } : {}),
|
|
383
|
+
...(searchTokenHashes.length > 0 ? { searchTokenHashes } : {}),
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function temporalSearchTokens(value: string | undefined): string[] {
|
|
388
|
+
if (!value) return [];
|
|
389
|
+
const tokens = value
|
|
390
|
+
.toLowerCase()
|
|
391
|
+
.match(/[\p{L}\p{N}][\p{L}\p{N}'_-]*/gu)
|
|
392
|
+
?.filter((token) => token.length > 1) ?? [];
|
|
393
|
+
return [...new Set(tokens)].slice(0, 128);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function temporalSearchTokenHashes(value: string | undefined): string[] {
|
|
397
|
+
return temporalSearchTokens(value).map((token) =>
|
|
398
|
+
crypto.createHash("sha256").update(token).digest("hex"),
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function compareTemporalIndexEvents(
|
|
403
|
+
left: TemporalIndexEvent,
|
|
404
|
+
right: TemporalIndexEvent,
|
|
405
|
+
): number {
|
|
406
|
+
const byEvent = left.eventAt.localeCompare(right.eventAt);
|
|
407
|
+
if (byEvent !== 0) return byEvent;
|
|
408
|
+
const byObservation = (left.observedAt ?? "").localeCompare(right.observedAt ?? "");
|
|
409
|
+
if (byObservation !== 0) return byObservation;
|
|
410
|
+
return left.path.localeCompare(right.path);
|
|
411
|
+
}
|
|
412
|
+
|
|
308
413
|
function addPathToSet(record: Record<string, string[]>, key: string, p: string): void {
|
|
309
414
|
if (!record[key]) {
|
|
310
415
|
record[key] = [];
|
|
@@ -590,14 +695,27 @@ function promptContainsAlias(prompt: string, alias: string): boolean {
|
|
|
590
695
|
* @param createdAt ISO timestamp of the memory's creation date
|
|
591
696
|
* @param tags Array of tag strings from the memory's frontmatter
|
|
592
697
|
*/
|
|
593
|
-
export function indexMemory(
|
|
698
|
+
export function indexMemory(
|
|
699
|
+
memoryDir: string,
|
|
700
|
+
memoryPath: string,
|
|
701
|
+
createdAt: string,
|
|
702
|
+
tags: string[],
|
|
703
|
+
temporal: Omit<Partial<TemporalIndexEntry>, "path" | "createdAt" | "tags"> = {},
|
|
704
|
+
): void {
|
|
594
705
|
try {
|
|
595
706
|
ensureStateDir(memoryDir);
|
|
596
707
|
|
|
597
708
|
const dateKey = isoDateFromTimestamp(createdAt);
|
|
598
709
|
updateTemporalIndex(memoryDir, (index) => {
|
|
710
|
+
index.version = INDEX_VERSION;
|
|
599
711
|
removePathFromAllSets(index.dates, memoryPath);
|
|
600
712
|
addPathToSet(index.dates, dateKey, memoryPath);
|
|
713
|
+
index.events[memoryPath] = temporalEventFromEntry({
|
|
714
|
+
path: memoryPath,
|
|
715
|
+
createdAt,
|
|
716
|
+
tags,
|
|
717
|
+
...temporal,
|
|
718
|
+
});
|
|
601
719
|
});
|
|
602
720
|
|
|
603
721
|
updateTagIndex(memoryDir, (index) => {
|
|
@@ -623,6 +741,7 @@ export function deindexMemory(memoryDir: string, memoryPath: string, createdAt:
|
|
|
623
741
|
const dateKey = isoDateFromTimestamp(createdAt);
|
|
624
742
|
updateTemporalIndex(memoryDir, (index) => {
|
|
625
743
|
removePathFromSet(index.dates, dateKey, memoryPath);
|
|
744
|
+
delete index.events[memoryPath];
|
|
626
745
|
});
|
|
627
746
|
|
|
628
747
|
updateTagIndex(memoryDir, (index) => {
|
|
@@ -649,6 +768,7 @@ export function clearIndexes(memoryDir: string): void {
|
|
|
649
768
|
index.version = INDEX_VERSION;
|
|
650
769
|
index.lastRebuildAt = undefined;
|
|
651
770
|
index.dates = {};
|
|
771
|
+
index.events = {};
|
|
652
772
|
});
|
|
653
773
|
updateTagIndex(memoryDir, (index) => {
|
|
654
774
|
index.version = TAG_INDEX_VERSION;
|
|
@@ -667,7 +787,9 @@ export function clearIndexes(memoryDir: string): void {
|
|
|
667
787
|
*/
|
|
668
788
|
export function indexesExist(memoryDir: string): boolean {
|
|
669
789
|
try {
|
|
670
|
-
|
|
790
|
+
if (!fs.existsSync(tagIndexPath(memoryDir))) return false;
|
|
791
|
+
const raw = JSON.parse(fs.readFileSync(temporalIndexPath(memoryDir), "utf8")) as unknown;
|
|
792
|
+
return isRecord(raw) && raw.version === INDEX_VERSION && isRecord(raw.dates) && isRecord(raw.events);
|
|
671
793
|
} catch {
|
|
672
794
|
return false;
|
|
673
795
|
}
|
|
@@ -679,17 +801,19 @@ export function indexesExist(memoryDir: string): boolean {
|
|
|
679
801
|
*/
|
|
680
802
|
export function indexMemoriesBatch(
|
|
681
803
|
memoryDir: string,
|
|
682
|
-
entries:
|
|
804
|
+
entries: TemporalIndexEntry[]
|
|
683
805
|
): void {
|
|
684
806
|
if (entries.length === 0) return;
|
|
685
807
|
try {
|
|
686
808
|
ensureStateDir(memoryDir);
|
|
687
809
|
|
|
688
810
|
updateTemporalIndex(memoryDir, (index) => {
|
|
811
|
+
index.version = INDEX_VERSION;
|
|
689
812
|
for (const entry of entries) {
|
|
690
813
|
const dateKey = isoDateFromTimestamp(entry.createdAt);
|
|
691
814
|
removePathFromAllSets(index.dates, entry.path);
|
|
692
815
|
addPathToSet(index.dates, dateKey, entry.path);
|
|
816
|
+
index.events[entry.path] = temporalEventFromEntry(entry);
|
|
693
817
|
}
|
|
694
818
|
});
|
|
695
819
|
|
|
@@ -738,9 +862,9 @@ export async function queryByDateRangeAsync(
|
|
|
738
862
|
}
|
|
739
863
|
let tIndex: TemporalIndex;
|
|
740
864
|
try {
|
|
741
|
-
tIndex = JSON.parse(raw) as TemporalIndex;
|
|
865
|
+
tIndex = normalizeTemporalIndex(JSON.parse(raw) as TemporalIndex);
|
|
742
866
|
} catch {
|
|
743
|
-
tIndex =
|
|
867
|
+
tIndex = emptyTemporalIndex();
|
|
744
868
|
}
|
|
745
869
|
// toDate is exclusive (first day NOT included). Default: tomorrow so "all of today" is included.
|
|
746
870
|
const end = toDate ?? new Date(Date.now() + 86_400_000).toISOString().slice(0, 10);
|
|
@@ -759,6 +883,86 @@ export async function queryByDateRangeAsync(
|
|
|
759
883
|
}
|
|
760
884
|
}
|
|
761
885
|
|
|
886
|
+
/**
|
|
887
|
+
* Return the persisted event-time rows as one deterministic cross-session
|
|
888
|
+
* chronology. Equal event times are ordered by observation time, then path.
|
|
889
|
+
* Returns `null` when the index is missing, corrupt, or from an older schema;
|
|
890
|
+
* callers can then fail open to the pre-index recall path.
|
|
891
|
+
*/
|
|
892
|
+
export async function queryTemporalTimelineAsync(
|
|
893
|
+
memoryDir: string,
|
|
894
|
+
options: { query?: string; limit?: number } = {},
|
|
895
|
+
): Promise<TemporalIndexEvent[] | null> {
|
|
896
|
+
try {
|
|
897
|
+
const raw = await fs.promises.readFile(temporalIndexPath(memoryDir), "utf8");
|
|
898
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
899
|
+
if (
|
|
900
|
+
!isRecord(parsed) ||
|
|
901
|
+
parsed.version !== INDEX_VERSION ||
|
|
902
|
+
!isRecord(parsed.dates) ||
|
|
903
|
+
!isRecord(parsed.events)
|
|
904
|
+
) return null;
|
|
905
|
+
const index = normalizeTemporalIndex(parsed as unknown as TemporalIndex);
|
|
906
|
+
const chronology = Object.values(index.events)
|
|
907
|
+
.filter((event) =>
|
|
908
|
+
typeof event?.path === "string" &&
|
|
909
|
+
event.path.length > 0 &&
|
|
910
|
+
normalizedIsoTimestamp(event.eventAt) !== undefined
|
|
911
|
+
)
|
|
912
|
+
.map((event) => ({
|
|
913
|
+
path: event.path,
|
|
914
|
+
eventAt: normalizedIsoTimestamp(event.eventAt)!,
|
|
915
|
+
...(normalizedIsoTimestamp(event.observedAt)
|
|
916
|
+
? { observedAt: normalizedIsoTimestamp(event.observedAt) }
|
|
917
|
+
: {}),
|
|
918
|
+
...(typeof event.sessionKey === "string" && event.sessionKey.trim().length > 0
|
|
919
|
+
? { sessionKey: event.sessionKey.trim() }
|
|
920
|
+
: {}),
|
|
921
|
+
...(normalizedIsoTimestamp(event.validUntil)
|
|
922
|
+
? { validUntil: normalizedIsoTimestamp(event.validUntil) }
|
|
923
|
+
: {}),
|
|
924
|
+
...(Array.isArray(event.searchTokenHashes)
|
|
925
|
+
? { searchTokenHashes: event.searchTokenHashes.filter((hash) => typeof hash === "string") }
|
|
926
|
+
: {}),
|
|
927
|
+
}))
|
|
928
|
+
.sort(compareTemporalIndexEvents);
|
|
929
|
+
const requestedLimit = typeof options.limit === "number" && Number.isFinite(options.limit)
|
|
930
|
+
? Math.max(0, Math.floor(options.limit))
|
|
931
|
+
: chronology.length;
|
|
932
|
+
if (requestedLimit === 0) return [];
|
|
933
|
+
if (chronology.length <= requestedLimit) return chronology;
|
|
934
|
+
|
|
935
|
+
const queryHashes = new Set(temporalSearchTokenHashes(options.query));
|
|
936
|
+
const scored = chronology.map((event) => ({
|
|
937
|
+
event,
|
|
938
|
+
score: (event.searchTokenHashes ?? []).reduce(
|
|
939
|
+
(sum, hash) => sum + (queryHashes.has(hash) ? 1 : 0),
|
|
940
|
+
0,
|
|
941
|
+
),
|
|
942
|
+
}));
|
|
943
|
+
if (scored.some(({ score }) => score > 0)) {
|
|
944
|
+
return scored
|
|
945
|
+
.filter(({ score }) => score > 0)
|
|
946
|
+
.sort((left, right) => right.score - left.score || compareTemporalIndexEvents(left.event, right.event))
|
|
947
|
+
.slice(0, requestedLimit)
|
|
948
|
+
.map(({ event }) => event)
|
|
949
|
+
.sort(compareTemporalIndexEvents);
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// No lexical match (legacy rows or generic "first/last" query): retain
|
|
953
|
+
// both chronology edges so earliest and latest questions remain answerable.
|
|
954
|
+
const earlyCount = Math.ceil(requestedLimit / 2);
|
|
955
|
+
const lateCount = requestedLimit - earlyCount;
|
|
956
|
+
const selected = [
|
|
957
|
+
...chronology.slice(0, earlyCount),
|
|
958
|
+
...(lateCount > 0 ? chronology.slice(-lateCount) : []),
|
|
959
|
+
];
|
|
960
|
+
return selected.sort(compareTemporalIndexEvents);
|
|
961
|
+
} catch {
|
|
962
|
+
return null;
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
762
966
|
/**
|
|
763
967
|
* Async version of queryByTags — uses non-blocking fs.promises.readFile
|
|
764
968
|
* to avoid blocking the Node.js event loop.
|