@remnic/core 9.3.762 → 9.3.763

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/access-admin-ops-surface.js +5 -5
  2. package/dist/access-boundary.js +6 -6
  3. package/dist/access-cli.js +26 -26
  4. package/dist/access-http.js +10 -10
  5. package/dist/access-identity-continuity-surface.js +5 -5
  6. package/dist/access-lcm-surface.js +5 -5
  7. package/dist/access-mcp.js +9 -9
  8. package/dist/access-observe-write-surface.js +5 -5
  9. package/dist/access-operations-batch.js +7 -7
  10. package/dist/access-operations.js +8 -8
  11. package/dist/access-recall-surface.js +5 -5
  12. package/dist/access-service.js +5 -5
  13. package/dist/briefing.js +3 -3
  14. package/dist/causal-consolidation.js +4 -4
  15. package/dist/{chunk-OQ6F3WBX.js → chunk-2RLECZ6J.js} +2 -2
  16. package/dist/{chunk-TIWRP5NW.js → chunk-6YEENRE3.js} +4 -4
  17. package/dist/{chunk-WAIMHMO3.js → chunk-6ZFE2TPO.js} +3 -3
  18. package/dist/{chunk-DPZ5GJMD.js → chunk-A4FMQIST.js} +2 -2
  19. package/dist/{chunk-ZTBH2OHD.js → chunk-AS2NRB45.js} +2 -2
  20. package/dist/{chunk-EDC7RZCW.js → chunk-BAF7MBEH.js} +13 -13
  21. package/dist/{chunk-HDFS6RWQ.js → chunk-CVINPTKG.js} +2 -2
  22. package/dist/{chunk-DB647OZ4.js → chunk-D6YTILJO.js} +13 -13
  23. package/dist/{chunk-P7W4CDX6.js → chunk-DTTVX6EO.js} +11 -3
  24. package/dist/chunk-DTTVX6EO.js.map +1 -0
  25. package/dist/{chunk-HDYFNXVO.js → chunk-IVFIYAIP.js} +2 -2
  26. package/dist/{chunk-JU6BKZHX.js → chunk-K4EDULCP.js} +3 -3
  27. package/dist/{chunk-QPFO54VC.js → chunk-KXG7HHIQ.js} +2 -2
  28. package/dist/{chunk-2KYXXMTE.js → chunk-LDCFFJ67.js} +4 -4
  29. package/dist/{chunk-ESDBHQY6.js → chunk-N3QDLE2N.js} +5 -5
  30. package/dist/{chunk-4HYJETVO.js → chunk-N4ZA2I7M.js} +2 -2
  31. package/dist/{chunk-TEKTCF2V.js → chunk-NDXJJKT2.js} +2 -2
  32. package/dist/{chunk-2PGB63VU.js → chunk-NFWTMM2M.js} +2 -2
  33. package/dist/{chunk-PNFN7SIN.js → chunk-OX272PBW.js} +2 -2
  34. package/dist/{chunk-HMUEAKSD.js → chunk-QPTP6SKC.js} +2 -2
  35. package/dist/{chunk-VHYFY3JI.js → chunk-R3K4CXPR.js} +2 -2
  36. package/dist/{chunk-IIURIWJG.js → chunk-SFNQDXXL.js} +9 -5
  37. package/dist/chunk-SFNQDXXL.js.map +1 -0
  38. package/dist/{chunk-MB5RSUW6.js → chunk-SXN3LCUT.js} +114 -28
  39. package/dist/chunk-SXN3LCUT.js.map +1 -0
  40. package/dist/{chunk-LAJIXABF.js → chunk-TIOXQPD7.js} +5 -5
  41. package/dist/{chunk-ES3GAMSQ.js → chunk-VKHZFF5P.js} +2 -2
  42. package/dist/{chunk-KZWUJC7Y.js → chunk-X7HTU4HO.js} +2 -2
  43. package/dist/{chunk-QWIE43BS.js → chunk-YDYC6PIS.js} +2 -2
  44. package/dist/cli.js +20 -20
  45. package/dist/compounding/engine.js +3 -3
  46. package/dist/connectors/codex-materialize-runner.js +3 -3
  47. package/dist/connectors/index.js +3 -3
  48. package/dist/entity-retrieval.js +3 -3
  49. package/dist/index.js +26 -26
  50. package/dist/maintenance/memory-governance.js +3 -3
  51. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
  52. package/dist/maintenance/rebuild-memory-projection.js +4 -4
  53. package/dist/memory-projection-store.d.ts +6 -2
  54. package/dist/memory-projection-store.js +9 -3
  55. package/dist/namespaces/migrate.js +4 -4
  56. package/dist/namespaces/storage.js +3 -3
  57. package/dist/operator-toolkit.js +7 -7
  58. package/dist/orchestration/maintenance.js +5 -5
  59. package/dist/orchestrator.js +26 -26
  60. package/dist/{purge-6ATBGT77.js → purge-IHRX4Z7X.js} +7 -1
  61. package/dist/purge-IHRX4Z7X.js.map +1 -0
  62. package/dist/semantic-consolidation.js +4 -4
  63. package/dist/semantic-rule-promotion.js +3 -3
  64. package/dist/semantic-rule-verifier.js +3 -3
  65. package/dist/storage.js +2 -2
  66. package/dist/verified-recall.js +3 -3
  67. package/package.json +2 -2
  68. package/src/maintenance/purge.test.ts +76 -0
  69. package/src/maintenance/purge.ts +3 -0
  70. package/src/maintenance/rebuild-memory-projection.ts +5 -1
  71. package/src/memory-projection-store.ts +143 -35
  72. package/src/storage.ts +8 -8
  73. package/dist/chunk-IIURIWJG.js.map +0 -1
  74. package/dist/chunk-MB5RSUW6.js.map +0 -1
  75. package/dist/chunk-P7W4CDX6.js.map +0 -1
  76. package/dist/purge-6ATBGT77.js.map +0 -1
  77. /package/dist/{chunk-OQ6F3WBX.js.map → chunk-2RLECZ6J.js.map} +0 -0
  78. /package/dist/{chunk-TIWRP5NW.js.map → chunk-6YEENRE3.js.map} +0 -0
  79. /package/dist/{chunk-WAIMHMO3.js.map → chunk-6ZFE2TPO.js.map} +0 -0
  80. /package/dist/{chunk-DPZ5GJMD.js.map → chunk-A4FMQIST.js.map} +0 -0
  81. /package/dist/{chunk-ZTBH2OHD.js.map → chunk-AS2NRB45.js.map} +0 -0
  82. /package/dist/{chunk-EDC7RZCW.js.map → chunk-BAF7MBEH.js.map} +0 -0
  83. /package/dist/{chunk-HDFS6RWQ.js.map → chunk-CVINPTKG.js.map} +0 -0
  84. /package/dist/{chunk-DB647OZ4.js.map → chunk-D6YTILJO.js.map} +0 -0
  85. /package/dist/{chunk-HDYFNXVO.js.map → chunk-IVFIYAIP.js.map} +0 -0
  86. /package/dist/{chunk-JU6BKZHX.js.map → chunk-K4EDULCP.js.map} +0 -0
  87. /package/dist/{chunk-QPFO54VC.js.map → chunk-KXG7HHIQ.js.map} +0 -0
  88. /package/dist/{chunk-2KYXXMTE.js.map → chunk-LDCFFJ67.js.map} +0 -0
  89. /package/dist/{chunk-ESDBHQY6.js.map → chunk-N3QDLE2N.js.map} +0 -0
  90. /package/dist/{chunk-4HYJETVO.js.map → chunk-N4ZA2I7M.js.map} +0 -0
  91. /package/dist/{chunk-TEKTCF2V.js.map → chunk-NDXJJKT2.js.map} +0 -0
  92. /package/dist/{chunk-2PGB63VU.js.map → chunk-NFWTMM2M.js.map} +0 -0
  93. /package/dist/{chunk-PNFN7SIN.js.map → chunk-OX272PBW.js.map} +0 -0
  94. /package/dist/{chunk-HMUEAKSD.js.map → chunk-QPTP6SKC.js.map} +0 -0
  95. /package/dist/{chunk-VHYFY3JI.js.map → chunk-R3K4CXPR.js.map} +0 -0
  96. /package/dist/{chunk-LAJIXABF.js.map → chunk-TIOXQPD7.js.map} +0 -0
  97. /package/dist/{chunk-ES3GAMSQ.js.map → chunk-VKHZFF5P.js.map} +0 -0
  98. /package/dist/{chunk-KZWUJC7Y.js.map → chunk-X7HTU4HO.js.map} +0 -0
  99. /package/dist/{chunk-QWIE43BS.js.map → chunk-YDYC6PIS.js.map} +0 -0
@@ -7,14 +7,14 @@ import {
7
7
  materializeAfterSemanticConsolidation,
8
8
  parseConsolidationResponse,
9
9
  parseOperatorAwareConsolidationResponse
10
- } from "./chunk-HDFS6RWQ.js";
11
- import "./chunk-OQ6F3WBX.js";
10
+ } from "./chunk-CVINPTKG.js";
11
+ import "./chunk-2RLECZ6J.js";
12
12
  import "./chunk-UUH4YQOF.js";
13
13
  import {
14
14
  resolveExtensionsRoot
15
15
  } from "./chunk-JLNBQWZ2.js";
16
16
  import "./chunk-3UXOZBHV.js";
17
- import "./chunk-P7W4CDX6.js";
17
+ import "./chunk-DTTVX6EO.js";
18
18
  import "./chunk-M7XQSUBB.js";
19
19
  import "./chunk-L2B3EE7B.js";
20
20
  import "./chunk-5UZXUTVO.js";
@@ -30,7 +30,7 @@ import "./chunk-AWJ2FHCF.js";
30
30
  import "./chunk-42JKGUFJ.js";
31
31
  import "./chunk-RULE4VG5.js";
32
32
  import "./chunk-SCU65EZI.js";
33
- import "./chunk-MB5RSUW6.js";
33
+ import "./chunk-SXN3LCUT.js";
34
34
  import "./chunk-6KYMPV2O.js";
35
35
  import "./chunk-DM2T26WE.js";
36
36
  import "./chunk-LDXUBPMO.js";
@@ -1,8 +1,8 @@
1
1
  import {
2
2
  promoteSemanticRuleFromMemory,
3
3
  setSemanticRulePromotionTestHooks
4
- } from "./chunk-KZWUJC7Y.js";
5
- import "./chunk-P7W4CDX6.js";
4
+ } from "./chunk-X7HTU4HO.js";
5
+ import "./chunk-DTTVX6EO.js";
6
6
  import "./chunk-M7XQSUBB.js";
7
7
  import "./chunk-L2B3EE7B.js";
8
8
  import "./chunk-5UZXUTVO.js";
@@ -18,7 +18,7 @@ import "./chunk-AWJ2FHCF.js";
18
18
  import "./chunk-42JKGUFJ.js";
19
19
  import "./chunk-RULE4VG5.js";
20
20
  import "./chunk-SCU65EZI.js";
21
- import "./chunk-MB5RSUW6.js";
21
+ import "./chunk-SXN3LCUT.js";
22
22
  import "./chunk-6KYMPV2O.js";
23
23
  import "./chunk-DM2T26WE.js";
24
24
  import "./chunk-LDXUBPMO.js";
@@ -1,8 +1,8 @@
1
1
  import {
2
2
  compareVerifiedSemanticRuleResults,
3
3
  searchVerifiedSemanticRules
4
- } from "./chunk-ES3GAMSQ.js";
5
- import "./chunk-P7W4CDX6.js";
4
+ } from "./chunk-VKHZFF5P.js";
5
+ import "./chunk-DTTVX6EO.js";
6
6
  import "./chunk-M7XQSUBB.js";
7
7
  import "./chunk-L2B3EE7B.js";
8
8
  import "./chunk-5UZXUTVO.js";
@@ -18,7 +18,7 @@ import "./chunk-AWJ2FHCF.js";
18
18
  import "./chunk-42JKGUFJ.js";
19
19
  import "./chunk-RULE4VG5.js";
20
20
  import "./chunk-SCU65EZI.js";
21
- import "./chunk-MB5RSUW6.js";
21
+ import "./chunk-SXN3LCUT.js";
22
22
  import "./chunk-6KYMPV2O.js";
23
23
  import "./chunk-DM2T26WE.js";
24
24
  import "./chunk-LDXUBPMO.js";
package/dist/storage.js CHANGED
@@ -16,7 +16,7 @@ import {
16
16
  parseFrontmatter,
17
17
  parseFrontmatterStringValue,
18
18
  serializeEntityFile
19
- } from "./chunk-P7W4CDX6.js";
19
+ } from "./chunk-DTTVX6EO.js";
20
20
  import "./chunk-M7XQSUBB.js";
21
21
  import "./chunk-L2B3EE7B.js";
22
22
  import "./chunk-5UZXUTVO.js";
@@ -34,7 +34,7 @@ import "./chunk-AWJ2FHCF.js";
34
34
  import "./chunk-42JKGUFJ.js";
35
35
  import "./chunk-RULE4VG5.js";
36
36
  import "./chunk-SCU65EZI.js";
37
- import "./chunk-MB5RSUW6.js";
37
+ import "./chunk-SXN3LCUT.js";
38
38
  import "./chunk-6KYMPV2O.js";
39
39
  import "./chunk-DM2T26WE.js";
40
40
  import "./chunk-LDXUBPMO.js";
@@ -1,9 +1,9 @@
1
1
  import {
2
2
  compareVerifiedEpisodeResults,
3
3
  searchVerifiedEpisodes
4
- } from "./chunk-4HYJETVO.js";
4
+ } from "./chunk-N4ZA2I7M.js";
5
5
  import "./chunk-7FL4CNPV.js";
6
- import "./chunk-P7W4CDX6.js";
6
+ import "./chunk-DTTVX6EO.js";
7
7
  import "./chunk-M7XQSUBB.js";
8
8
  import "./chunk-L2B3EE7B.js";
9
9
  import "./chunk-5UZXUTVO.js";
@@ -19,7 +19,7 @@ import "./chunk-AWJ2FHCF.js";
19
19
  import "./chunk-42JKGUFJ.js";
20
20
  import "./chunk-RULE4VG5.js";
21
21
  import "./chunk-SCU65EZI.js";
22
- import "./chunk-MB5RSUW6.js";
22
+ import "./chunk-SXN3LCUT.js";
23
23
  import "./chunk-6KYMPV2O.js";
24
24
  import "./chunk-DM2T26WE.js";
25
25
  import "./chunk-LDXUBPMO.js";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remnic/core",
3
- "version": "9.3.762",
3
+ "version": "9.3.763",
4
4
  "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -3000,7 +3000,7 @@
3000
3000
  "core"
3001
3001
  ],
3002
3002
  "peerDependencies": {
3003
- "@remnic/coding-graph": "^9.3.762"
3003
+ "@remnic/coding-graph": "^9.3.763"
3004
3004
  },
3005
3005
  "peerDependenciesMeta": {
3006
3006
  "@remnic/coding-graph": {
@@ -5,6 +5,9 @@ import path from "node:path";
5
5
  import test from "node:test";
6
6
 
7
7
  import { purgeMemories } from "./purge.js";
8
+ import { rebuildMemoryProjection } from "./rebuild-memory-projection.js";
9
+ import { readProjectedMemoryBrowse } from "../memory-projection-store.js";
10
+ import { StorageManager } from "../storage.js";
8
11
  import type { MemoryFile } from "../types.js";
9
12
 
10
13
  test("purgeMemories records audit errors without blocking hard delete", async () => {
@@ -54,6 +57,79 @@ test("purgeMemories records audit errors without blocking hard delete", async ()
54
57
  }
55
58
  });
56
59
 
60
+ test("purgeMemories invalidates projection rows for hard-deleted memories", async () => {
61
+ const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-projection-"));
62
+ try {
63
+ const storage = new StorageManager(dir);
64
+ const { id } = await storage.writeMemory("fact", "purge projection target", {
65
+ source: "test",
66
+ });
67
+ const memory = await storage.getMemoryById(id);
68
+ assert.ok(memory);
69
+ await rebuildMemoryProjection({ memoryDir: dir, dryRun: false });
70
+ assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 1);
71
+
72
+ const result = await purgeMemories({
73
+ storage,
74
+ olderThanMs: 1,
75
+ tier: "all",
76
+ dryRun: false,
77
+ now: () => new Date(Date.now() + 86_400_000),
78
+ });
79
+
80
+ assert.equal(result.purgedCount, 1);
81
+ assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 0);
82
+ } finally {
83
+ await rm(dir, { recursive: true, force: true });
84
+ }
85
+ });
86
+
87
+ test("purgeMemories invalidates projection rows for already-absent files but not failed deletes, and dry-run never mutates", async () => {
88
+ const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-projection-branches-"));
89
+ try {
90
+ // Real projection with two rows; candidates come from a mock storage so the
91
+ // unlink outcomes are deterministic: one path is already absent (ENOENT),
92
+ // the other is a non-empty directory (unlink fails, not ENOENT).
93
+ const realStorage = new StorageManager(dir);
94
+ const first = await realStorage.writeMemory("fact", "purge branch enoent", { source: "test" });
95
+ const second = await realStorage.writeMemory("fact", "purge branch failure", { source: "test" });
96
+ await rebuildMemoryProjection({ memoryDir: dir, dryRun: false });
97
+ assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 2);
98
+
99
+ const firstMemory = await realStorage.getMemoryById(first.id);
100
+ const secondMemory = await realStorage.getMemoryById(second.id);
101
+ assert.ok(firstMemory && secondMemory);
102
+ await rm(firstMemory.path);
103
+ await rm(secondMemory.path);
104
+ await mkdir(secondMemory.path);
105
+ await writeFile(path.join(secondMemory.path, "block.txt"), "x", "utf8");
106
+
107
+ const mockStorage = {
108
+ dir,
109
+ readAllMemories: async () => [firstMemory, secondMemory],
110
+ readAllColdMemories: async () => [],
111
+ readArchivedMemories: async () => [],
112
+ };
113
+ const future = () => new Date(Date.now() + 86_400_000);
114
+
115
+ // Dry-run: no projection mutation.
116
+ await purgeMemories({ storage: mockStorage as never, olderThanMs: 1, tier: "all", dryRun: true, now: future });
117
+ assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 2);
118
+
119
+ const result = await purgeMemories({ storage: mockStorage as never, olderThanMs: 1, tier: "all", dryRun: false, now: future });
120
+
121
+ assert.equal(result.alreadyAbsentCount, 1);
122
+ assert.equal(result.purgedCount, 0);
123
+ assert.equal(result.errors.filter((e) => e.id === second.id).length, 1);
124
+ const page = readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 });
125
+ assert.ok(page);
126
+ // ENOENT row invalidated; failed-delete row keeps its projection entry.
127
+ assert.equal(page.total, 1);
128
+ } finally {
129
+ await rm(dir, { recursive: true, force: true });
130
+ }
131
+ });
132
+
57
133
  async function fileExists(filePath: string): Promise<boolean> {
58
134
  try {
59
135
  await access(filePath);
@@ -23,6 +23,7 @@ import path from "node:path";
23
23
  import { appendFile, mkdir, unlink } from "node:fs/promises";
24
24
  import type { StorageManager } from "../storage.js";
25
25
  import type { MemoryFile } from "../types.js";
26
+ import { markProjectedMemoryPathInvalid } from "../memory-projection-store.js";
26
27
  import type { SearchBackend } from "../search/port.js";
27
28
 
28
29
  export type PurgeTierFilter = "cold" | "all";
@@ -236,12 +237,14 @@ export async function purgeMemories(
236
237
  await unlink(candidate.path);
237
238
  actuallyPurged.push(candidate);
238
239
  addCollectionForCandidate(candidate);
240
+ markProjectedMemoryPathInvalid(storage.dir, candidate.id);
239
241
  } catch (err) {
240
242
  const message = err instanceof Error ? err.message : String(err);
241
243
  // ENOENT is fine — already gone
242
244
  if (hasErrorCode(err, "ENOENT")) {
243
245
  alreadyAbsent.push(candidate);
244
246
  addCollectionForCandidate(candidate);
247
+ markProjectedMemoryPathInvalid(storage.dir, candidate.id);
245
248
  } else {
246
249
  errors.push({ id: candidate.id, path: candidate.path, error: message });
247
250
  }
@@ -24,6 +24,7 @@ import {
24
24
  import {
25
25
  getMemoryProjectionPath,
26
26
  initializeMemoryProjectionDb,
27
+ isProjectedMemoryPathValid,
27
28
  type ProjectedEntityMentionRow,
28
29
  type MemoryProjectionGovernanceAppliedActionRow,
29
30
  type MemoryProjectionGovernanceReviewQueueRow,
@@ -794,6 +795,7 @@ function writeProjectionDb(
794
795
  usedLifecycleLedger: boolean,
795
796
  ): void {
796
797
  const db = openBetterSqlite3(dbPath);
798
+ const memoryDir = path.dirname(path.dirname(dbPath));
797
799
  try {
798
800
  initializeMemoryProjectionDb(db);
799
801
 
@@ -810,6 +812,7 @@ function writeProjectionDb(
810
812
  status,
811
813
  lifecycle_state,
812
814
  path_rel,
815
+ path_valid,
813
816
  created_at,
814
817
  updated_at,
815
818
  archived_at,
@@ -823,7 +826,7 @@ function writeProjectionDb(
823
826
  last_accessed,
824
827
  tags_json,
825
828
  preview_text
826
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
829
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
827
830
  `);
828
831
 
829
832
  const insertTimeline = db.prepare(`
@@ -921,6 +924,7 @@ function writeProjectionDb(
921
924
  row.status,
922
925
  row.lifecycleState ?? null,
923
926
  row.pathRel,
927
+ isProjectedMemoryPathValid(memoryDir, row.pathRel) ? 1 : 0,
924
928
  row.created,
925
929
  row.updated,
926
930
  row.archivedAt ?? null,
@@ -17,7 +17,7 @@ import {
17
17
  type BetterSqlite3Database,
18
18
  } from "./runtime/better-sqlite.js";
19
19
 
20
- export const MEMORY_PROJECTION_SCHEMA_VERSION = 2;
20
+ export const MEMORY_PROJECTION_SCHEMA_VERSION = 3;
21
21
 
22
22
  export interface ProjectedMemoryBrowseOptions {
23
23
  query?: string;
@@ -153,11 +153,13 @@ function migrateProjectionSchemaIfNeeded(memoryDir: string): void {
153
153
  export function memoryCurrentSelectExpressions(db: BetterSqlite3Database): {
154
154
  tagsJson: string;
155
155
  previewText: string;
156
+ hasPathValid: boolean;
156
157
  } {
157
158
  const columns = listTableColumns(db, "memory_current");
158
159
  return {
159
160
  tagsJson: columns.has("tags_json") ? "tags_json" : `'[]' AS tags_json`,
160
161
  previewText: columns.has("preview_text") ? "preview_text" : `'' AS preview_text`,
162
+ hasPathValid: columns.has("path_valid"),
161
163
  };
162
164
  }
163
165
 
@@ -174,6 +176,7 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
174
176
  status TEXT NOT NULL,
175
177
  lifecycle_state TEXT,
176
178
  path_rel TEXT NOT NULL,
179
+ path_valid INTEGER NOT NULL DEFAULT 0 CHECK (path_valid IN (0, 1)),
177
180
  created_at TEXT NOT NULL,
178
181
  updated_at TEXT NOT NULL,
179
182
  archived_at TEXT,
@@ -195,8 +198,6 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
195
198
  CREATE INDEX IF NOT EXISTS idx_memory_current_category
196
199
  ON memory_current(category);
197
200
 
198
- CREATE INDEX IF NOT EXISTS idx_memory_current_updated
199
- ON memory_current(updated_at DESC);
200
201
 
201
202
  CREATE TABLE IF NOT EXISTS memory_timeline (
202
203
  event_id TEXT PRIMARY KEY,
@@ -297,6 +298,18 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
297
298
  `);
298
299
 
299
300
  migrateMemoryCurrentTable(db);
301
+ if (listTableColumns(db, "memory_current").has("path_valid")) {
302
+ db.exec(`
303
+ CREATE INDEX IF NOT EXISTS idx_memory_current_browse_updated_desc
304
+ ON memory_current(path_valid, updated_at DESC, created_at DESC, memory_id ASC);
305
+ CREATE INDEX IF NOT EXISTS idx_memory_current_browse_updated_asc
306
+ ON memory_current(path_valid, updated_at ASC, created_at ASC, memory_id ASC);
307
+ CREATE INDEX IF NOT EXISTS idx_memory_current_browse_created_desc
308
+ ON memory_current(path_valid, created_at DESC, updated_at DESC, memory_id ASC);
309
+ CREATE INDEX IF NOT EXISTS idx_memory_current_browse_created_asc
310
+ ON memory_current(path_valid, created_at ASC, updated_at ASC, memory_id ASC);
311
+ `);
312
+ }
300
313
  db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)")
301
314
  .run("schemaVersion", String(MEMORY_PROJECTION_SCHEMA_VERSION));
302
315
  }
@@ -309,6 +322,42 @@ function openProjectionReadonly(memoryDir: string): BetterSqlite3Database | null
309
322
  return null;
310
323
  }
311
324
  }
325
+ function updateProjectionBestEffort(
326
+ memoryDir: string,
327
+ sql: string,
328
+ params: unknown[],
329
+ ): void {
330
+ let db: BetterSqlite3Database | null = null;
331
+ try {
332
+ db = openBetterSqlite3(getMemoryProjectionPath(memoryDir), { fileMustExist: true });
333
+ db.prepare(sql).run(...params);
334
+ } catch {
335
+ // Projection updates must never block the canonical filesystem mutation.
336
+ } finally {
337
+ db?.close();
338
+ }
339
+ }
340
+
341
+ export function markProjectedMemoryPathInvalid(memoryDir: string, memoryId: string): void {
342
+ updateProjectionBestEffort(
343
+ memoryDir,
344
+ "UPDATE memory_current SET path_valid = 0 WHERE memory_id = ?",
345
+ [memoryId],
346
+ );
347
+ }
348
+
349
+ export function updateProjectedMemoryPath(
350
+ memoryDir: string,
351
+ memoryId: string,
352
+ pathRel: string,
353
+ ): void {
354
+ const pathValid = isProjectedMemoryPathValid(memoryDir, pathRel) ? 1 : 0;
355
+ updateProjectionBestEffort(
356
+ memoryDir,
357
+ "UPDATE memory_current SET path_rel = ?, path_valid = ? WHERE memory_id = ?",
358
+ [pathRel, pathValid, memoryId],
359
+ );
360
+ }
312
361
 
313
362
  function withProjectionReadonly<T>(
314
363
  memoryDir: string,
@@ -363,7 +412,11 @@ function isPathInsideRoot(candidatePath: string, rootPath: string): boolean {
363
412
  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
364
413
  }
365
414
 
366
- function resolveProjectedMemoryPath(memoryDir: string, pathRel: string): string | null {
415
+ function resolveProjectedMemoryPath(
416
+ memoryDir: string,
417
+ pathRel: string,
418
+ requireExisting = false,
419
+ ): string | null {
367
420
  if (path.isAbsolute(pathRel)) return null;
368
421
  const root = resolveMemoryDirRoot(memoryDir);
369
422
  const candidate = path.resolve(root, pathRel);
@@ -372,14 +425,21 @@ function resolveProjectedMemoryPath(memoryDir: string, pathRel: string): string
372
425
  const realCandidate = fs.realpathSync(candidate);
373
426
  if (!isPathInsideRoot(realCandidate, root)) return null;
374
427
  } catch {
428
+ if (requireExisting) return null;
375
429
  // Missing files are handled by callers. Still keep the lexical guard above.
376
430
  }
377
431
  return candidate;
378
432
  }
379
433
 
434
+ export function isProjectedMemoryPathValid(memoryDir: string, pathRel: string): boolean {
435
+ return resolveProjectedMemoryPath(memoryDir, pathRel) !== null;
436
+ }
437
+
380
438
  function projectedBrowseRowFromCurrentRow(
381
439
  memoryDir: string,
382
440
  row: Record<string, unknown>,
441
+ resolvedFilePath?: string,
442
+ requireExisting = false,
383
443
  ): ProjectedMemoryBrowseRow | null {
384
444
  if (
385
445
  typeof row.memory_id !== "string" ||
@@ -389,7 +449,7 @@ function projectedBrowseRowFromCurrentRow(
389
449
  ) {
390
450
  return null;
391
451
  }
392
- const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
452
+ const filePath = resolvedFilePath ?? resolveProjectedMemoryPath(memoryDir, row.path_rel, requireExisting);
393
453
  if (!filePath) return null;
394
454
  return {
395
455
  id: row.memory_id,
@@ -613,8 +673,15 @@ export function readProjectedMemoryBrowse(
613
673
  const whereSql = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
614
674
 
615
675
  if (normalizedQuery) {
616
- // Query-based browse: fetch all matching rows, filter by full file content, then paginate in JS
617
- const allRows = db
676
+ // Full-content search still examines every eligible row so totals and
677
+ // matches stay exact, but iteration keeps memory use proportional to the page.
678
+ const queryWhereClauses = currentSelect.hasPathValid
679
+ ? [...whereClauses, ...projectedBrowsePathSqlClauses(), "path_valid = 1"]
680
+ : whereClauses;
681
+ const queryWhereSql = queryWhereClauses.length > 0
682
+ ? `WHERE ${queryWhereClauses.join(" AND ")}`
683
+ : "";
684
+ const rows = db
618
685
  .prepare(`
619
686
  SELECT
620
687
  memory_id,
@@ -627,47 +694,88 @@ export function readProjectedMemoryBrowse(
627
694
  ${currentSelect.tagsJson},
628
695
  ${currentSelect.previewText}
629
696
  FROM memory_current
630
- ${whereSql}
697
+ ${queryWhereSql}
631
698
  ORDER BY ${orderBySql}
632
699
  `)
633
- .all(...params) as Array<Record<string, unknown>>;
700
+ .iterate(...params) as Iterable<Record<string, unknown>>;
701
+ const pageRows: ProjectedMemoryBrowseRow[] = [];
702
+ let total = 0;
634
703
 
635
- const filtered = allRows.filter((row) => {
636
- if (typeof row.memory_id !== "string" || typeof row.path_rel !== "string") return false;
637
- const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
638
- if (!filePath) return false;
639
- // Check preview, category, entity_ref, tags first (fast)
704
+ for (const row of rows) {
705
+ if (typeof row.memory_id !== "string" || typeof row.path_rel !== "string") continue;
706
+ const filePath = resolveProjectedMemoryPath(
707
+ memoryDir,
708
+ row.path_rel,
709
+ currentSelect.hasPathValid,
710
+ );
711
+ if (!filePath) continue;
640
712
  const preview = typeof row.preview_text === "string" ? row.preview_text.toLowerCase() : "";
641
713
  const category = typeof row.category === "string" ? row.category.toLowerCase() : "";
642
714
  const entityRef = typeof row.entity_ref === "string" ? row.entity_ref.toLowerCase() : "";
643
715
  const tags = typeof row.tags_json === "string" ? row.tags_json.toLowerCase() : "";
644
- if (preview.includes(normalizedQuery) || category.includes(normalizedQuery) ||
645
- entityRef.includes(normalizedQuery) || tags.includes(normalizedQuery)) {
646
- return true;
716
+ let matches = preview.includes(normalizedQuery) || category.includes(normalizedQuery) ||
717
+ entityRef.includes(normalizedQuery) || tags.includes(normalizedQuery);
718
+ if (!matches) {
719
+ try {
720
+ matches = fs.readFileSync(filePath, "utf-8").toLowerCase().includes(normalizedQuery);
721
+ } catch {
722
+ matches = false;
723
+ }
647
724
  }
648
- // Fall back to reading full file content from disk
649
- try {
650
- const content = fs.readFileSync(filePath, "utf-8").toLowerCase();
651
- return content.includes(normalizedQuery);
652
- } catch {
653
- return false;
725
+ if (!matches) continue;
726
+ if (total >= options.offset && pageRows.length < options.limit) {
727
+ const browseRow = projectedBrowseRowFromCurrentRow(memoryDir, row, filePath);
728
+ if (browseRow) pageRows.push(browseRow);
654
729
  }
655
- });
656
-
657
- const pageRows = filtered.slice(options.offset, options.offset + options.limit);
658
- return {
659
- total: filtered.length,
660
- memories: pageRows
661
- .map((row) => projectedBrowseRowFromCurrentRow(memoryDir, row))
662
- .filter((row): row is ProjectedMemoryBrowseRow => row !== null),
663
- };
730
+ total += 1;
731
+ }
732
+
733
+ return { total, memories: pageRows };
664
734
  }
665
735
 
666
- // No query: push lexical path safety into SQL, then count through the same
667
- // realpath-aware row parser used for returned rows so symlink escapes do not
668
- // inflate totals.
669
736
  const browseWhereClauses = [...whereClauses, ...projectedBrowsePathSqlClauses()];
737
+ if (currentSelect.hasPathValid) {
738
+ browseWhereClauses.push("path_valid = 1");
739
+ }
670
740
  const browseWhereSql = `WHERE ${browseWhereClauses.join(" AND ")}`;
741
+
742
+ if (currentSelect.hasPathValid) {
743
+ const rows = db
744
+ .prepare(`
745
+ SELECT
746
+ memory_id,
747
+ path_rel,
748
+ category,
749
+ status,
750
+ created_at,
751
+ updated_at,
752
+ entity_ref,
753
+ ${currentSelect.tagsJson},
754
+ ${currentSelect.previewText}
755
+ FROM memory_current
756
+ ${browseWhereSql}
757
+ ORDER BY ${orderBySql}
758
+ `)
759
+ .iterate(...params) as Iterable<Record<string, unknown>>;
760
+ const pageRows: ProjectedMemoryBrowseRow[] = [];
761
+ let validRowsSeen = 0;
762
+ for (const row of rows) {
763
+ const browseRow = projectedBrowseRowFromCurrentRow(memoryDir, row, undefined, true);
764
+ if (!browseRow) continue;
765
+ if (validRowsSeen >= options.offset) {
766
+ pageRows.push(browseRow);
767
+ if (pageRows.length >= options.limit) break;
768
+ }
769
+ validRowsSeen += 1;
770
+ }
771
+ const totalRow = db
772
+ .prepare(`SELECT COUNT(*) AS total FROM memory_current ${browseWhereSql}`)
773
+ .get(...params) as { total: number };
774
+ return { total: totalRow.total, memories: pageRows };
775
+ }
776
+
777
+ // Legacy projections have no materialized path validity. Preserve the
778
+ // realpath-aware full scan rather than trusting rows from an older writer.
671
779
  const pageRows: ProjectedMemoryBrowseRow[] = [];
672
780
  const fetchSize = Math.max(options.limit * 2, 50);
673
781
  let validRowsSeen = 0;
package/src/storage.ts CHANGED
@@ -103,11 +103,11 @@ import type {
103
103
  import { confidenceTier, SPECULATIVE_TTL_DAYS } from "./types.js";
104
104
  import {
105
105
  type ProjectedMemoryBrowseOptions,
106
- type ProjectedMemoryBrowsePage,
106
+ type ProjectedMemoryBrowsePage, markProjectedMemoryPathInvalid,
107
107
  readProjectedMemoryState,
108
108
  readProjectedMemoryBrowse,
109
109
  readProjectedGovernanceRecord,
110
- readProjectedMemoryTimeline,
110
+ readProjectedMemoryTimeline, updateProjectedMemoryPath,
111
111
  } from "./memory-projection-store.js";
112
112
  import {
113
113
  inferMemoryStatus,
@@ -4566,14 +4566,12 @@ export class StorageManager {
4566
4566
  const dir = categoryDirName(memory.frontmatter.category);
4567
4567
  return path.join(root, dir, this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4568
4568
  }
4569
-
4570
4569
  private async writeMemoryFileAtomic(targetPath: string, memory: MemoryFile): Promise<void> {
4571
4570
  const fileContent = `${serializeFrontmatter(memory.frontmatter)}\n\n${memory.content}\n`;
4572
4571
  await writeMaybeEncryptedFile(targetPath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
4573
4572
  this.invalidateAllMemoriesCache();
4574
4573
  this.notifyCatalogWrite();
4575
4574
  }
4576
-
4577
4575
  async moveMemoryToPath(memory: MemoryFile, targetPath: string): Promise<void> {
4578
4576
  await this.writeMemoryFileAtomic(targetPath, memory);
4579
4577
  const sourcePath = path.resolve(memory.path);
@@ -4591,6 +4589,7 @@ export class StorageManager {
4591
4589
  // invalidated, but a concurrent readAllMemories() may have re-populated
4592
4590
  // the cache between the write and the unlink.
4593
4591
  this.invalidateAllMemoriesCache();
4592
+ updateProjectedMemoryPath(this.baseDir, memory.frontmatter.id, toMemoryPathRel(this.baseDir, targetPath));
4594
4593
  }
4595
4594
  }
4596
4595
 
@@ -4615,6 +4614,7 @@ export class StorageManager {
4615
4614
  throw err;
4616
4615
  }
4617
4616
  }
4617
+ updateProjectedMemoryPath(this.baseDir, memory.frontmatter.id, toMemoryPathRel(this.baseDir, targetPath));
4618
4618
  this.bumpMemoryStatusVersion();
4619
4619
  return { changed: false, targetPath };
4620
4620
  }
@@ -4662,7 +4662,7 @@ export class StorageManager {
4662
4662
 
4663
4663
  // Write to archive location first (encrypted if applicable), then remove original.
4664
4664
  await this.writeStorageSecureFile(destPath, fileContent);
4665
- await unlink(memory.path);
4665
+ await unlink(memory.path); markProjectedMemoryPathInvalid(this.baseDir, memory.frontmatter.id);
4666
4666
  this.invalidateAllMemoriesCache();
4667
4667
  await this.appendGeneratedMemoryLifecycleEventFailOpen(
4668
4668
  "storage.archiveMemory",
@@ -4767,7 +4767,7 @@ export class StorageManager {
4767
4767
  if (!memory) return false;
4768
4768
 
4769
4769
  try {
4770
- await unlink(memory.path);
4770
+ await unlink(memory.path); markProjectedMemoryPathInvalid(this.baseDir, id);
4771
4771
  this.invalidateAllMemoriesCache();
4772
4772
  this.bumpMemoryStatusVersion();
4773
4773
  log.debug(`invalidated memory ${id}`);
@@ -4906,7 +4906,7 @@ export class StorageManager {
4906
4906
  const expiresAt = new Date(m.frontmatter.expiresAt).getTime();
4907
4907
  if (expiresAt < now) {
4908
4908
  try {
4909
- await unlink(m.path);
4909
+ await unlink(m.path); markProjectedMemoryPathInvalid(this.baseDir, m.frontmatter.id);
4910
4910
  deleted.push(m);
4911
4911
  log.debug(`cleaned expired memory ${m.frontmatter.id} (TTL expired)`);
4912
4912
  } catch {
@@ -5981,7 +5981,7 @@ export class StorageManager {
5981
5981
  if (updatedAt < cutoff) {
5982
5982
  // Remove the file
5983
5983
  try {
5984
- await unlink(m.path);
5984
+ await unlink(m.path); markProjectedMemoryPathInvalid(this.baseDir, m.frontmatter.id);
5985
5985
  deleted.push(m);
5986
5986
  log.debug(`cleaned expired commitment ${m.frontmatter.id}`);
5987
5987
  } catch {