@remnic/core 9.3.663 → 9.3.665

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/dist/access-cli.js +25 -23
  2. package/dist/access-cli.js.map +1 -1
  3. package/dist/access-http.js +20 -18
  4. package/dist/access-mcp.js +19 -17
  5. package/dist/access-schema.d.ts +36 -36
  6. package/dist/access-schema.js +4 -3
  7. package/dist/access-service.js +17 -15
  8. package/dist/briefing.js +5 -4
  9. package/dist/{capsule-merge-T2JRE46P.js → capsule-merge-GK5E647P.js} +3 -2
  10. package/dist/{capsule-merge-T2JRE46P.js.map → capsule-merge-GK5E647P.js.map} +1 -1
  11. package/dist/causal-consolidation.js +6 -5
  12. package/dist/causal-consolidation.js.map +1 -1
  13. package/dist/{chunk-2KDQI363.js → chunk-2HEZXPYU.js} +4 -4
  14. package/dist/{chunk-HSCJYHYV.js → chunk-2OPARZ4B.js} +49 -19
  15. package/dist/chunk-2OPARZ4B.js.map +1 -0
  16. package/dist/chunk-5GPPACXK.js +16 -0
  17. package/dist/chunk-5GPPACXK.js.map +1 -0
  18. package/dist/{chunk-F6O7IOS3.js → chunk-6JBKHTQD.js} +2 -2
  19. package/dist/{chunk-YYQRVNSV.js → chunk-7C4MPEPE.js} +6 -6
  20. package/dist/{chunk-AL4RAJL5.js → chunk-7XH7VJN4.js} +6 -4
  21. package/dist/chunk-7XH7VJN4.js.map +1 -0
  22. package/dist/{chunk-Q4CAQGKQ.js → chunk-AER6MT24.js} +12 -21
  23. package/dist/chunk-AER6MT24.js.map +1 -0
  24. package/dist/{chunk-DHGSZ3UD.js → chunk-ARV3AUOM.js} +2 -2
  25. package/dist/{chunk-PXVFMQLD.js → chunk-BZG2CWOQ.js} +3 -3
  26. package/dist/{chunk-ANJOULTP.js → chunk-C7AF236A.js} +2 -2
  27. package/dist/{chunk-TBLGI2LT.js → chunk-D7IXTY5E.js} +31 -4
  28. package/dist/chunk-D7IXTY5E.js.map +1 -0
  29. package/dist/{chunk-FZC2WSDB.js → chunk-DOCTITOP.js} +2 -2
  30. package/dist/{chunk-WOQIHC67.js → chunk-DQY7NJ5L.js} +2 -2
  31. package/dist/{chunk-NMPEJV5M.js → chunk-DSLUOQDY.js} +2 -2
  32. package/dist/{chunk-A7EF2XRO.js → chunk-EXXBA5OM.js} +30 -8
  33. package/dist/chunk-EXXBA5OM.js.map +1 -0
  34. package/dist/{chunk-QXHBWFR3.js → chunk-IHG6CC7T.js} +2 -2
  35. package/dist/{chunk-4KDLCMLK.js → chunk-IROWLAWG.js} +5 -5
  36. package/dist/{chunk-ILXTATKK.js → chunk-J2HSAU72.js} +5 -5
  37. package/dist/chunk-J2HSAU72.js.map +1 -0
  38. package/dist/{chunk-DFAXGZKI.js → chunk-JIX3ZL2J.js} +8 -8
  39. package/dist/{chunk-GY3V3SUI.js → chunk-KHGE6PMF.js} +2 -2
  40. package/dist/{chunk-TWAJICBN.js → chunk-OHJFJ4HI.js} +2 -2
  41. package/dist/{chunk-WSQG37DV.js → chunk-OUWAQVDJ.js} +2 -2
  42. package/dist/{chunk-ZLDUQWT2.js → chunk-PWWWLD7D.js} +2 -2
  43. package/dist/{chunk-ZJH723NM.js → chunk-Q5ZU3RNY.js} +2 -2
  44. package/dist/{chunk-35HP3TGR.js → chunk-ROHLEUTH.js} +4 -4
  45. package/dist/{chunk-5RIRL3XL.js → chunk-RS25QOKZ.js} +2 -2
  46. package/dist/{chunk-RQGR3ETH.js → chunk-T2AN3BSP.js} +2 -2
  47. package/dist/{chunk-UAU5U5ML.js → chunk-UDJLF3BO.js} +2 -2
  48. package/dist/{chunk-ALEPI75L.js → chunk-VF4XKTX3.js} +6 -4
  49. package/dist/{chunk-ALEPI75L.js.map → chunk-VF4XKTX3.js.map} +1 -1
  50. package/dist/{chunk-AX5O25EF.js → chunk-VH6EIKVS.js} +152 -190
  51. package/dist/chunk-VH6EIKVS.js.map +1 -0
  52. package/dist/chunk-VS2IYZRU.js +43 -0
  53. package/dist/chunk-VS2IYZRU.js.map +1 -0
  54. package/dist/{chunk-TGOOJCGA.js → chunk-WH4SKYPX.js} +76 -54
  55. package/dist/chunk-WH4SKYPX.js.map +1 -0
  56. package/dist/{chunk-5AYAZN45.js → chunk-XRSIGVTS.js} +5 -5
  57. package/dist/{chunk-D2EFNQMY.js → chunk-XW3W4PV4.js} +2 -2
  58. package/dist/{chunk-TYIXG4VR.js → chunk-YW52BQSU.js} +2 -2
  59. package/dist/{cli-C6twwe84.d.ts → cli-BQRqR9N-.d.ts} +12 -1
  60. package/dist/cli.d.ts +1 -1
  61. package/dist/cli.js +32 -28
  62. package/dist/compounding/engine.js +5 -4
  63. package/dist/connectors/codex-materialize-runner.js +5 -4
  64. package/dist/connectors/index.js +5 -4
  65. package/dist/consolidation-provenance-check.js +3 -2
  66. package/dist/consolidation-undo.js +2 -1
  67. package/dist/consolidation-undo.js.map +1 -1
  68. package/dist/entity-retrieval.js +5 -4
  69. package/dist/index.d.ts +1 -1
  70. package/dist/index.js +39 -36
  71. package/dist/index.js.map +1 -1
  72. package/dist/maintenance/memory-governance.js +6 -4
  73. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +5 -4
  74. package/dist/maintenance/rebuild-memory-projection.js +7 -5
  75. package/dist/namespaces/migrate.js +13 -11
  76. package/dist/namespaces/search.js +8 -6
  77. package/dist/namespaces/storage.d.ts +13 -0
  78. package/dist/namespaces/storage.js +5 -4
  79. package/dist/offline-sync.js +3 -2
  80. package/dist/operator-toolkit.js +16 -14
  81. package/dist/orchestrator.js +21 -19
  82. package/dist/page-versioning.js +2 -1
  83. package/dist/schemas.d.ts +64 -64
  84. package/dist/search/document-scanner.d.ts +11 -7
  85. package/dist/search/document-scanner.js +3 -1
  86. package/dist/search/factory.js +7 -5
  87. package/dist/search/index.js +7 -5
  88. package/dist/search/lancedb-backend.js +4 -2
  89. package/dist/search/meilisearch-backend.js +4 -2
  90. package/dist/search/orama-backend.js +4 -2
  91. package/dist/secure-store/index.js +3 -2
  92. package/dist/semantic-consolidation.js +6 -5
  93. package/dist/semantic-rule-promotion.js +5 -4
  94. package/dist/semantic-rule-verifier.js +5 -4
  95. package/dist/shared-context/manager.d.ts +2 -2
  96. package/dist/storage.d.ts +17 -3
  97. package/dist/storage.js +4 -3
  98. package/dist/transfer/capsule-import.js +3 -2
  99. package/dist/transfer/types.d.ts +12 -12
  100. package/dist/verified-recall.js +5 -4
  101. package/package.json +1 -1
  102. package/src/cli.ts +62 -23
  103. package/src/consolidation-provenance-check.ts +7 -6
  104. package/src/maintenance/memory-governance.ts +47 -7
  105. package/src/namespaces/catalog.test.ts +12 -12
  106. package/src/namespaces/storage.ts +28 -1
  107. package/src/orchestrator.ts +84 -58
  108. package/src/page-versioning.ts +7 -4
  109. package/src/search/document-scanner.test.ts +29 -0
  110. package/src/search/document-scanner.ts +17 -29
  111. package/src/secure-store/secure-fs.ts +19 -5
  112. package/src/secure-store/secure-store.test.ts +28 -0
  113. package/src/storage.ts +42 -43
  114. package/src/training-export/converter.test.ts +19 -0
  115. package/src/training-export/converter.ts +8 -5
  116. package/src/utils/category-dir.ts +10 -4
  117. package/src/utils/path-containment.ts +40 -0
  118. package/dist/chunk-A7EF2XRO.js.map +0 -1
  119. package/dist/chunk-AL4RAJL5.js.map +0 -1
  120. package/dist/chunk-AX5O25EF.js.map +0 -1
  121. package/dist/chunk-HSCJYHYV.js.map +0 -1
  122. package/dist/chunk-ILXTATKK.js.map +0 -1
  123. package/dist/chunk-Q4CAQGKQ.js.map +0 -1
  124. package/dist/chunk-TBLGI2LT.js.map +0 -1
  125. package/dist/chunk-TGOOJCGA.js.map +0 -1
  126. /package/dist/{chunk-2KDQI363.js.map → chunk-2HEZXPYU.js.map} +0 -0
  127. /package/dist/{chunk-F6O7IOS3.js.map → chunk-6JBKHTQD.js.map} +0 -0
  128. /package/dist/{chunk-YYQRVNSV.js.map → chunk-7C4MPEPE.js.map} +0 -0
  129. /package/dist/{chunk-DHGSZ3UD.js.map → chunk-ARV3AUOM.js.map} +0 -0
  130. /package/dist/{chunk-PXVFMQLD.js.map → chunk-BZG2CWOQ.js.map} +0 -0
  131. /package/dist/{chunk-ANJOULTP.js.map → chunk-C7AF236A.js.map} +0 -0
  132. /package/dist/{chunk-FZC2WSDB.js.map → chunk-DOCTITOP.js.map} +0 -0
  133. /package/dist/{chunk-WOQIHC67.js.map → chunk-DQY7NJ5L.js.map} +0 -0
  134. /package/dist/{chunk-NMPEJV5M.js.map → chunk-DSLUOQDY.js.map} +0 -0
  135. /package/dist/{chunk-QXHBWFR3.js.map → chunk-IHG6CC7T.js.map} +0 -0
  136. /package/dist/{chunk-4KDLCMLK.js.map → chunk-IROWLAWG.js.map} +0 -0
  137. /package/dist/{chunk-DFAXGZKI.js.map → chunk-JIX3ZL2J.js.map} +0 -0
  138. /package/dist/{chunk-GY3V3SUI.js.map → chunk-KHGE6PMF.js.map} +0 -0
  139. /package/dist/{chunk-TWAJICBN.js.map → chunk-OHJFJ4HI.js.map} +0 -0
  140. /package/dist/{chunk-WSQG37DV.js.map → chunk-OUWAQVDJ.js.map} +0 -0
  141. /package/dist/{chunk-ZLDUQWT2.js.map → chunk-PWWWLD7D.js.map} +0 -0
  142. /package/dist/{chunk-ZJH723NM.js.map → chunk-Q5ZU3RNY.js.map} +0 -0
  143. /package/dist/{chunk-35HP3TGR.js.map → chunk-ROHLEUTH.js.map} +0 -0
  144. /package/dist/{chunk-5RIRL3XL.js.map → chunk-RS25QOKZ.js.map} +0 -0
  145. /package/dist/{chunk-RQGR3ETH.js.map → chunk-T2AN3BSP.js.map} +0 -0
  146. /package/dist/{chunk-UAU5U5ML.js.map → chunk-UDJLF3BO.js.map} +0 -0
  147. /package/dist/{chunk-5AYAZN45.js.map → chunk-XRSIGVTS.js.map} +0 -0
  148. /package/dist/{chunk-D2EFNQMY.js.map → chunk-XW3W4PV4.js.map} +0 -0
  149. /package/dist/{chunk-TYIXG4VR.js.map → chunk-YW52BQSU.js.map} +0 -0
@@ -487,13 +487,13 @@ test("StorageRouter integration: catalog registers namespace on storageFor", asy
487
487
  const config = makeConfig(memoryDir);
488
488
  const catalog = new NamespaceCatalog(config);
489
489
  const router = new NamespaceStorageRouter(config, {
490
- onResolve: (namespace, storageDir) => {
491
- void catalog.registerResolved(namespace, storageDir);
492
- },
490
+ // Return the registration promise so the router tracks it as in-flight and
491
+ // `whenResolveHooksSettled()` can await it deterministically (no timer race).
492
+ onResolve: (namespace, storageDir) => catalog.registerResolved(namespace, storageDir),
493
493
  });
494
494
  await router.storageFor("project-origin-abc123");
495
- // allow the fire-and-forget registration to settle
496
- await new Promise((r) => setTimeout(r, 10));
495
+ // Deterministically await the fire-and-forget registration instead of sleeping.
496
+ await router.whenResolveHooksSettled();
497
497
 
498
498
  const record = await catalog.getNamespaceRecord("project-origin-abc123");
499
499
  assert.ok(record, "storageFor should have registered the namespace");
@@ -2523,8 +2523,8 @@ test("an async onResolve hook rejection does not crash storage resolution", asyn
2523
2523
  // Must not throw or produce an unhandled rejection that fails the test.
2524
2524
  const sm = await router.storageFor("default");
2525
2525
  assert.ok(sm, "storage resolution succeeds despite a rejecting async hook");
2526
- // Give the swallowed rejection a tick to settle.
2527
- await new Promise((r) => setTimeout(r, 10));
2526
+ // Deterministically await the swallowed rejection instead of sleeping.
2527
+ await router.whenResolveHooksSettled();
2528
2528
  assert.ok(called >= 1, "the async hook was invoked");
2529
2529
  } finally {
2530
2530
  await rm(memoryDir, { recursive: true, force: true });
@@ -2564,7 +2564,7 @@ test("concurrent storageFor() for one namespace fires the resolve hook ONCE whil
2564
2564
  // Let the in-flight registration settle, then a steady-state cache hit must
2565
2565
  // still be a catalog no-op (now deduped via notifiedResolved).
2566
2566
  release();
2567
- await new Promise((r) => setTimeout(r, 10));
2567
+ await router.whenResolveHooksSettled();
2568
2568
  await router.storageFor("project-origin-inflight");
2569
2569
  assert.equal(calls, 1, "a steady-state cache hit after settle must not re-fire the hook");
2570
2570
  } finally {
@@ -2589,20 +2589,20 @@ test("a dropped resolve registration (hook returns false) is retried on a later
2589
2589
  });
2590
2590
 
2591
2591
  await router.storageFor("project-origin-retry");
2592
- // Give the async hook a tick to settle and clear the in-flight marker.
2593
- await new Promise((r) => setTimeout(r, 10));
2592
+ // Deterministically await the async hook so the in-flight marker is cleared.
2593
+ await router.whenResolveHooksSettled();
2594
2594
  assert.equal(calls, 1, "the hook fired once for the dropped registration");
2595
2595
 
2596
2596
  // Now the registration will succeed; a later resolve must RETRY (not be
2597
2597
  // suppressed by a stale in-flight/notified marker from the dropped attempt).
2598
2598
  result = undefined; // success (legacy void)
2599
2599
  await router.storageFor("project-origin-retry");
2600
- await new Promise((r) => setTimeout(r, 10));
2600
+ await router.whenResolveHooksSettled();
2601
2601
  assert.equal(calls, 2, "a dropped registration must be retried on the next storageFor()");
2602
2602
 
2603
2603
  // After a SUCCESSFUL registration, further cache hits are deduped (no retry).
2604
2604
  await router.storageFor("project-origin-retry");
2605
- await new Promise((r) => setTimeout(r, 10));
2605
+ await router.whenResolveHooksSettled();
2606
2606
  assert.equal(calls, 2, "a successful registration is not re-fired on subsequent cache hits");
2607
2607
  } finally {
2608
2608
  await rm(memoryDir, { recursive: true, force: true });
@@ -223,6 +223,11 @@ export class NamespaceStorageRouter {
223
223
  // entry is always removed when the promise settles, so the map cannot grow
224
224
  // unbounded (one transient entry per concurrently-resolving namespace).
225
225
  private readonly inFlightResolved = new Map<string, string>();
226
+ // Tracks every in-flight resolve-hook promise so callers can deterministically
227
+ // await the fire-and-forget registrations that `storageFor()` kicks off (see
228
+ // `whenResolveHooksSettled`). Entries are removed as each hook settles, so the
229
+ // set holds at most one promise per concurrently-resolving namespace.
230
+ private readonly pendingResolveHooks = new Set<Promise<unknown>>();
226
231
 
227
232
  // Normalized (trimmed) default namespace identity (NH-FH). `storageFor`
228
233
  // normalizes its input, so default-namespace branches must compare against the
@@ -325,7 +330,10 @@ export class NamespaceStorageRouter {
325
330
  // the map holds at most one transient entry per concurrently-resolving
326
331
  // namespace and cannot grow unbounded.
327
332
  this.inFlightResolved.set(namespace, storageDir);
328
- Promise.resolve(hook(namespace, storageDir)).then(
333
+ const hookResult = Promise.resolve(hook(namespace, storageDir));
334
+ // Track the in-flight promise so `whenResolveHooksSettled()` can await it.
335
+ this.pendingResolveHooks.add(hookResult);
336
+ hookResult.then(
329
337
  (persisted) => {
330
338
  // Clear the in-flight marker ONLY if it is still ours (a newer resolve
331
339
  // for a different dir may have replaced it).
@@ -338,6 +346,7 @@ export class NamespaceStorageRouter {
338
346
  // On `false` (dropped touch) we intentionally do NOT mark notified, so
339
347
  // a later `storageFor()` retries the registration. Clearing the
340
348
  // in-flight marker above is what re-enables that retry.
349
+ this.pendingResolveHooks.delete(hookResult);
341
350
  },
342
351
  () => {
343
352
  // Registration failed — clear in-flight AND do NOT mark as notified, so
@@ -348,6 +357,7 @@ export class NamespaceStorageRouter {
348
357
  if (this.notifiedResolved.get(namespace) === storageDir) {
349
358
  this.notifiedResolved.delete(namespace);
350
359
  }
360
+ this.pendingResolveHooks.delete(hookResult);
351
361
  },
352
362
  );
353
363
  } catch {
@@ -358,4 +368,21 @@ export class NamespaceStorageRouter {
358
368
  }
359
369
  }
360
370
  }
371
+
372
+ /**
373
+ * Resolve once every in-flight `onResolve` registration has settled.
374
+ *
375
+ * `storageFor()` fires the resolve hook fire-and-forget, so its catalog side
376
+ * effect (e.g. `registerResolved(...)`) is not observable the moment
377
+ * `storageFor()` returns. Callers that must act on that side effect — notably
378
+ * tests asserting the catalog was updated — should await this instead of
379
+ * racing a timer. Resolves immediately when no hook is registered or nothing
380
+ * is in flight. The loop re-checks because a settling hook could, in
381
+ * principle, trigger a follow-on resolution.
382
+ */
383
+ async whenResolveHooksSettled(): Promise<void> {
384
+ while (this.pendingResolveHooks.size > 0) {
385
+ await Promise.allSettled([...this.pendingResolveHooks]);
386
+ }
387
+ }
361
388
  }
@@ -8,9 +8,11 @@ import os from "node:os";
8
8
  import { createHash, randomBytes } from "node:crypto";
9
9
  import { existsSync, readFileSync } from "node:fs";
10
10
  import {
11
+ lstat,
11
12
  mkdir,
12
13
  readdir,
13
14
  readFile,
15
+ realpath,
14
16
  stat,
15
17
  unlink,
16
18
  writeFile,
@@ -343,6 +345,8 @@ import {
343
345
  defaultTierMigrationCycleBudget,
344
346
  } from "./compounding/engine.js";
345
347
  import { parseFlexibleIsoTimestamp } from "./utils/iso-timestamp.js";
348
+ import { categoryDirName, RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
349
+ import { assertPathInsideRoot } from "./utils/path-containment.js";
346
350
  // IRC preference consolidation — used by eval adapter directly;
347
351
  // orchestrator integration planned for future PR.
348
352
  // import { consolidatePreferences, buildQueryAwarePreferenceSection, synthesizePreferencesFromLcm } from "./compounding/preference-consolidator.js";
@@ -1758,16 +1762,13 @@ export function resolvePersistedMemoryRelativePath(options: {
1758
1762
  }
1759
1763
  // Pick the subtree that matches the StorageManager.writeMemory routing
1760
1764
  // so fallback paths (used before memoryPathById has seen the fresh
1761
- // write) agree with where the file actually lives. Without this branch,
1762
- // reasoning_trace graph edges point at facts/<date>/, and subsequent
1763
- // graph expansion silently drops those nodes when readMemoryByPath
1764
- // cannot resolve them (issue #564 PR 3 review).
1765
- const subtree =
1766
- options.category === "procedure"
1767
- ? "procedures"
1768
- : options.category === "reasoning_trace"
1769
- ? "reasoning-traces"
1770
- : "facts";
1765
+ // write) agree with where the file actually lives. Routing goes through
1766
+ // the shared categoryDirName() chokepoint (utils/category-dir.ts) so
1767
+ // every category decisions/, preferences/, reasoning-traces/, ...
1768
+ // resolves to the same dir the writer used; otherwise graph edges point
1769
+ // at the wrong subtree and graph expansion silently drops those nodes
1770
+ // when readMemoryByPath cannot resolve them (issue #564 PR 3 / #1546).
1771
+ const subtree = categoryDirName(options.category);
1771
1772
  const idParts = options.memoryId.split("-");
1772
1773
  const maybeTimestamp = Number(idParts[1]);
1773
1774
  if (Number.isFinite(maybeTimestamp) && maybeTimestamp > 0) {
@@ -4787,60 +4788,85 @@ export class Orchestrator {
4787
4788
  // a local calendar day. Scan the UTC-date envelope that overlaps the local
4788
4789
  // day, then filter parseable fact timestamps to that configured local day.
4789
4790
  const datesToScan = utcDateKeysForLocalDay(now, timeZone);
4790
- const factsBaseDir = path.join(storage.dir, "facts");
4791
4791
  const MAX_CHARS = 100_000;
4792
4792
 
4793
- // --- Read fact files from each date directory ---
4793
+ // --- Read memory files from each category dir × date directory ---
4794
+ // Iterate every recall category dir (RECALL_FALLBACK_DIRS — single source
4795
+ // of truth) so the day summary includes decisions/, moments/, ... not just
4796
+ // facts/ (#1546). corrections/ is flat, so corrections/<date>/ never exists
4797
+ // and is skipped by the ENOENT guard — preserving the prior exclusion. The
4798
+ // per-file created→local-day filter below is unchanged.
4799
+ //
4800
+ // Symlink/containment hardening (mirrors scanDir / the CLI walker): the
4801
+ // gathered contents feed the day-summary LLM input, so a symlinked category
4802
+ // dir (decisions/ → outside memoryDir) must not be followed and leak files.
4803
+ // Resolve the store root once; skip symlinked / out-of-root dirs and
4804
+ // entries; skip the scan gracefully if the root can't be resolved.
4794
4805
  const facts: MemoryFile[] = [];
4795
- for (const date of datesToScan) {
4796
- const factsDir = path.join(factsBaseDir, date);
4797
- try {
4798
- const entries = await readdir(factsDir, { withFileTypes: true });
4799
- for (const entry of entries) {
4800
- if (!entry.name.endsWith(".md")) continue;
4801
- const fullPath = path.join(factsDir, entry.name);
4802
- try {
4803
- const raw = await readFile(fullPath, "utf-8");
4804
- const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
4805
- if (!fmMatch) continue;
4806
- const fmBlock = fmMatch[1];
4807
- const content = fmMatch[2].trim();
4808
- const fm: Record<string, string> = {};
4809
- for (const line of fmBlock.split("\n")) {
4810
- const colonIdx = line.indexOf(":");
4811
- if (colonIdx === -1) continue;
4812
- fm[line.slice(0, colonIdx).trim()] = line
4813
- .slice(colonIdx + 1)
4814
- .trim();
4815
- }
4816
- const created = fm.created || "unknown";
4817
- const createdAt = parseFiniteDate(created);
4818
- if (
4819
- createdAt &&
4820
- formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
4821
- ) {
4822
- continue;
4806
+ let memoryRootReal: string | null = null;
4807
+ try {
4808
+ memoryRootReal = await realpath(storage.dir);
4809
+ } catch {
4810
+ memoryRootReal = null;
4811
+ }
4812
+ for (const categoryDir of RECALL_FALLBACK_DIRS) {
4813
+ if (memoryRootReal === null) break;
4814
+ for (const date of datesToScan) {
4815
+ const dateDir = path.join(storage.dir, categoryDir, date);
4816
+ try {
4817
+ const dirStat = await lstat(dateDir);
4818
+ if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) continue;
4819
+ assertPathInsideRoot(memoryRootReal, await realpath(dateDir), dateDir);
4820
+ const entries = await readdir(dateDir, { withFileTypes: true });
4821
+ for (const entry of entries) {
4822
+ if (entry.isSymbolicLink()) continue;
4823
+ if (!entry.name.endsWith(".md")) continue;
4824
+ const fullPath = path.join(dateDir, entry.name);
4825
+ try {
4826
+ assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
4827
+ const raw = await readFile(fullPath, "utf-8");
4828
+ const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
4829
+ if (!fmMatch) continue;
4830
+ const fmBlock = fmMatch[1];
4831
+ const content = fmMatch[2].trim();
4832
+ const fm: Record<string, string> = {};
4833
+ for (const line of fmBlock.split("\n")) {
4834
+ const colonIdx = line.indexOf(":");
4835
+ if (colonIdx === -1) continue;
4836
+ fm[line.slice(0, colonIdx).trim()] = line
4837
+ .slice(colonIdx + 1)
4838
+ .trim();
4839
+ }
4840
+ const created = fm.created || "unknown";
4841
+ const createdAt = parseFiniteDate(created);
4842
+ if (
4843
+ createdAt &&
4844
+ formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
4845
+ ) {
4846
+ continue;
4847
+ }
4848
+ facts.push({
4849
+ path: fullPath,
4850
+ frontmatter: {
4851
+ id: fm.id || path.basename(entry.name, ".md"),
4852
+ category: (fm.category as any) || "fact",
4853
+ created,
4854
+ updated: fm.updated || created,
4855
+ source: fm.source || "unknown",
4856
+ confidence: parseFloat(fm.confidence || "0.8"),
4857
+ confidenceTier: (fm.confidenceTier as any) || "implied",
4858
+ tags: [],
4859
+ },
4860
+ content,
4861
+ });
4862
+ } catch {
4863
+ // Skip unreadable files
4823
4864
  }
4824
- facts.push({
4825
- path: fullPath,
4826
- frontmatter: {
4827
- id: fm.id || path.basename(entry.name, ".md"),
4828
- category: (fm.category as any) || "fact",
4829
- created,
4830
- updated: fm.updated || created,
4831
- source: fm.source || "unknown",
4832
- confidence: parseFloat(fm.confidence || "0.8"),
4833
- confidenceTier: (fm.confidenceTier as any) || "implied",
4834
- tags: [],
4835
- },
4836
- content,
4837
- });
4838
- } catch {
4839
- // Skip unreadable files
4840
4865
  }
4866
+ } catch {
4867
+ // Absent dir (ENOENT), symlinked/out-of-root dir, or containment
4868
+ // violation — skip this category/date without aborting the summary.
4841
4869
  }
4842
- } catch {
4843
- // Directory doesn't exist — no facts for this date
4844
4870
  }
4845
4871
  }
4846
4872
 
@@ -25,6 +25,7 @@ import {
25
25
  writeFile,
26
26
  unlink,
27
27
  } from "node:fs/promises";
28
+ import { ALL_CATEGORY_DIRS } from "./utils/category-dir.js";
28
29
 
29
30
  // ---------------------------------------------------------------------------
30
31
  // Public interfaces
@@ -445,13 +446,15 @@ function computeLCS(a: string[], b: string[]): string[] {
445
446
  * optional `memoryDir` parameter is omitted.
446
447
  */
447
448
  function resolveMemoryDir(pagePath: string): string {
448
- const knownSubdirs = new Set([
449
- "facts",
450
- "corrections",
449
+ // Derive the recall category dirs from ALL_CATEGORY_DIRS (single source of
450
+ // truth) so newly-routed categories (decisions/, preferences/, ...) are
451
+ // recognized when walking up to the memory root (#1546); the non-category
452
+ // subdirs are listed explicitly.
453
+ const knownSubdirs = new Set<string>([
454
+ ...ALL_CATEGORY_DIRS,
451
455
  "entities",
452
456
  "state",
453
457
  "artifacts",
454
- "questions",
455
458
  "profiles",
456
459
  ]);
457
460
 
@@ -78,3 +78,32 @@ test("scanMemoryDir skips nested symlink entries", async (t) => {
78
78
  await rm(outsideDir, { recursive: true, force: true });
79
79
  }
80
80
  });
81
+
82
+ test("scanMemoryDir indexes memories under category dirs beyond facts/ (issue #1546)", async () => {
83
+ const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-scan-category-"));
84
+ try {
85
+ // A decision routed into decisions/<date>/ must be picked up by the
86
+ // non-QMD index scanner (Orama/Meilisearch/LanceDB), not just facts/.
87
+ const decisionDir = path.join(memoryDir, "decisions", "2026-02-22");
88
+ await mkdir(decisionDir, { recursive: true });
89
+ await writeFile(
90
+ path.join(decisionDir, "decision-1.md"),
91
+ ["---", "id: decision-1", "category: decision", "---", "We chose blue-green deploys."].join("\n"),
92
+ "utf8",
93
+ );
94
+ const factsDir = path.join(memoryDir, "facts", "2026-02-22");
95
+ await mkdir(factsDir, { recursive: true });
96
+ await writeFile(
97
+ path.join(factsDir, "fact-1.md"),
98
+ ["---", "id: fact-1", "category: fact", "---", "The worker retries three times."].join("\n"),
99
+ "utf8",
100
+ );
101
+
102
+ const docs = await scanMemoryDir(memoryDir);
103
+ const ids = docs.map((doc) => doc.docid).sort();
104
+
105
+ assert.deepEqual(ids, ["decision-1", "fact-1"]);
106
+ } finally {
107
+ await rm(memoryDir, { recursive: true, force: true });
108
+ }
109
+ });
@@ -1,5 +1,7 @@
1
1
  import path from "node:path";
2
2
  import { lstat, readdir, readFile, realpath } from "node:fs/promises";
3
+ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
4
+ import { assertPathInsideRoot } from "../utils/path-containment.js";
3
5
 
4
6
  export interface IndexableDocument {
5
7
  /** Memory ID from frontmatter or filename stem */
@@ -94,25 +96,18 @@ function isNodeError(err: unknown): err is NodeJS.ErrnoException {
94
96
  return typeof err === "object" && err !== null && "code" in err;
95
97
  }
96
98
 
97
- function pathIsInside(parent: string, child: string): boolean {
98
- const relative = path.relative(parent, child);
99
- return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
100
- }
101
-
102
- function assertPathInsideRoot(rootReal: string, candidateReal: string, originalPath: string): void {
103
- if (!pathIsInside(rootReal, candidateReal)) {
104
- throw new Error(`Refusing to scan memory path outside memoryDir: ${originalPath}`);
105
- }
106
- }
107
-
108
99
  /**
109
- * Scan `facts/`, `corrections/`, `procedures/`, and `reasoning-traces/`
110
- * subdirs of memoryDir for indexable markdown documents.
111
- *
112
- * Note: reasoning-traces live under their own subtree (issue #564 PR 3).
113
- * Non-QMD backends (Orama / Meilisearch / LanceDB) build their index
114
- * through this helper, so any new category subtree must be listed here
115
- * or those backends silently stop seeing the new memories.
100
+ * Scan every recall category subdir of memoryDir for indexable markdown
101
+ * documents. The directory set is derived from `RECALL_FALLBACK_DIRS`
102
+ * (utils/category-dir.ts → ALL_CATEGORY_DIRS minus non-recall queue dirs) —
103
+ * the single source of truth so adding a new category never requires
104
+ * touching this scanner. Non-QMD backends (Orama / Meilisearch / LanceDB)
105
+ * build their index through this helper; deriving from RECALL_FALLBACK_DIRS
106
+ * keeps them in parity with writeMemory's category-dir routing (issue #1546)
107
+ * and the QMD filesystem-fallback corpus. reasoning-traces/ and the other
108
+ * category dirs are covered automatically (issue #564 PR 3 no longer needs a
109
+ * hand-maintained list). scanDir tolerates missing dirs (ENOENT), so category
110
+ * dirs that do not exist yet are skipped.
116
111
  */
117
112
  export async function scanMemoryDir(memoryDir: string): Promise<IndexableDocument[]> {
118
113
  let memoryRootReal: string;
@@ -124,15 +119,8 @@ export async function scanMemoryDir(memoryDir: string): Promise<IndexableDocumen
124
119
  }
125
120
  throw err;
126
121
  }
127
- const factsDir = path.join(memoryDir, "facts");
128
- const correctionsDir = path.join(memoryDir, "corrections");
129
- const proceduresDir = path.join(memoryDir, "procedures");
130
- const reasoningTracesDir = path.join(memoryDir, "reasoning-traces");
131
- const [facts, corrections, procedures, reasoningTraces] = await Promise.all([
132
- scanDir(factsDir, memoryRootReal),
133
- scanDir(correctionsDir, memoryRootReal),
134
- scanDir(proceduresDir, memoryRootReal),
135
- scanDir(reasoningTracesDir, memoryRootReal),
136
- ]);
137
- return [...facts, ...corrections, ...procedures, ...reasoningTraces];
122
+ const perDir = await Promise.all(
123
+ RECALL_FALLBACK_DIRS.map((dir) => scanDir(path.join(memoryDir, dir), memoryRootReal)),
124
+ );
125
+ return perDir.flat();
138
126
  }
@@ -59,6 +59,7 @@ import {
59
59
  parseEnvelope,
60
60
  seal,
61
61
  } from "./cipher.js";
62
+ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
62
63
 
63
64
  // ---------------------------------------------------------------------------
64
65
  // Error classes
@@ -704,11 +705,24 @@ function normalizeStorageRelativePath(rel: string): string {
704
705
  return normalized;
705
706
  }
706
707
 
707
- const ENCRYPTABLE_MARKDOWN_STORAGE_ROOTS = new Set([
708
- "facts",
709
- "corrections",
710
- "procedures",
711
- "reasoning-traces",
708
+ // Every recall MEMORY category directory (facts/ + the CATEGORY_DIR_MAP memory
709
+ // dirs) must be encryptable at rest — #1546 routes decision/preference/moment/...
710
+ // memories into their own dirs, so hardcoding only facts/corrections/procedures/
711
+ // reasoning-traces would silently write those categories in plaintext on
712
+ // encrypted stores. RECALL_FALLBACK_DIRS is the single source of truth for that
713
+ // set; the extra non-category markdown roots (artifacts/archive/entities/
714
+ // identity) are appended.
715
+ //
716
+ // questions/ is deliberately EXCLUDED (RECALL_FALLBACK_DIRS omits the non-memory
717
+ // queue dirs): the question queue is written/resolved through plain readFile/
718
+ // writeFile (StorageManager.writeQuestion/resolveQuestion), NOT the secure-file
719
+ // helpers. Encrypting questions/ here would make migration seal those files,
720
+ // after which resolveQuestion() would read ciphertext as UTF-8, fail to update
721
+ // the frontmatter, and overwrite/corrupt the file while returning success
722
+ // (codex #1563 review). Keeping questions/ plaintext preserves the queue's
723
+ // existing behavior; #1546 does not change question encryption.
724
+ const ENCRYPTABLE_MARKDOWN_STORAGE_ROOTS = new Set<string>([
725
+ ...RECALL_FALLBACK_DIRS,
712
726
  "artifacts",
713
727
  "archive",
714
728
  "entities",
@@ -935,3 +935,31 @@ test("backspace on BMP character removes exactly one character (thread 7 — reg
935
935
 
936
936
  assert.equal(result, "a");
937
937
  });
938
+
939
+ test("secure-store migration encrypts memory category dirs but leaves the questions queue plaintext", async () => {
940
+ // #1546 routes decision/preference/... memories into their own dirs, which
941
+ // must be encryptable at rest. But questions/ is written/resolved via plain
942
+ // readFile/writeFile (writeQuestion/resolveQuestion), so encrypting it would
943
+ // make resolveQuestion() read ciphertext as UTF-8 and corrupt the file
944
+ // (codex #1563 review). Migration must seal decisions/ but skip questions/.
945
+ const tempRoot = await mkdtemp(path.join(tmpdir(), "remnic-secure-store-questions-"));
946
+ try {
947
+ const memoryDir = path.join(tempRoot, "memory");
948
+ const decisionPath = path.join(memoryDir, "decisions", "2026-02-22", "decision-1.md");
949
+ const questionPath = path.join(memoryDir, "questions", "q-1.md");
950
+ await mkdir(path.dirname(decisionPath), { recursive: true });
951
+ await mkdir(path.dirname(questionPath), { recursive: true });
952
+ await writeFile(decisionPath, "---\ncategory: decision\n---\n\nChose Postgres.\n", "utf8");
953
+ await writeFile(questionPath, "---\nid: q-1\nresolved: false\n---\n\nWhat DB?\n", "utf8");
954
+
955
+ const key = deriveKeyScrypt("questions-plaintext", Buffer.alloc(KDF_SALT_LENGTH, 0x5a), FAST_SCRYPT);
956
+ const migrated = await migrateMemoryDirToEncrypted(memoryDir, key);
957
+
958
+ assert.equal(migrated.encrypted, 1, "only the decision memory should be encrypted");
959
+ assert.equal(isEncryptedFile(await readFile(decisionPath)), true, "decisions/ is encrypted at rest");
960
+ assert.equal(isEncryptedFile(await readFile(questionPath)), false, "questions/ stays plaintext");
961
+ assert.match(await readFile(questionPath, "utf8"), /What DB\?/, "question body remains readable UTF-8");
962
+ } finally {
963
+ await rm(tempRoot, { recursive: true, force: true });
964
+ }
965
+ });
package/src/storage.ts CHANGED
@@ -4,7 +4,7 @@ import { createHash } from "node:crypto";
4
4
  import path from "node:path";
5
5
  import { log } from "./logger.js";
6
6
  import { isErrnoCode } from "./utils/errno.js";
7
- import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
7
+ import { RECALL_FALLBACK_DIRS, getCategoryDir, categoryDirName } from "./utils/category-dir.js";
8
8
  import { getCachedEntities, invalidateAllForDir, setCachedEntities } from "./memory-cache.js";
9
9
  import { rotateMarkdownFileToArchive } from "./hygiene.js";
10
10
  import { sanitizeMemoryContent } from "./sanitize.js";
@@ -3297,6 +3297,32 @@ export class StorageManager {
3297
3297
  await mkdir(path.join(this.baseDir, "config"), { recursive: true });
3298
3298
  }
3299
3299
 
3300
+ /**
3301
+ * Resolve the on-disk write path for a memory of the given category, creating
3302
+ * the target directory. Category routing goes through the shared
3303
+ * `getCategoryDir()` chokepoint (utils/category-dir.ts → CATEGORY_DIR_MAP) so
3304
+ * decision/preference/moment/etc. outputs land in their dedicated dirs
3305
+ * (`decisions/`, `preferences/`, ...) instead of collapsing into `facts/`
3306
+ * (issue #1546; CLAUDE.md rule 39). `correction` keeps its historical flat
3307
+ * layout (no `<date>` subdir) as the corrections pipeline expects; every other
3308
+ * category — including `fact`/`entity`, which fall back to `facts/` — is dated
3309
+ * as `<dir>/<date>/`. Read/scan/reindex already iterate every category dir
3310
+ * (RECALL_FALLBACK_DIRS; QMD scans baseDir recursively), so writes stay found.
3311
+ */
3312
+ private async resolveCategoryWritePath(
3313
+ category: MemoryCategory,
3314
+ id: string,
3315
+ today: string,
3316
+ ): Promise<string> {
3317
+ if (category === "correction") {
3318
+ await mkdir(this.correctionsDir, { recursive: true });
3319
+ return path.join(this.correctionsDir, `${id}.md`);
3320
+ }
3321
+ const datedDir = path.join(getCategoryDir(this.baseDir, category), today);
3322
+ await mkdir(datedDir, { recursive: true });
3323
+ return path.join(datedDir, `${id}.md`);
3324
+ }
3325
+
3300
3326
  async writeMemory(
3301
3327
  category: MemoryCategory,
3302
3328
  content: string,
@@ -3435,20 +3461,7 @@ export class StorageManager {
3435
3461
 
3436
3462
  const fileContent = `${serializeFrontmatter(fm)}\n\n${sanitized.text}\n`;
3437
3463
 
3438
- let filePath: string;
3439
- if (category === "correction") {
3440
- filePath = path.join(this.correctionsDir, `${id}.md`);
3441
- } else if (category === "procedure") {
3442
- await mkdir(path.join(this.proceduresDir, today), { recursive: true });
3443
- filePath = path.join(this.proceduresDir, today, `${id}.md`);
3444
- } else if (category === "reasoning_trace") {
3445
- // Issue #564 PR 3: reasoning traces live in their own subtree so recall
3446
- // can filter on path cheaply without parsing frontmatter.
3447
- await mkdir(path.join(this.reasoningTracesDir, today), { recursive: true });
3448
- filePath = path.join(this.reasoningTracesDir, today, `${id}.md`);
3449
- } else {
3450
- filePath = path.join(this.factsDir, today, `${id}.md`);
3451
- }
3464
+ const filePath = await this.resolveCategoryWritePath(category, id, today);
3452
3465
 
3453
3466
  await this.snapshotBeforeWrite(filePath, "write");
3454
3467
  await writeMaybeEncryptedFile(filePath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
@@ -4325,9 +4338,10 @@ export class StorageManager {
4325
4338
  * Read all memories from the cold tier by scanning the entire cold/ root
4326
4339
  * tree. Previously this only scanned cold/facts/ and cold/corrections/, but
4327
4340
  * structuredAttributes can appear on any MemoryCategory (preference, decision,
4328
- * entity, etc.). Although buildTierMemoryPath currently routes all
4329
- * non-correction, non-artifact memories to cold/facts/, scanning the full
4330
- * coldRoot ensures correctness if that routing ever changes and guards against
4341
+ * entity, etc.). buildTierMemoryPath now routes each category to its own
4342
+ * cold/<dir>/ subtree via the shared categoryDirName() chokepoint (issue
4343
+ * #1546), so cold decisions/preferences/... live outside cold/facts/.
4344
+ * Scanning the full coldRoot covers every category dir and guards against
4331
4345
  * files placed in unexpected subdirectories during manual operations or future
4332
4346
  * refactors.
4333
4347
  *
@@ -4535,19 +4549,17 @@ export class StorageManager {
4535
4549
  return path.join(root, "artifacts", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4536
4550
  }
4537
4551
  if (memory.frontmatter.category === "correction") {
4552
+ // corrections/ is flat (no date subdir); preserved across tier moves.
4538
4553
  return path.join(root, "corrections", `${memory.frontmatter.id}.md`);
4539
4554
  }
4540
- if (memory.frontmatter.category === "procedure") {
4541
- return path.join(root, "procedures", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4542
- }
4543
- if (memory.frontmatter.category === "reasoning_trace") {
4544
- // Issue #564 PR 3: preserve the dedicated reasoning-traces/ subtree
4545
- // across tier moves. Without this branch, hot→cold migration would
4546
- // funnel the memory into facts/, breaking isReasoningTracePath() and
4547
- // silently disabling the recall boost for migrated traces.
4548
- return path.join(root, "reasoning-traces", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4549
- }
4550
- return path.join(root, "facts", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4555
+ // Every other category decisions/, preferences/, reasoning-traces/, ...
4556
+ // plus the facts/ fallback for fact/entity/unknown — resolves through the
4557
+ // shared categoryDirName() chokepoint so tier moves land in the SAME dir the
4558
+ // writer used, instead of funneling non-{correction,procedure,reasoning_trace}
4559
+ // categories into facts/ (issue #564 PR 3 preserved reasoning-traces/; #1546
4560
+ // generalizes it to every category dir).
4561
+ const dir = categoryDirName(memory.frontmatter.category);
4562
+ return path.join(root, dir, this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
4551
4563
  }
4552
4564
 
4553
4565
  private async writeMemoryFileAtomic(targetPath: string, memory: MemoryFile): Promise<void> {
@@ -6984,20 +6996,7 @@ export class StorageManager {
6984
6996
  }
6985
6997
  const fileContent = `${serializeFrontmatter(fm)}\n\n${sanitized.text}\n`;
6986
6998
 
6987
- let filePath: string;
6988
- if (category === "correction") {
6989
- filePath = path.join(this.correctionsDir, `${id}.md`);
6990
- } else if (category === "procedure") {
6991
- await mkdir(path.join(this.proceduresDir, today), { recursive: true });
6992
- filePath = path.join(this.proceduresDir, today, `${id}.md`);
6993
- } else if (category === "reasoning_trace") {
6994
- // Issue #564 PR 3: chunks of a reasoning_trace memory live alongside the
6995
- // parent in reasoning-traces/<date>/.
6996
- await mkdir(path.join(this.reasoningTracesDir, today), { recursive: true });
6997
- filePath = path.join(this.reasoningTracesDir, today, `${id}.md`);
6998
- } else {
6999
- filePath = path.join(this.factsDir, today, `${id}.md`);
7000
- }
6999
+ const filePath = await this.resolveCategoryWritePath(category, id, today);
7001
7000
 
7002
7001
  await this.writeStorageSecureFile(filePath, fileContent);
7003
7002
  log.debug(`wrote chunk ${id} (${chunkIndex + 1}/${chunkTotal}) to ${filePath}`);