@remnic/core 9.3.662 → 9.3.664

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/dist/access-cli.js +25 -23
  2. package/dist/access-cli.js.map +1 -1
  3. package/dist/access-http.js +20 -18
  4. package/dist/access-mcp.js +19 -17
  5. package/dist/access-schema.js +4 -3
  6. package/dist/access-service.js +17 -15
  7. package/dist/briefing.js +5 -4
  8. package/dist/{capsule-merge-T2JRE46P.js → capsule-merge-GK5E647P.js} +3 -2
  9. package/dist/{capsule-merge-T2JRE46P.js.map → capsule-merge-GK5E647P.js.map} +1 -1
  10. package/dist/causal-consolidation.js +6 -5
  11. package/dist/causal-consolidation.js.map +1 -1
  12. package/dist/{chunk-2KDQI363.js → chunk-2HEZXPYU.js} +4 -4
  13. package/dist/chunk-5GPPACXK.js +16 -0
  14. package/dist/chunk-5GPPACXK.js.map +1 -0
  15. package/dist/{chunk-F6O7IOS3.js → chunk-6JBKHTQD.js} +2 -2
  16. package/dist/{chunk-TBLGI2LT.js → chunk-7ILWCUWH.js} +5 -3
  17. package/dist/{chunk-TBLGI2LT.js.map → chunk-7ILWCUWH.js.map} +1 -1
  18. package/dist/{chunk-AL4RAJL5.js → chunk-7XH7VJN4.js} +6 -4
  19. package/dist/chunk-7XH7VJN4.js.map +1 -0
  20. package/dist/{chunk-Q4CAQGKQ.js → chunk-AER6MT24.js} +12 -21
  21. package/dist/chunk-AER6MT24.js.map +1 -0
  22. package/dist/{chunk-DHGSZ3UD.js → chunk-ARV3AUOM.js} +2 -2
  23. package/dist/{chunk-PXVFMQLD.js → chunk-BZG2CWOQ.js} +3 -3
  24. package/dist/{chunk-ANJOULTP.js → chunk-C7AF236A.js} +2 -2
  25. package/dist/{chunk-FZC2WSDB.js → chunk-DOCTITOP.js} +2 -2
  26. package/dist/{chunk-WOQIHC67.js → chunk-DQY7NJ5L.js} +2 -2
  27. package/dist/{chunk-NMPEJV5M.js → chunk-DSLUOQDY.js} +2 -2
  28. package/dist/{chunk-A7EF2XRO.js → chunk-EXXBA5OM.js} +30 -8
  29. package/dist/chunk-EXXBA5OM.js.map +1 -0
  30. package/dist/{chunk-QXHBWFR3.js → chunk-IHG6CC7T.js} +2 -2
  31. package/dist/{chunk-4KDLCMLK.js → chunk-IROWLAWG.js} +5 -5
  32. package/dist/{chunk-ILXTATKK.js → chunk-J2HSAU72.js} +5 -5
  33. package/dist/chunk-J2HSAU72.js.map +1 -0
  34. package/dist/{chunk-DFAXGZKI.js → chunk-JIX3ZL2J.js} +8 -8
  35. package/dist/{chunk-GY3V3SUI.js → chunk-KHGE6PMF.js} +2 -2
  36. package/dist/{chunk-TGOOJCGA.js → chunk-LIERUFPO.js} +76 -54
  37. package/dist/chunk-LIERUFPO.js.map +1 -0
  38. package/dist/{chunk-HSCJYHYV.js → chunk-NLF54XMD.js} +49 -19
  39. package/dist/chunk-NLF54XMD.js.map +1 -0
  40. package/dist/{chunk-TWAJICBN.js → chunk-OHJFJ4HI.js} +2 -2
  41. package/dist/{chunk-WSQG37DV.js → chunk-OUWAQVDJ.js} +2 -2
  42. package/dist/{chunk-ZLDUQWT2.js → chunk-PWWWLD7D.js} +2 -2
  43. package/dist/{chunk-ZJH723NM.js → chunk-Q5ZU3RNY.js} +2 -2
  44. package/dist/{chunk-35HP3TGR.js → chunk-ROHLEUTH.js} +4 -4
  45. package/dist/{chunk-5RIRL3XL.js → chunk-RS25QOKZ.js} +2 -2
  46. package/dist/{chunk-RQGR3ETH.js → chunk-T2AN3BSP.js} +2 -2
  47. package/dist/{chunk-UAU5U5ML.js → chunk-UDJLF3BO.js} +2 -2
  48. package/dist/{chunk-ALEPI75L.js → chunk-VF4XKTX3.js} +6 -4
  49. package/dist/{chunk-ALEPI75L.js.map → chunk-VF4XKTX3.js.map} +1 -1
  50. package/dist/{chunk-AX5O25EF.js → chunk-VH6EIKVS.js} +152 -190
  51. package/dist/chunk-VH6EIKVS.js.map +1 -0
  52. package/dist/chunk-VS2IYZRU.js +43 -0
  53. package/dist/chunk-VS2IYZRU.js.map +1 -0
  54. package/dist/{chunk-YYQRVNSV.js → chunk-XB5P5P2L.js} +6 -6
  55. package/dist/{chunk-D2EFNQMY.js → chunk-XW3W4PV4.js} +2 -2
  56. package/dist/{chunk-5AYAZN45.js → chunk-YKX63GBK.js} +5 -5
  57. package/dist/{chunk-TYIXG4VR.js → chunk-YW52BQSU.js} +2 -2
  58. package/dist/{cli-C6twwe84.d.ts → cli-BQRqR9N-.d.ts} +12 -1
  59. package/dist/cli.d.ts +1 -1
  60. package/dist/cli.js +32 -28
  61. package/dist/compounding/engine.js +5 -4
  62. package/dist/connectors/codex-materialize-runner.js +5 -4
  63. package/dist/connectors/index.js +5 -4
  64. package/dist/consolidation-provenance-check.js +3 -2
  65. package/dist/consolidation-undo.js +2 -1
  66. package/dist/consolidation-undo.js.map +1 -1
  67. package/dist/entity-retrieval.js +5 -4
  68. package/dist/index.d.ts +1 -1
  69. package/dist/index.js +39 -36
  70. package/dist/index.js.map +1 -1
  71. package/dist/maintenance/memory-governance.js +6 -4
  72. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +5 -4
  73. package/dist/maintenance/rebuild-memory-projection.js +7 -5
  74. package/dist/namespaces/migrate.js +13 -11
  75. package/dist/namespaces/search.js +8 -6
  76. package/dist/namespaces/storage.js +5 -4
  77. package/dist/offline-sync.js +3 -2
  78. package/dist/operator-toolkit.js +16 -14
  79. package/dist/orchestrator.js +21 -19
  80. package/dist/page-versioning.js +2 -1
  81. package/dist/schemas.d.ts +22 -22
  82. package/dist/search/document-scanner.d.ts +11 -7
  83. package/dist/search/document-scanner.js +3 -1
  84. package/dist/search/factory.js +7 -5
  85. package/dist/search/index.js +7 -5
  86. package/dist/search/lancedb-backend.js +4 -2
  87. package/dist/search/meilisearch-backend.js +4 -2
  88. package/dist/search/orama-backend.js +4 -2
  89. package/dist/secure-store/index.js +3 -2
  90. package/dist/semantic-consolidation.js +6 -5
  91. package/dist/semantic-rule-promotion.js +5 -4
  92. package/dist/semantic-rule-verifier.js +5 -4
  93. package/dist/storage.d.ts +17 -3
  94. package/dist/storage.js +4 -3
  95. package/dist/transfer/capsule-import.js +3 -2
  96. package/dist/transfer/types.d.ts +12 -12
  97. package/dist/verified-recall.js +5 -4
  98. package/package.json +1 -1
  99. package/src/cli.ts +62 -23
  100. package/src/consolidation-provenance-check.ts +7 -6
  101. package/src/maintenance/memory-governance.ts +47 -7
  102. package/src/orchestrator.ts +84 -58
  103. package/src/page-versioning.ts +7 -4
  104. package/src/search/document-scanner.test.ts +29 -0
  105. package/src/search/document-scanner.ts +17 -29
  106. package/src/secure-store/secure-fs.ts +19 -5
  107. package/src/secure-store/secure-store.test.ts +28 -0
  108. package/src/storage.ts +42 -43
  109. package/src/training-export/converter.test.ts +19 -0
  110. package/src/training-export/converter.ts +8 -5
  111. package/src/utils/category-dir.ts +10 -4
  112. package/src/utils/path-containment.ts +40 -0
  113. package/dist/chunk-A7EF2XRO.js.map +0 -1
  114. package/dist/chunk-AL4RAJL5.js.map +0 -1
  115. package/dist/chunk-AX5O25EF.js.map +0 -1
  116. package/dist/chunk-HSCJYHYV.js.map +0 -1
  117. package/dist/chunk-ILXTATKK.js.map +0 -1
  118. package/dist/chunk-Q4CAQGKQ.js.map +0 -1
  119. package/dist/chunk-TGOOJCGA.js.map +0 -1
  120. /package/dist/{chunk-2KDQI363.js.map → chunk-2HEZXPYU.js.map} +0 -0
  121. /package/dist/{chunk-F6O7IOS3.js.map → chunk-6JBKHTQD.js.map} +0 -0
  122. /package/dist/{chunk-DHGSZ3UD.js.map → chunk-ARV3AUOM.js.map} +0 -0
  123. /package/dist/{chunk-PXVFMQLD.js.map → chunk-BZG2CWOQ.js.map} +0 -0
  124. /package/dist/{chunk-ANJOULTP.js.map → chunk-C7AF236A.js.map} +0 -0
  125. /package/dist/{chunk-FZC2WSDB.js.map → chunk-DOCTITOP.js.map} +0 -0
  126. /package/dist/{chunk-WOQIHC67.js.map → chunk-DQY7NJ5L.js.map} +0 -0
  127. /package/dist/{chunk-NMPEJV5M.js.map → chunk-DSLUOQDY.js.map} +0 -0
  128. /package/dist/{chunk-QXHBWFR3.js.map → chunk-IHG6CC7T.js.map} +0 -0
  129. /package/dist/{chunk-4KDLCMLK.js.map → chunk-IROWLAWG.js.map} +0 -0
  130. /package/dist/{chunk-DFAXGZKI.js.map → chunk-JIX3ZL2J.js.map} +0 -0
  131. /package/dist/{chunk-GY3V3SUI.js.map → chunk-KHGE6PMF.js.map} +0 -0
  132. /package/dist/{chunk-TWAJICBN.js.map → chunk-OHJFJ4HI.js.map} +0 -0
  133. /package/dist/{chunk-WSQG37DV.js.map → chunk-OUWAQVDJ.js.map} +0 -0
  134. /package/dist/{chunk-ZLDUQWT2.js.map → chunk-PWWWLD7D.js.map} +0 -0
  135. /package/dist/{chunk-ZJH723NM.js.map → chunk-Q5ZU3RNY.js.map} +0 -0
  136. /package/dist/{chunk-35HP3TGR.js.map → chunk-ROHLEUTH.js.map} +0 -0
  137. /package/dist/{chunk-5RIRL3XL.js.map → chunk-RS25QOKZ.js.map} +0 -0
  138. /package/dist/{chunk-RQGR3ETH.js.map → chunk-T2AN3BSP.js.map} +0 -0
  139. /package/dist/{chunk-UAU5U5ML.js.map → chunk-UDJLF3BO.js.map} +0 -0
  140. /package/dist/{chunk-YYQRVNSV.js.map → chunk-XB5P5P2L.js.map} +0 -0
  141. /package/dist/{chunk-D2EFNQMY.js.map → chunk-XW3W4PV4.js.map} +0 -0
  142. /package/dist/{chunk-5AYAZN45.js.map → chunk-YKX63GBK.js.map} +0 -0
  143. /package/dist/{chunk-TYIXG4VR.js.map → chunk-YW52BQSU.js.map} +0 -0
package/src/cli.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import os from "node:os";
2
2
  import path from "node:path";
3
- import { access, readFile, readdir, unlink } from "node:fs/promises";
3
+ import { access, lstat, readFile, readdir, realpath, unlink } from "node:fs/promises";
4
4
  import { createHash } from "node:crypto";
5
5
  import type { Readable, Writable } from "node:stream";
6
6
  import type { Orchestrator } from "./orchestrator.js";
@@ -215,6 +215,8 @@ import {
215
215
  } from "./policy-runtime.js";
216
216
  import { resolveHomeDir } from "./runtime/env.js";
217
217
  import { expandTildePath } from "./utils/path.js";
218
+ import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
219
+ import { assertPathInsideRoot } from "./utils/path-containment.js";
218
220
  import { convertMemoriesToRecords } from "./training-export/converter.js";
219
221
  import { parseStrictCliDate as parseStrictCliDateShared } from "./training-export/date-parse.js";
220
222
  import { getTrainingExportAdapter, listTrainingExportAdapters } from "./training-export/registry.js";
@@ -3320,32 +3322,69 @@ export async function resolveMemoryDirForNamespace(
3320
3322
  }
3321
3323
 
3322
3324
  /**
3323
- * Walk `memoryDir/{facts,corrections}` recursively and invoke `visit` for
3324
- * every `*.md` file. Intentionally swallows per-directory errors so a missing
3325
- * subdir reads as empty. Shared primitive for `listMemoryMarkdownFilePaths`,
3326
- * `readAllMemoryFiles`, and any future walker that needs the same roots +
3327
- * `.md` filter.
3325
+ * Walk every recall category directory under `memoryDir` recursively and invoke
3326
+ * `visit` for every `*.md` file. The directory set is `RECALL_FALLBACK_DIRS`
3327
+ * (single source of truth), so newly-routed categories (decisions/, ...) are
3328
+ * covered without touching this walker again (#1546). Swallows per-directory
3329
+ * errors so a missing subdir reads as empty. Shared primitive for
3330
+ * `listMemoryMarkdownFilePaths`, `readAllMemoryFiles`, and future walkers.
3331
+ *
3332
+ * Symlink/containment hardening (mirrors `scanDir` in
3333
+ * search/document-scanner.ts): downstream consumers include `readAllMemoryFiles`
3334
+ * → the `dedupe-exact` / `dedupe-aggressive` commands, which `unlink()` files.
3335
+ * Because this walker scans EVERY category root, a symlinked category dir
3336
+ * (e.g. `decisions/` → outside `memoryDir`) could otherwise redirect the walk —
3337
+ * and a destructive dedupe delete — outside the memory store. We resolve the
3338
+ * memory root once, skip symlinked dirs/entries, and assert every realpath stays
3339
+ * inside the root before descending or visiting. A single poisoned entry is
3340
+ * skipped (logged), never aborting a legitimate dedupe run.
3328
3341
  */
3329
3342
  async function walkMemoryMarkdownFiles(
3330
3343
  memoryDir: string,
3331
3344
  visit: (fullPath: string) => void | Promise<void>,
3332
3345
  ): Promise<void> {
3333
- const roots = [path.join(memoryDir, "facts"), path.join(memoryDir, "corrections")];
3346
+ let memoryRootReal: string;
3347
+ try {
3348
+ memoryRootReal = await realpath(memoryDir);
3349
+ } catch {
3350
+ return; // memoryDir itself does not exist — nothing to walk.
3351
+ }
3352
+
3353
+ const skip = (target: string, err: unknown): void => {
3354
+ // ENOENT (optional dir absent) is silent; containment/other errors log.
3355
+ if (!(err instanceof Error && /ENOENT/.test(err.message))) {
3356
+ console.debug(`walkMemoryMarkdownFiles: skipping ${target}: ${err instanceof Error ? err.message : String(err)}`);
3357
+ }
3358
+ };
3334
3359
 
3335
3360
  const walk = async (dir: string): Promise<void> => {
3336
- let entries: Array<{ isDirectory(): boolean; isFile(): boolean; name: string | Buffer }>;
3361
+ // Reject symlinked / non-directory roots and anything resolving outside the
3362
+ // memory store before reading it.
3337
3363
  try {
3338
- entries = (await readdir(dir, { withFileTypes: true })) as Array<{
3339
- isDirectory(): boolean;
3340
- isFile(): boolean;
3341
- name: string | Buffer;
3342
- }>;
3364
+ const dirStat = await lstat(dir);
3365
+ if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) return;
3366
+ assertPathInsideRoot(memoryRootReal, await realpath(dir), dir);
3367
+ } catch (err) {
3368
+ skip(dir, err);
3369
+ return;
3370
+ }
3371
+
3372
+ let entries: Array<{ isDirectory(): boolean; isFile(): boolean; isSymbolicLink(): boolean; name: string | Buffer }>;
3373
+ try {
3374
+ entries = (await readdir(dir, { withFileTypes: true })) as typeof entries;
3343
3375
  } catch {
3344
3376
  return;
3345
3377
  }
3346
3378
  for (const entry of entries) {
3379
+ if (entry.isSymbolicLink()) continue; // never follow symlinked entries
3347
3380
  const entryName = typeof entry.name === "string" ? entry.name : entry.name.toString("utf-8");
3348
3381
  const fullPath = path.join(dir, entryName);
3382
+ try {
3383
+ assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
3384
+ } catch (err) {
3385
+ skip(fullPath, err); // poisoned entry — skip, keep walking the rest.
3386
+ continue;
3387
+ }
3349
3388
  if (entry.isDirectory()) {
3350
3389
  await walk(fullPath);
3351
3390
  continue;
@@ -3355,22 +3394,22 @@ async function walkMemoryMarkdownFiles(
3355
3394
  }
3356
3395
  };
3357
3396
 
3358
- for (const root of roots) {
3397
+ for (const root of RECALL_FALLBACK_DIRS.map((dir) => path.join(memoryDir, dir))) {
3359
3398
  await walk(root);
3360
3399
  }
3361
3400
  }
3362
3401
 
3363
3402
  /**
3364
- * List absolute paths of every `*.md` file under `memoryDir/{facts,corrections}`.
3365
- * Used by the bulk-import CLI to derive a per-batch `memoriesCreated` count
3366
- * via set-subtraction of "paths after extraction" against "paths before
3367
- * extraction". Caveat: the extraction queue is shared across sessions, so
3368
- * concurrent organic extractions that write memories between the two
3369
- * snapshots will still inflate the reported count. Filename-set diff at
3370
- * least correctly ignores pre-existing files and files that were deleted
3371
- * while the batch ran.
3403
+ * List absolute paths of every `*.md` file under each recall category directory
3404
+ * of `memoryDir` (facts/, corrections/, decisions/, ...; see
3405
+ * `walkMemoryMarkdownFiles`). Used by the bulk-import CLI to derive a per-batch
3406
+ * `memoriesCreated` count via set-subtraction of "paths after extraction"
3407
+ * against "paths before extraction". Caveat: the extraction queue is shared
3408
+ * across sessions, so concurrent organic extractions between the two snapshots
3409
+ * can still inflate the count; the set diff at least ignores pre-existing and
3410
+ * deleted files. Exported for category-dir coverage tests (#1546).
3372
3411
  */
3373
- async function listMemoryMarkdownFilePaths(memoryDir: string): Promise<string[]> {
3412
+ export async function listMemoryMarkdownFilePaths(memoryDir: string): Promise<string[]> {
3374
3413
  const paths: string[] = [];
3375
3414
  await walkMemoryMarkdownFiles(memoryDir, (fullPath) => {
3376
3415
  paths.push(fullPath);
@@ -29,6 +29,7 @@ import {
29
29
  // review, cursor Medium) so a future key-format change stays in
30
30
  // lock-step with the doctor scan.
31
31
  import { sidecarKey } from "./page-versioning.js";
32
+ import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
32
33
 
33
34
  /**
34
35
  * Regex to spot a `derived_via: <value>` line in the raw YAML frontmatter
@@ -490,14 +491,14 @@ export async function runConsolidationProvenanceCheck(options: {
490
491
 
491
492
  // Parse-failure detection (PR #634 round-4 review, codex P2):
492
493
  // `readAllMemories()` silently drops files whose frontmatter
493
- // doesn't parse. Walk the facts/ and corrections/ directories for
494
- // `.md` files that DO reference provenance frontmatter but didn't
495
- // come back from the reader — those are the corruption cases the
496
- // doctor is meant to surface.
494
+ // doesn't parse. Walk every recall category directory for `.md` files
495
+ // that DO reference provenance frontmatter but didn't come back from the
496
+ // reader — those are the corruption cases the doctor is meant to surface.
497
+ // Uses RECALL_FALLBACK_DIRS (the single source of truth) so newly-routed
498
+ // categories (decisions/, preferences/, ...) are scanned too (#1546).
497
499
  try {
498
500
  const seenPaths = new Set(memories.map((m) => m.path));
499
- const scanRoots = ["facts", "corrections", "procedures", "reasoning-traces"];
500
- for (const rootName of scanRoots) {
501
+ for (const rootName of RECALL_FALLBACK_DIRS) {
501
502
  const rootPath = path.join(memoryDir, rootName);
502
503
  for await (const file of walkMarkdownFiles(rootPath, memoryDir)) {
503
504
  if (seenPaths.has(file)) continue;
@@ -1,8 +1,10 @@
1
1
  import path from "node:path";
2
- import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
2
+ import { lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
3
3
  import { StorageManager } from "../storage.js";
4
4
  import { decideLifecycleTransition } from "../lifecycle.js";
5
5
  import type { MemoryFile, MemoryStatus } from "../types.js";
6
+ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
7
+ import { assertPathInsideRoot } from "../utils/path-containment.js";
6
8
 
7
9
  export type MemoryGovernanceMode = "shadow" | "apply";
8
10
  export type MemoryGovernanceReasonCode =
@@ -337,23 +339,42 @@ function buildExplicitCaptureReviewEntries(
337
339
  }));
338
340
  }
339
341
 
340
- async function listMarkdownFiles(root: string): Promise<string[]> {
342
+ /**
343
+ * List every `*.md` under `root`, refusing to follow symlinked directories or
344
+ * entries that resolve outside `containmentRoot` (a realpath-resolved memory
345
+ * store root). Mirrors the containment guard in consolidation-provenance's
346
+ * walkMarkdownFiles / the CLI walker, reusing the shared assertPathInsideRoot
347
+ * (rule 22). Without this, a symlinked category dir (e.g. decisions/ → outside
348
+ * memoryDir) would be walked and its files surfaced by the malformed-import
349
+ * governance sweep — an out-of-store info leak.
350
+ */
351
+ async function listMarkdownFiles(root: string, containmentRoot: string): Promise<string[]> {
341
352
  const files: string[] = [];
342
353
  const walk = async (dir: string) => {
343
354
  try {
355
+ const dirStat = await lstat(dir);
356
+ if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) return;
357
+ assertPathInsideRoot(containmentRoot, await realpath(dir), dir);
344
358
  const entries = await readdir(dir, { withFileTypes: true });
345
359
  for (const entry of entries) {
360
+ if (entry.isSymbolicLink()) continue; // never follow symlinked entries
346
361
  const fullPath = path.join(dir, entry.name);
347
362
  if (entry.isDirectory()) {
348
363
  await walk(fullPath);
349
364
  continue;
350
365
  }
351
366
  if (entry.isFile() && entry.name.endsWith(".md")) {
367
+ try {
368
+ assertPathInsideRoot(containmentRoot, await realpath(fullPath), fullPath);
369
+ } catch {
370
+ continue; // poisoned entry escaping the root — skip it.
371
+ }
352
372
  files.push(fullPath);
353
373
  }
354
374
  }
355
375
  } catch {
356
- // Directory may not exist yet.
376
+ // Absent dir (ENOENT), symlinked/out-of-root dir, or containment
377
+ // violation — skip this subtree without aborting the sweep.
357
378
  }
358
379
  };
359
380
 
@@ -372,10 +393,29 @@ async function buildMalformedImportEntries(
372
393
  candidateFiles?: string[],
373
394
  ): Promise<MemoryGovernanceReviewQueueEntry[]> {
374
395
  const parsedPaths = new Set(parsedMemories.map((memory) => memory.path));
375
- const filesToInspect = candidateFiles ?? [
376
- ...await listMarkdownFiles(path.join(memoryDir, "facts")),
377
- ...await listMarkdownFiles(path.join(memoryDir, "corrections")),
378
- ];
396
+ // Inspect every recall category directory (RECALL_FALLBACK_DIRS — the single
397
+ // source of truth) so malformed files under newly-routed categories
398
+ // (decisions/, preferences/, ...) are surfaced too, not just facts/ (#1546).
399
+ // Resolve the containment root (realpath) once and thread it into the walker
400
+ // so a symlinked category dir can't escape memoryDir. If memoryDir itself
401
+ // can't be resolved, there is nothing valid to inspect.
402
+ let containmentRoot: string | null = null;
403
+ try {
404
+ containmentRoot = await realpath(memoryDir);
405
+ } catch {
406
+ containmentRoot = null;
407
+ }
408
+ const filesToInspect =
409
+ candidateFiles ??
410
+ (containmentRoot === null
411
+ ? []
412
+ : (
413
+ await Promise.all(
414
+ RECALL_FALLBACK_DIRS.map((dir) =>
415
+ listMarkdownFiles(path.join(memoryDir, dir), containmentRoot as string),
416
+ ),
417
+ )
418
+ ).flat());
379
419
  const entries: MemoryGovernanceReviewQueueEntry[] = [];
380
420
 
381
421
  for (const filePath of filesToInspect) {
@@ -8,9 +8,11 @@ import os from "node:os";
8
8
  import { createHash, randomBytes } from "node:crypto";
9
9
  import { existsSync, readFileSync } from "node:fs";
10
10
  import {
11
+ lstat,
11
12
  mkdir,
12
13
  readdir,
13
14
  readFile,
15
+ realpath,
14
16
  stat,
15
17
  unlink,
16
18
  writeFile,
@@ -343,6 +345,8 @@ import {
343
345
  defaultTierMigrationCycleBudget,
344
346
  } from "./compounding/engine.js";
345
347
  import { parseFlexibleIsoTimestamp } from "./utils/iso-timestamp.js";
348
+ import { categoryDirName, RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
349
+ import { assertPathInsideRoot } from "./utils/path-containment.js";
346
350
  // IRC preference consolidation — used by eval adapter directly;
347
351
  // orchestrator integration planned for future PR.
348
352
  // import { consolidatePreferences, buildQueryAwarePreferenceSection, synthesizePreferencesFromLcm } from "./compounding/preference-consolidator.js";
@@ -1758,16 +1762,13 @@ export function resolvePersistedMemoryRelativePath(options: {
1758
1762
  }
1759
1763
  // Pick the subtree that matches the StorageManager.writeMemory routing
1760
1764
  // so fallback paths (used before memoryPathById has seen the fresh
1761
- // write) agree with where the file actually lives. Without this branch,
1762
- // reasoning_trace graph edges point at facts/<date>/, and subsequent
1763
- // graph expansion silently drops those nodes when readMemoryByPath
1764
- // cannot resolve them (issue #564 PR 3 review).
1765
- const subtree =
1766
- options.category === "procedure"
1767
- ? "procedures"
1768
- : options.category === "reasoning_trace"
1769
- ? "reasoning-traces"
1770
- : "facts";
1765
+ // write) agree with where the file actually lives. Routing goes through
1766
+ // the shared categoryDirName() chokepoint (utils/category-dir.ts) so
1767
+ // every category decisions/, preferences/, reasoning-traces/, ...
1768
+ // resolves to the same dir the writer used; otherwise graph edges point
1769
+ // at the wrong subtree and graph expansion silently drops those nodes
1770
+ // when readMemoryByPath cannot resolve them (issue #564 PR 3 / #1546).
1771
+ const subtree = categoryDirName(options.category);
1771
1772
  const idParts = options.memoryId.split("-");
1772
1773
  const maybeTimestamp = Number(idParts[1]);
1773
1774
  if (Number.isFinite(maybeTimestamp) && maybeTimestamp > 0) {
@@ -4787,60 +4788,85 @@ export class Orchestrator {
4787
4788
  // a local calendar day. Scan the UTC-date envelope that overlaps the local
4788
4789
  // day, then filter parseable fact timestamps to that configured local day.
4789
4790
  const datesToScan = utcDateKeysForLocalDay(now, timeZone);
4790
- const factsBaseDir = path.join(storage.dir, "facts");
4791
4791
  const MAX_CHARS = 100_000;
4792
4792
 
4793
- // --- Read fact files from each date directory ---
4793
+ // --- Read memory files from each category dir × date directory ---
4794
+ // Iterate every recall category dir (RECALL_FALLBACK_DIRS — single source
4795
+ // of truth) so the day summary includes decisions/, moments/, ... not just
4796
+ // facts/ (#1546). corrections/ is flat, so corrections/<date>/ never exists
4797
+ // and is skipped by the ENOENT guard — preserving the prior exclusion. The
4798
+ // per-file created→local-day filter below is unchanged.
4799
+ //
4800
+ // Symlink/containment hardening (mirrors scanDir / the CLI walker): the
4801
+ // gathered contents feed the day-summary LLM input, so a symlinked category
4802
+ // dir (decisions/ → outside memoryDir) must not be followed and leak files.
4803
+ // Resolve the store root once; skip symlinked / out-of-root dirs and
4804
+ // entries; skip the scan gracefully if the root can't be resolved.
4794
4805
  const facts: MemoryFile[] = [];
4795
- for (const date of datesToScan) {
4796
- const factsDir = path.join(factsBaseDir, date);
4797
- try {
4798
- const entries = await readdir(factsDir, { withFileTypes: true });
4799
- for (const entry of entries) {
4800
- if (!entry.name.endsWith(".md")) continue;
4801
- const fullPath = path.join(factsDir, entry.name);
4802
- try {
4803
- const raw = await readFile(fullPath, "utf-8");
4804
- const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
4805
- if (!fmMatch) continue;
4806
- const fmBlock = fmMatch[1];
4807
- const content = fmMatch[2].trim();
4808
- const fm: Record<string, string> = {};
4809
- for (const line of fmBlock.split("\n")) {
4810
- const colonIdx = line.indexOf(":");
4811
- if (colonIdx === -1) continue;
4812
- fm[line.slice(0, colonIdx).trim()] = line
4813
- .slice(colonIdx + 1)
4814
- .trim();
4815
- }
4816
- const created = fm.created || "unknown";
4817
- const createdAt = parseFiniteDate(created);
4818
- if (
4819
- createdAt &&
4820
- formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
4821
- ) {
4822
- continue;
4806
+ let memoryRootReal: string | null = null;
4807
+ try {
4808
+ memoryRootReal = await realpath(storage.dir);
4809
+ } catch {
4810
+ memoryRootReal = null;
4811
+ }
4812
+ for (const categoryDir of RECALL_FALLBACK_DIRS) {
4813
+ if (memoryRootReal === null) break;
4814
+ for (const date of datesToScan) {
4815
+ const dateDir = path.join(storage.dir, categoryDir, date);
4816
+ try {
4817
+ const dirStat = await lstat(dateDir);
4818
+ if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) continue;
4819
+ assertPathInsideRoot(memoryRootReal, await realpath(dateDir), dateDir);
4820
+ const entries = await readdir(dateDir, { withFileTypes: true });
4821
+ for (const entry of entries) {
4822
+ if (entry.isSymbolicLink()) continue;
4823
+ if (!entry.name.endsWith(".md")) continue;
4824
+ const fullPath = path.join(dateDir, entry.name);
4825
+ try {
4826
+ assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
4827
+ const raw = await readFile(fullPath, "utf-8");
4828
+ const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
4829
+ if (!fmMatch) continue;
4830
+ const fmBlock = fmMatch[1];
4831
+ const content = fmMatch[2].trim();
4832
+ const fm: Record<string, string> = {};
4833
+ for (const line of fmBlock.split("\n")) {
4834
+ const colonIdx = line.indexOf(":");
4835
+ if (colonIdx === -1) continue;
4836
+ fm[line.slice(0, colonIdx).trim()] = line
4837
+ .slice(colonIdx + 1)
4838
+ .trim();
4839
+ }
4840
+ const created = fm.created || "unknown";
4841
+ const createdAt = parseFiniteDate(created);
4842
+ if (
4843
+ createdAt &&
4844
+ formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
4845
+ ) {
4846
+ continue;
4847
+ }
4848
+ facts.push({
4849
+ path: fullPath,
4850
+ frontmatter: {
4851
+ id: fm.id || path.basename(entry.name, ".md"),
4852
+ category: (fm.category as any) || "fact",
4853
+ created,
4854
+ updated: fm.updated || created,
4855
+ source: fm.source || "unknown",
4856
+ confidence: parseFloat(fm.confidence || "0.8"),
4857
+ confidenceTier: (fm.confidenceTier as any) || "implied",
4858
+ tags: [],
4859
+ },
4860
+ content,
4861
+ });
4862
+ } catch {
4863
+ // Skip unreadable files
4823
4864
  }
4824
- facts.push({
4825
- path: fullPath,
4826
- frontmatter: {
4827
- id: fm.id || path.basename(entry.name, ".md"),
4828
- category: (fm.category as any) || "fact",
4829
- created,
4830
- updated: fm.updated || created,
4831
- source: fm.source || "unknown",
4832
- confidence: parseFloat(fm.confidence || "0.8"),
4833
- confidenceTier: (fm.confidenceTier as any) || "implied",
4834
- tags: [],
4835
- },
4836
- content,
4837
- });
4838
- } catch {
4839
- // Skip unreadable files
4840
4865
  }
4866
+ } catch {
4867
+ // Absent dir (ENOENT), symlinked/out-of-root dir, or containment
4868
+ // violation — skip this category/date without aborting the summary.
4841
4869
  }
4842
- } catch {
4843
- // Directory doesn't exist — no facts for this date
4844
4870
  }
4845
4871
  }
4846
4872
 
@@ -25,6 +25,7 @@ import {
25
25
  writeFile,
26
26
  unlink,
27
27
  } from "node:fs/promises";
28
+ import { ALL_CATEGORY_DIRS } from "./utils/category-dir.js";
28
29
 
29
30
  // ---------------------------------------------------------------------------
30
31
  // Public interfaces
@@ -445,13 +446,15 @@ function computeLCS(a: string[], b: string[]): string[] {
445
446
  * optional `memoryDir` parameter is omitted.
446
447
  */
447
448
  function resolveMemoryDir(pagePath: string): string {
448
- const knownSubdirs = new Set([
449
- "facts",
450
- "corrections",
449
+ // Derive the recall category dirs from ALL_CATEGORY_DIRS (single source of
450
+ // truth) so newly-routed categories (decisions/, preferences/, ...) are
451
+ // recognized when walking up to the memory root (#1546); the non-category
452
+ // subdirs are listed explicitly.
453
+ const knownSubdirs = new Set<string>([
454
+ ...ALL_CATEGORY_DIRS,
451
455
  "entities",
452
456
  "state",
453
457
  "artifacts",
454
- "questions",
455
458
  "profiles",
456
459
  ]);
457
460
 
@@ -78,3 +78,32 @@ test("scanMemoryDir skips nested symlink entries", async (t) => {
78
78
  await rm(outsideDir, { recursive: true, force: true });
79
79
  }
80
80
  });
81
+
82
+ test("scanMemoryDir indexes memories under category dirs beyond facts/ (issue #1546)", async () => {
83
+ const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-scan-category-"));
84
+ try {
85
+ // A decision routed into decisions/<date>/ must be picked up by the
86
+ // non-QMD index scanner (Orama/Meilisearch/LanceDB), not just facts/.
87
+ const decisionDir = path.join(memoryDir, "decisions", "2026-02-22");
88
+ await mkdir(decisionDir, { recursive: true });
89
+ await writeFile(
90
+ path.join(decisionDir, "decision-1.md"),
91
+ ["---", "id: decision-1", "category: decision", "---", "We chose blue-green deploys."].join("\n"),
92
+ "utf8",
93
+ );
94
+ const factsDir = path.join(memoryDir, "facts", "2026-02-22");
95
+ await mkdir(factsDir, { recursive: true });
96
+ await writeFile(
97
+ path.join(factsDir, "fact-1.md"),
98
+ ["---", "id: fact-1", "category: fact", "---", "The worker retries three times."].join("\n"),
99
+ "utf8",
100
+ );
101
+
102
+ const docs = await scanMemoryDir(memoryDir);
103
+ const ids = docs.map((doc) => doc.docid).sort();
104
+
105
+ assert.deepEqual(ids, ["decision-1", "fact-1"]);
106
+ } finally {
107
+ await rm(memoryDir, { recursive: true, force: true });
108
+ }
109
+ });
@@ -1,5 +1,7 @@
1
1
  import path from "node:path";
2
2
  import { lstat, readdir, readFile, realpath } from "node:fs/promises";
3
+ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
4
+ import { assertPathInsideRoot } from "../utils/path-containment.js";
3
5
 
4
6
  export interface IndexableDocument {
5
7
  /** Memory ID from frontmatter or filename stem */
@@ -94,25 +96,18 @@ function isNodeError(err: unknown): err is NodeJS.ErrnoException {
94
96
  return typeof err === "object" && err !== null && "code" in err;
95
97
  }
96
98
 
97
- function pathIsInside(parent: string, child: string): boolean {
98
- const relative = path.relative(parent, child);
99
- return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
100
- }
101
-
102
- function assertPathInsideRoot(rootReal: string, candidateReal: string, originalPath: string): void {
103
- if (!pathIsInside(rootReal, candidateReal)) {
104
- throw new Error(`Refusing to scan memory path outside memoryDir: ${originalPath}`);
105
- }
106
- }
107
-
108
99
  /**
109
- * Scan `facts/`, `corrections/`, `procedures/`, and `reasoning-traces/`
110
- * subdirs of memoryDir for indexable markdown documents.
111
- *
112
- * Note: reasoning-traces live under their own subtree (issue #564 PR 3).
113
- * Non-QMD backends (Orama / Meilisearch / LanceDB) build their index
114
- * through this helper, so any new category subtree must be listed here
115
- * or those backends silently stop seeing the new memories.
100
+ * Scan every recall category subdir of memoryDir for indexable markdown
101
+ * documents. The directory set is derived from `RECALL_FALLBACK_DIRS`
102
+ * (utils/category-dir.ts → ALL_CATEGORY_DIRS minus non-recall queue dirs) —
103
+ * the single source of truth so adding a new category never requires
104
+ * touching this scanner. Non-QMD backends (Orama / Meilisearch / LanceDB)
105
+ * build their index through this helper; deriving from RECALL_FALLBACK_DIRS
106
+ * keeps them in parity with writeMemory's category-dir routing (issue #1546)
107
+ * and the QMD filesystem-fallback corpus. reasoning-traces/ and the other
108
+ * category dirs are covered automatically (issue #564 PR 3 no longer needs a
109
+ * hand-maintained list). scanDir tolerates missing dirs (ENOENT), so category
110
+ * dirs that do not exist yet are skipped.
116
111
  */
117
112
  export async function scanMemoryDir(memoryDir: string): Promise<IndexableDocument[]> {
118
113
  let memoryRootReal: string;
@@ -124,15 +119,8 @@ export async function scanMemoryDir(memoryDir: string): Promise<IndexableDocumen
124
119
  }
125
120
  throw err;
126
121
  }
127
- const factsDir = path.join(memoryDir, "facts");
128
- const correctionsDir = path.join(memoryDir, "corrections");
129
- const proceduresDir = path.join(memoryDir, "procedures");
130
- const reasoningTracesDir = path.join(memoryDir, "reasoning-traces");
131
- const [facts, corrections, procedures, reasoningTraces] = await Promise.all([
132
- scanDir(factsDir, memoryRootReal),
133
- scanDir(correctionsDir, memoryRootReal),
134
- scanDir(proceduresDir, memoryRootReal),
135
- scanDir(reasoningTracesDir, memoryRootReal),
136
- ]);
137
- return [...facts, ...corrections, ...procedures, ...reasoningTraces];
122
+ const perDir = await Promise.all(
123
+ RECALL_FALLBACK_DIRS.map((dir) => scanDir(path.join(memoryDir, dir), memoryRootReal)),
124
+ );
125
+ return perDir.flat();
138
126
  }
@@ -59,6 +59,7 @@ import {
59
59
  parseEnvelope,
60
60
  seal,
61
61
  } from "./cipher.js";
62
+ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
62
63
 
63
64
  // ---------------------------------------------------------------------------
64
65
  // Error classes
@@ -704,11 +705,24 @@ function normalizeStorageRelativePath(rel: string): string {
704
705
  return normalized;
705
706
  }
706
707
 
707
- const ENCRYPTABLE_MARKDOWN_STORAGE_ROOTS = new Set([
708
- "facts",
709
- "corrections",
710
- "procedures",
711
- "reasoning-traces",
708
+ // Every recall MEMORY category directory (facts/ + the CATEGORY_DIR_MAP memory
709
+ // dirs) must be encryptable at rest — #1546 routes decision/preference/moment/...
710
+ // memories into their own dirs, so hardcoding only facts/corrections/procedures/
711
+ // reasoning-traces would silently write those categories in plaintext on
712
+ // encrypted stores. RECALL_FALLBACK_DIRS is the single source of truth for that
713
+ // set; the extra non-category markdown roots (artifacts/archive/entities/
714
+ // identity) are appended.
715
+ //
716
+ // questions/ is deliberately EXCLUDED (RECALL_FALLBACK_DIRS omits the non-memory
717
+ // queue dirs): the question queue is written/resolved through plain readFile/
718
+ // writeFile (StorageManager.writeQuestion/resolveQuestion), NOT the secure-file
719
+ // helpers. Encrypting questions/ here would make migration seal those files,
720
+ // after which resolveQuestion() would read ciphertext as UTF-8, fail to update
721
+ // the frontmatter, and overwrite/corrupt the file while returning success
722
+ // (codex #1563 review). Keeping questions/ plaintext preserves the queue's
723
+ // existing behavior; #1546 does not change question encryption.
724
+ const ENCRYPTABLE_MARKDOWN_STORAGE_ROOTS = new Set<string>([
725
+ ...RECALL_FALLBACK_DIRS,
712
726
  "artifacts",
713
727
  "archive",
714
728
  "entities",
@@ -935,3 +935,31 @@ test("backspace on BMP character removes exactly one character (thread 7 — reg
935
935
 
936
936
  assert.equal(result, "a");
937
937
  });
938
+
939
+ test("secure-store migration encrypts memory category dirs but leaves the questions queue plaintext", async () => {
940
+ // #1546 routes decision/preference/... memories into their own dirs, which
941
+ // must be encryptable at rest. But questions/ is written/resolved via plain
942
+ // readFile/writeFile (writeQuestion/resolveQuestion), so encrypting it would
943
+ // make resolveQuestion() read ciphertext as UTF-8 and corrupt the file
944
+ // (codex #1563 review). Migration must seal decisions/ but skip questions/.
945
+ const tempRoot = await mkdtemp(path.join(tmpdir(), "remnic-secure-store-questions-"));
946
+ try {
947
+ const memoryDir = path.join(tempRoot, "memory");
948
+ const decisionPath = path.join(memoryDir, "decisions", "2026-02-22", "decision-1.md");
949
+ const questionPath = path.join(memoryDir, "questions", "q-1.md");
950
+ await mkdir(path.dirname(decisionPath), { recursive: true });
951
+ await mkdir(path.dirname(questionPath), { recursive: true });
952
+ await writeFile(decisionPath, "---\ncategory: decision\n---\n\nChose Postgres.\n", "utf8");
953
+ await writeFile(questionPath, "---\nid: q-1\nresolved: false\n---\n\nWhat DB?\n", "utf8");
954
+
955
+ const key = deriveKeyScrypt("questions-plaintext", Buffer.alloc(KDF_SALT_LENGTH, 0x5a), FAST_SCRYPT);
956
+ const migrated = await migrateMemoryDirToEncrypted(memoryDir, key);
957
+
958
+ assert.equal(migrated.encrypted, 1, "only the decision memory should be encrypted");
959
+ assert.equal(isEncryptedFile(await readFile(decisionPath)), true, "decisions/ is encrypted at rest");
960
+ assert.equal(isEncryptedFile(await readFile(questionPath)), false, "questions/ stays plaintext");
961
+ assert.match(await readFile(questionPath, "utf8"), /What DB\?/, "question body remains readable UTF-8");
962
+ } finally {
963
+ await rm(tempRoot, { recursive: true, force: true });
964
+ }
965
+ });