@ijfw/memory-server 1.5.4 → 1.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/package.json +15 -1
  2. package/src/brain/dream-pipeline.js +77 -14
  3. package/src/brain/dump-ingest.js +32 -0
  4. package/src/brain/entity-collapse.js +2 -2
  5. package/src/brain/export.js +60 -6
  6. package/src/brain/extractors/markdown.js +28 -2
  7. package/src/brain/layout-sentinel.js +19 -14
  8. package/src/brain/path-guard.js +17 -0
  9. package/src/brain/wiki-compiler.js +35 -39
  10. package/src/codex-agents.js +25 -2
  11. package/src/cross-orchestrator-cli.js +176 -18
  12. package/src/dashboard-server.js +36 -3
  13. package/src/dispatch/override.js +18 -2
  14. package/src/dispatch/signer-cli.js +14 -9
  15. package/src/dream/stage-runner.js +17 -0
  16. package/src/dream/state-file.js +15 -1
  17. package/src/extension-installer.js +91 -2
  18. package/src/extension-registry.js +15 -4
  19. package/src/handlers/brain-handler.js +44 -5
  20. package/src/lib/atomic-io.js +69 -12
  21. package/src/lib/shasum-verify.js +46 -22
  22. package/src/lib/ui-review-runner.js +7 -2
  23. package/src/lib/uispec-drift.js +8 -3
  24. package/src/lib/uispec-intake.js +5 -2
  25. package/src/memory/layout-migrations/001-visible-layer.js +71 -7
  26. package/src/memory/reader.js +111 -58
  27. package/src/orchestrator/merge-block-aware.js +75 -37
  28. package/src/orchestrator/post-done-runner.js +6 -1
  29. package/src/orchestrator/state-sdk.js +242 -14
  30. package/src/orchestrator/wave-state.js +22 -69
  31. package/src/recovery/checkpoint.js +30 -6
  32. package/src/recovery/code-fixer.js +52 -7
  33. package/src/runtime-mediator.js +2 -2
  34. package/src/server.js +57 -8
  35. package/src/swarm/planner.js +46 -1
  36. package/src/update-apply.js +27 -35
  37. package/src/update-check.js +6 -2
package/package.json CHANGED
@@ -1,9 +1,23 @@
1
1
  {
2
2
  "name": "@ijfw/memory-server",
3
- "version": "1.5.4",
3
+ "version": "1.5.6",
4
4
  "description": "Cross-platform persistent memory server for IJFW. 14 MCP tools (memory + admin/update + brain). Works with 15 platforms: 14 via MCP (Claude Code, Codex, Gemini CLI, Cursor, Windsurf, Copilot, Hermes, Wayland, OpenCode, QwenCode, Cline, KimiCode, OpenClaw, Antigravity) plus Aider via the rules-only tier.",
5
5
  "author": "Sean Donahoe",
6
+ "contributors": [
7
+ {
8
+ "name": "Ferrox Labs",
9
+ "url": "https://ferroxlabs.com"
10
+ }
11
+ ],
6
12
  "license": "MIT",
13
+ "homepage": "https://github.com/FerroxLabs/ijfw",
14
+ "bugs": {
15
+ "url": "https://github.com/FerroxLabs/ijfw/issues"
16
+ },
17
+ "repository": {
18
+ "type": "git",
19
+ "url": "git+https://github.com/FerroxLabs/ijfw.git"
20
+ },
7
21
  "type": "module",
8
22
  "main": "src/server.js",
9
23
  "bin": {
@@ -16,8 +16,10 @@
16
16
  // Budget: every LLM call goes through BudgetGuard. budgetExhausted=true
17
17
  // signals the cycle stopped voluntarily (not crashed).
18
18
 
19
- import { mkdirSync, appendFileSync, lstatSync } from 'node:fs';
20
- import { dirname } from 'node:path';
19
+ import {
20
+ mkdirSync, existsSync, openSync, writeSync, closeSync, constants as fsConstants,
21
+ } from 'node:fs';
22
+ import { join, dirname } from 'node:path';
21
23
  import { resolveBrainPaths } from './paths.js';
22
24
  import { scanInbox, writeManifest, commitProcessed, isProcessed } from './dump-ingest.js';
23
25
  import { extractFile } from './extractors/index.js';
@@ -25,6 +27,7 @@ import { BudgetGuard } from './budget-guard.js';
25
27
  import { compileWikiPage } from './wiki-compiler.js';
26
28
  import { callTiered } from './tiered-llm.js';
27
29
  import { validateSafeRepoPath } from './path-guard.js';
30
+ import { withFsLock, lockPathFor } from '../fs-lock.js';
28
31
 
29
32
  function ensureFactsTable(db) {
30
33
  // Idempotent: matches the schema downstream consumers expect.
@@ -41,18 +44,33 @@ function ensureFactsTable(db) {
41
44
  function appendLog(wikiLogPath, line, repoRoot) {
42
45
  try {
43
46
  // F-LENS2-05/11: refuse to follow a symlinked log path out of the repo.
44
- // lstat (NOT stat) so we see the symlink itself; if the path is a
45
- // symlink, drop the append rather than write through it.
46
47
  if (repoRoot) {
47
48
  const guard = validateSafeRepoPath(repoRoot, wikiLogPath);
48
49
  if (!guard.ok) return;
49
50
  }
50
- try {
51
- const lst = lstatSync(wikiLogPath);
52
- if (lst.isSymbolicLink()) return; // F-LENS2-11: refuse symlink follow
53
- } catch { /* file doesn't exist yet — ok to create */ }
54
51
  mkdirSync(dirname(wikiLogPath), { recursive: true });
55
- appendFileSync(wikiLogPath, line + '\n');
52
+ // V155-039 (MED): the prior `lstatSync` → `appendFileSync` sequence had
53
+ // a TOCTOU window — an attacker could swap a regular file for a symlink
54
+ // between the two syscalls so the write follows the new symlink (e.g.
55
+ // to /etc/passwd). `O_NOFOLLOW` makes `openSync` fail atomically if the
56
+ // FINAL path component is a symlink, closing the race with a single
57
+ // syscall. `O_APPEND` keeps append semantics; `O_CREAT` creates on
58
+ // first write. Mode 0o600 keeps log private.
59
+ let fd;
60
+ try {
61
+ // eslint-disable-next-line no-bitwise
62
+ fd = openSync(
63
+ wikiLogPath,
64
+ fsConstants.O_APPEND | fsConstants.O_CREAT | fsConstants.O_WRONLY | fsConstants.O_NOFOLLOW,
65
+ 0o600,
66
+ );
67
+ } catch {
68
+ // ELOOP / EACCES → refuse to write. No fallback path — logging is
69
+ // best-effort and a symlinked log path is a refusal, not a degrade.
70
+ return;
71
+ }
72
+ try { writeSync(fd, line + '\n'); }
73
+ finally { try { closeSync(fd); } catch { /* best-effort */ } }
56
74
  } catch { /* logging is best-effort */ }
57
75
  }
58
76
 
@@ -179,6 +197,23 @@ export async function defaultExtractFacts({ file: _file, text, chunks, env, guar
179
197
 
180
198
  function nowIso() { return new Date().toISOString(); }
181
199
 
200
+ // V155-020 (HIGH): facts-table source check. `isProcessed(processed, name)`
201
+ // only inspects the manifest file — `rm -rf <processed>/` looks like a
202
+ // clean slate to the gate, so re-ingest doubles every triple (cycle 2 = 2x,
203
+ // cycle 3 = 3x, …). Cross-check the facts table: if ANY fact already
204
+ // references this file as its `source`, treat it as processed regardless
205
+ // of manifest presence. Cleaner than adding a UNIQUE constraint (no
206
+ // migration required); the source column already stores file.name.
207
+ function isProcessedDouble(db, processedDir, fileName) {
208
+ if (isProcessed(processedDir, fileName)) return true;
209
+ try {
210
+ const row = db.prepare(
211
+ 'SELECT 1 FROM facts WHERE source = ? LIMIT 1'
212
+ ).get(fileName);
213
+ return !!row;
214
+ } catch { return false; }
215
+ }
216
+
182
217
  export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId, extractFacts } = {}) {
183
218
  if (!db) throw new Error('dream-pipeline: db required');
184
219
  if (!repoRoot) throw new Error('dream-pipeline: repoRoot required');
@@ -195,6 +230,32 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
195
230
  const guard = BudgetGuard(guardOpts);
196
231
  const extractor = extractFacts || defaultExtractFacts;
197
232
 
233
+ // V155-064 (LOW): gate the mkdirSyncs on existsSync so the syscalls don't
234
+ // fire on every cycle (cheap-and-correct today, but if a future watcher
235
+ // layers fs-events on those dirs the mkdir wakeups become hot-path noise).
236
+ // `mkdir … recursive:true` is idempotent at the kernel layer; this is a
237
+ // userspace short-circuit.
238
+ if (!existsSync(paths.dumpInbox)) mkdirSync(paths.dumpInbox, { recursive: true });
239
+ if (!existsSync(paths.dumpProcessed)) mkdirSync(paths.dumpProcessed, { recursive: true });
240
+
241
+ // V155-038 (MED): project-scope lock around the entire run. Two
242
+ // simultaneous `ijfw_brain dream` triggers (manual + chokidar; or two
243
+ // operators on the same repo) would otherwise BOTH scan the same inbox,
244
+ // BOTH call the (paid) LLM extractor, and BOTH run the rename race for
245
+ // `commitProcessed` — the loser silently catches ENOENT but its facts
246
+ // already landed (no UNIQUE constraint on facts). Net: double-extract
247
+ // cost + duplicate facts + single committed file. The lock makes a
248
+ // concurrent invocation wait for the first to finish; the second then
249
+ // sees the manifests (and V155-020 facts-table cross-check) and exits
250
+ // with zero candidates.
251
+ const lockTarget = join(repoRoot, '.ijfw', 'state', '.dream-cycle.lock');
252
+ if (!existsSync(dirname(lockTarget))) mkdirSync(dirname(lockTarget), { recursive: true });
253
+ return withFsLock(lockPathFor(lockTarget), async () => runDreamCycleLocked({
254
+ db, repoRoot, env, paths, cid, guard, extractor,
255
+ }));
256
+ }
257
+
258
+ async function runDreamCycleLocked({ db, repoRoot, env, paths, cid, guard, extractor }) {
198
259
  let processed = 0;
199
260
  let factsInserted = 0;
200
261
  let pagesCompiled = 0;
@@ -202,11 +263,11 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
202
263
  const touchedSubjects = new Set();
203
264
  const errors = [];
204
265
 
205
- mkdirSync(paths.dumpInbox, { recursive: true });
206
- mkdirSync(paths.dumpProcessed, { recursive: true });
207
-
266
+ // V155-020: cross-check the facts table so a `rm -rf <processed>/` cannot
267
+ // trigger re-ingest doubling. The manifest gate stays as the cheap first
268
+ // check; only filename-collision survivors get the DB query.
208
269
  const candidates = scanInbox(paths.dumpInbox).filter(
209
- (f) => !isProcessed(paths.dumpProcessed, f.name)
270
+ (f) => !isProcessedDouble(db, paths.dumpProcessed, f.name)
210
271
  );
211
272
 
212
273
  for (const file of candidates) {
@@ -310,8 +371,10 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
310
371
  }
311
372
 
312
373
  // Compile pages for touched subjects. Failures are logged, not fatal.
374
+ // V155-015: compileWikiPage is now async (withFsLock primitive).
313
375
  for (const subject of touchedSubjects) {
314
- const r = compileWikiPage(db, { repoRoot, type: 'entity', subject });
376
+ // eslint-disable-next-line no-await-in-loop
377
+ const r = await compileWikiPage(db, { repoRoot, type: 'entity', subject });
315
378
  if (r.ok) {
316
379
  pagesCompiled += 1;
317
380
  appendLog(paths.wikiLog, `${nowIso()} compile entity ${subject} facts=${r.factsCount}`, repoRoot);
@@ -79,10 +79,42 @@ export function isProcessed(processedDir, fileName) {
79
79
  return existsSync(manifestPath(processedDir, fileName));
80
80
  }
81
81
 
82
+ // readManifest: legacy behaviour — returns the parsed JSON payload directly,
83
+ // throws on missing/unreadable/parse-fail. Callers in tests + E2E paths rely
84
+ // on the direct round-trip shape; do NOT break the contract.
85
+ //
86
+ // V155-068 (v1.5.5): added `readManifestSafe` (below) for callers — like the
87
+ // dream cycle's manifest-reprocess logic — that need to treat
88
+ // corrupt/absent as a graceful "needs reprocess" signal rather than letting
89
+ // JSON.parse abort the whole cycle.
82
90
  export function readManifest(processedDir, fileName) {
83
91
  return JSON.parse(readFileSync(manifestPath(processedDir, fileName), 'utf8'));
84
92
  }
85
93
 
94
+ /**
95
+ * Tolerant variant of readManifest for callers that need to distinguish:
96
+ * - { ok: true, data } — clean read
97
+ * - { ok: false, code: 'enoent' } — manifest absent
98
+ * - { ok: false, code: 'unreadable', message } — readFile threw
99
+ * - { ok: false, code: 'parse-fail', message, path } — JSON.parse threw
100
+ *
101
+ * V155-068 (v1.5.5): closes the dream-cycle crash on tampered manifests.
102
+ */
103
+ export function readManifestSafe(processedDir, fileName) {
104
+ const p = manifestPath(processedDir, fileName);
105
+ if (!existsSync(p)) return { ok: false, code: 'enoent' };
106
+ let raw;
107
+ try { raw = readFileSync(p, 'utf8'); }
108
+ catch (e) {
109
+ return { ok: false, code: 'unreadable', message: e?.message || String(e) };
110
+ }
111
+ try {
112
+ return { ok: true, data: JSON.parse(raw) };
113
+ } catch (e) {
114
+ return { ok: false, code: 'parse-fail', message: e?.message || String(e), path: p };
115
+ }
116
+ }
117
+
86
118
  function manifestPath(processedDir, fileName) {
87
119
  return join(processedDir, `${fileName}.manifest.json`);
88
120
  }
@@ -2,8 +2,8 @@
2
2
  //
3
3
  // canonicalize(s): lowercase + collapsed-whitespace + trim. findCandidateMerges(db)
4
4
  // surfaces groups of distinct stored subject strings that normalize to the same
5
- // form (e.g. "Sean Donahoe" / "sean donahoe" / " Sean Donahoe " all collapse
6
- // to "sean donahoe"). Promotion (actual merge) is operator-confirmed -- this
5
+ // form (e.g. "Jane Doe" / "jane doe" / " Jane Doe " all collapse
6
+ // to "jane doe"). Promotion (actual merge) is operator-confirmed -- this
7
7
  // module only surfaces candidates.
8
8
 
9
9
  export function canonicalize(s) {
@@ -12,7 +12,7 @@
12
12
  // Both functions tolerate the v1->v2 layout transition: they consult the
13
13
  // layout sentinel via resolveBrainPaths to pick the right wiki location.
14
14
 
15
- import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
15
+ import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from 'node:fs';
16
16
  import { dirname, join } from 'node:path';
17
17
  import { resolveBrainPaths } from './paths.js';
18
18
  import { validateSafeRepoPath } from './path-guard.js';
@@ -47,6 +47,34 @@ function parseWikilinks(md) {
47
47
  return [...out];
48
48
  }
49
49
 
50
+ // V155-034 (v1.5.5): export size caps. Wiki pages are written by the dream
51
+ // pipeline from user-droppable `dump/inbox/` content, so a poisoned dump can
52
+ // produce a page with `[[link1]]...[[linkN]]` plus a few 100MB linked pages.
53
+ // Without caps, `wiki.export` concatenates everything into memory and OOMs the
54
+ // MCP server. The caps bound:
55
+ // - any single page (root or linked) at MAX_PAGE_BYTES
56
+ // - the total bundle at MAX_BUNDLE_BYTES
57
+ // Truncated pages are replaced with a sentinel header so the consumer can
58
+ // tell the export was capped, not silently empty.
59
+ const MAX_PAGE_BYTES = 2 * 1024 * 1024; // 2 MB per page
60
+ const MAX_BUNDLE_BYTES = 10 * 1024 * 1024; // 10 MB per bundle
61
+
62
+ function readPageCapped(filePath) {
63
+ try {
64
+ const st = statSync(filePath);
65
+ if (st.size > MAX_PAGE_BYTES) {
66
+ return {
67
+ truncated: true,
68
+ sizeBytes: st.size,
69
+ body: `[... truncated — page is ${(st.size / 1024 / 1024).toFixed(2)} MB; export cap is ${(MAX_PAGE_BYTES / 1024 / 1024).toFixed(0)} MB ...]`,
70
+ };
71
+ }
72
+ return { truncated: false, sizeBytes: st.size, body: readFileSync(filePath, 'utf8') };
73
+ } catch (e) {
74
+ return { truncated: false, error: e?.message || String(e), body: null };
75
+ }
76
+ }
77
+
50
78
  export function exportPageBundle(repoRoot, slug, outFile) {
51
79
  // F-LENS2-05 defense-in-depth: brain-handler already validates outFile via
52
80
  // validateSafeRepoPath, but exportPageBundle is also exported and may be
@@ -57,26 +85,52 @@ export function exportPageBundle(repoRoot, slug, outFile) {
57
85
  const paths = resolveBrainPaths(repoRoot);
58
86
  const root = findPage(paths.wikiDir, slug);
59
87
  if (!root) return { error: 'page-not-found', slug };
60
- const rootBody = readFileSync(root.path, 'utf8');
88
+ const rootRead = readPageCapped(root.path);
89
+ if (rootRead.body == null) {
90
+ return { error: 'root-page-unreadable', slug, message: rootRead.error };
91
+ }
92
+ const rootBody = rootRead.body;
61
93
  const linked = parseWikilinks(rootBody);
62
94
 
63
95
  const parts = [
64
96
  `# Export: ${slug}\n\n_Generated by IJFW brain export from ${root.type}/${slug}.md._\n`,
65
97
  `## ${slug}\n\n${rootBody.trim()}`,
66
98
  ];
99
+ let runningBytes = Buffer.byteLength(parts.join('\n\n'), 'utf8');
67
100
  const includedLinks = [];
101
+ const truncatedLinks = [];
102
+ const skippedLinks = [];
103
+ let bundleTruncated = false;
68
104
  for (const target of linked) {
105
+ if (runningBytes >= MAX_BUNDLE_BYTES) {
106
+ bundleTruncated = true;
107
+ skippedLinks.push(target);
108
+ continue;
109
+ }
69
110
  const found = findPage(paths.wikiDir, target);
70
111
  if (!found) continue;
71
- let body;
72
- try { body = readFileSync(found.path, 'utf8'); } catch { continue; }
73
- parts.push(`### ${target}\n\n${body.trim()}`);
112
+ const r = readPageCapped(found.path);
113
+ if (r.body == null) continue;
114
+ const block = `### ${target}\n\n${r.body.trim()}`;
115
+ runningBytes += Buffer.byteLength(block, 'utf8') + 2; // approx '\n\n'
116
+ parts.push(block);
74
117
  includedLinks.push(target);
118
+ if (r.truncated) truncatedLinks.push(target);
75
119
  }
120
+ if (rootRead.truncated && !truncatedLinks.includes(slug)) truncatedLinks.unshift(slug);
121
+
76
122
  const out = parts.join('\n\n') + '\n';
77
123
  mkdirSync(dirname(outFile), { recursive: true });
78
124
  writeFileSync(outFile, out);
79
- return { outFile, bytes: Buffer.byteLength(out, 'utf8'), linkedPagesIncluded: includedLinks };
125
+ return {
126
+ outFile,
127
+ bytes: Buffer.byteLength(out, 'utf8'),
128
+ linkedPagesIncluded: includedLinks,
129
+ // Diagnostic fields — only present when caps actually fired so the common
130
+ // case stays terse.
131
+ ...(truncatedLinks.length > 0 ? { truncatedPages: truncatedLinks } : {}),
132
+ ...(bundleTruncated ? { bundleTruncated: true, skippedLinks } : {}),
133
+ };
80
134
  }
81
135
 
82
136
  const SHARE_README = `# Your IJFW Brain
@@ -1,7 +1,13 @@
1
1
  // markdown.js -- markdown + text extractor (chunk on blank-line boundaries).
2
- import { readFileSync } from 'node:fs';
2
+ import { readFileSync, statSync } from 'node:fs';
3
3
 
4
4
  const DEFAULT_MAX_CHARS = 3000;
5
+ // V155-067 (v1.5.5): default file-size cap. Inbox files come from
6
+ // user-droppable `dump/inbox/`; a multi-GB drop would OOM the dream cycle
7
+ // during `readFileSync('utf8')`. Cap at 10 MB — well above any reasonable
8
+ // hand-written brief or transcript. Caller can override (and `scanInbox`
9
+ // already records `sizeBytes` so the candidate filter can pre-skip).
10
+ export const DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024;
5
11
 
6
12
  export function chunkAtBlankLines(text, maxChars = DEFAULT_MAX_CHARS) {
7
13
  if (text.length <= maxChars) return [text];
@@ -21,7 +27,27 @@ export function chunkAtBlankLines(text, maxChars = DEFAULT_MAX_CHARS) {
21
27
  return chunks;
22
28
  }
23
29
 
24
- export function extractMarkdown(filePath, { maxChars = DEFAULT_MAX_CHARS } = {}) {
30
+ export function extractMarkdown(
31
+ filePath,
32
+ { maxChars = DEFAULT_MAX_CHARS, maxFileBytes = DEFAULT_MAX_FILE_BYTES } = {},
33
+ ) {
34
+ // V155-067 (v1.5.5): pre-stat + size cap. statSync is cheap and lets us
35
+ // surface a structured `error` for oversized files instead of hitting the
36
+ // OOM during readFileSync.
37
+ let st;
38
+ try { st = statSync(filePath); }
39
+ catch (e) {
40
+ return { text: '', chunks: [], error: 'stat-failed', message: e?.message || String(e) };
41
+ }
42
+ if (typeof maxFileBytes === 'number' && st.size > maxFileBytes) {
43
+ return {
44
+ text: '',
45
+ chunks: [],
46
+ error: 'file-too-large',
47
+ sizeBytes: st.size,
48
+ capBytes: maxFileBytes,
49
+ };
50
+ }
25
51
  const text = readFileSync(filePath, 'utf8');
26
52
  return { text, chunks: chunkAtBlankLines(text, maxChars) };
27
53
  }
@@ -1,5 +1,6 @@
1
- import { readFileSync, writeFileSync, existsSync, openSync, closeSync, unlinkSync } from 'node:fs';
1
+ import { readFileSync, writeFileSync, existsSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
+ import { withFsLock, lockPathFor } from '../fs-lock.js';
3
4
 
4
5
  export function readLayoutVersion(repoRoot) {
5
6
  const p = join(repoRoot, '.ijfw', '.layout-version');
@@ -12,18 +13,22 @@ export function writeLayoutVersion(repoRoot, version) {
12
13
  writeFileSync(join(repoRoot, '.ijfw', '.layout-version'), `${version}\n`);
13
14
  }
14
15
 
16
+ // V155-016 (HIGH): replace the bespoke `openSync('wx')` mutex with the
17
+ // canonical `withFsLock` primitive. The prior implementation had no stale
18
+ // recovery — on SIGKILL between `openSync` and the `finally` unlink, the
19
+ // lockfile orphaned forever; next server startup would throw after 5s and
20
+ // the operator had to manually `rm .ijfw/.migrate.lock`. `withFsLock`
21
+ // inherits canonical heartbeat-refreshed stale recovery (live processes
22
+ // always renew before staleMs fires; crashed holders age out cleanly).
23
+ // The .migrate.lock path sorts to LOCK_TIERS' tier-99 fallback (unknown
24
+ // path → tail of canonical order) so it cannot deadlock against numbered
25
+ // §3 locks. `timeoutMs` is preserved as `acquireTimeoutMs` so the documented
26
+ // startup-timeout behaviour is unchanged.
15
27
  export async function withLayoutLock(repoRoot, fn, { timeoutMs = 5000 } = {}) {
16
- const lockPath = join(repoRoot, '.ijfw', '.migrate.lock');
17
- const start = Date.now();
18
- let fd = null;
19
- while (true) {
20
- try { fd = openSync(lockPath, 'wx'); break; }
21
- catch (e) {
22
- if (e.code !== 'EEXIST') throw e;
23
- if (Date.now() - start > timeoutMs) throw new Error(`layout-sentinel: locked > ${timeoutMs}ms`);
24
- await new Promise(r => setTimeout(r, 25));
25
- }
26
- }
27
- try { return await fn(); }
28
- finally { try { closeSync(fd); } catch {} try { unlinkSync(lockPath); } catch {} }
28
+ const lockTarget = join(repoRoot, '.ijfw', '.migrate.lock');
29
+ return withFsLock(lockPathFor(lockTarget), fn, {
30
+ staleMs: 60_000,
31
+ heartbeatMs: 2_000,
32
+ acquireTimeoutMs: timeoutMs,
33
+ });
29
34
  }
@@ -89,11 +89,28 @@ export function validateSafeRepoPath(repoRoot, targetPath) {
89
89
  // fine — writers either overwrite or 'wx'-fail on their own. The residual
90
90
  // TOCTOU window between this check and the writer's open is microseconds
91
91
  // and requires a same-uid attacker; documented threat model.
92
+ //
93
+ // V155-055 (v1.5.5): hardlink hardening. isSymbolicLink() returns false for
94
+ // hardlinks, so a same-uid attacker can pre-create
95
+ // `<repoRoot>/.ijfw/wiki/concepts/foo.md` as a hardlink to
96
+ // `~/.ssh/authorized_keys`. writeAtomic via rename severs the hardlink
97
+ // (safe), but appendFileSync callers (budget-guard, dream-pipeline) write
98
+ // through the hardlink. We refuse any target file with nlink > 1 — under
99
+ // the brain's content tree, every file should be IJFW-owned and have nlink
100
+ // exactly 1.
92
101
  try {
93
102
  const st = lstatSync(resolved);
94
103
  if (st.isSymbolicLink()) {
95
104
  return { ok: false, error: 'outFile-is-symlink', resolved };
96
105
  }
106
+ if (st.isFile() && typeof st.nlink === 'number' && st.nlink > 1) {
107
+ return {
108
+ ok: false,
109
+ error: 'outFile-is-hardlink',
110
+ resolved,
111
+ nlink: st.nlink,
112
+ };
113
+ }
97
114
  } catch {
98
115
  // ENOENT — target doesn't exist yet; the most common case. OK to proceed.
99
116
  }
@@ -9,15 +9,20 @@
9
9
  // Wiki path: <ijfw|.ijfw>/wiki/<type>s/<slug>.md per the layout sentinel.
10
10
  // Slug is the subject lowercased + non-alphanum -> '-'.
11
11
 
12
- import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync, statSync } from 'node:fs';
12
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, statSync } from 'node:fs';
13
13
  import { join } from 'node:path';
14
14
 
15
- const STALE_LOCK_MS = 60_000; // 60s locks older than this are assumed dead-process orphans
15
+ // V155-054 (LOW): a poisoned wiki page on disk could blow up memory during
16
+ // compile via the existing-page readFileSync. Refuse compile if the prior
17
+ // page is larger than this cap.
18
+ const WIKI_PAGE_MAX_BYTES = 2 * 1024 * 1024; // 2 MB
19
+
16
20
  import { resolveBrainPaths } from './paths.js';
17
21
  import { applyTemplate } from './wiki-templates.js';
18
22
  import { resolveCitations } from './citation-resolver.js';
19
23
  import { getHistoryWindow } from '../memory/temporal.js';
20
24
  import { validateSafeRepoPath } from './path-guard.js';
25
+ import { withFsLock, lockPathFor } from '../fs-lock.js';
21
26
 
22
27
  export function slugify(s) {
23
28
  return String(s || '').toLowerCase().trim()
@@ -68,7 +73,7 @@ function querySources(db, subject) {
68
73
  } catch { return []; }
69
74
  }
70
75
 
71
- export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
76
+ export async function compileWikiPage(db, { repoRoot, type, subject } = {}) {
72
77
  if (!subject) return { ok: false, error: 'missing-subject' };
73
78
  const paths = resolveBrainPaths(repoRoot);
74
79
  const slug = slugify(subject);
@@ -83,42 +88,36 @@ export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
83
88
  const guard = validateSafeRepoPath(repoRoot, pagePath);
84
89
  if (!guard.ok) return guard;
85
90
 
86
- // F8: per-page advisory lock prevents two concurrent compiles for the
87
- // same subject from interleaving (both read existing → both render →
88
- // both rename → operator NOTES from the intermediate state could be
89
- // lost). EEXIST-based exclusive open is portable + atomic.
90
91
  mkdirSync(pageDir, { recursive: true });
91
- const lockPath = pagePath + '.lock';
92
- let lockFd;
93
- try {
94
- lockFd = openSync(lockPath, 'wx');
95
- } catch (e) {
96
- if (e.code === 'EEXIST') {
97
- // FLAG-8: stale-lock recovery. If a prior compile process was SIGKILL'd
98
- // between acquire and finally, the lockfile orphans and every subsequent
99
- // compile fails. Check the lockfile's age; if older than STALE_LOCK_MS,
100
- // assume it's a dead-process orphan and reclaim.
101
- let stale = false;
102
- try {
103
- const age = Date.now() - statSync(lockPath).mtimeMs;
104
- if (age > STALE_LOCK_MS) stale = true;
105
- } catch { /* lockfile vanished while we checked — race won by us */ stale = true; }
106
- if (stale) {
107
- try { unlinkSync(lockPath); } catch {}
108
- try { lockFd = openSync(lockPath, 'wx'); }
109
- catch (e2) { return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath, staleReclaimFailed: e2.code }; }
110
- } else {
111
- return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath };
112
- }
113
- } else {
114
- throw e;
115
- }
116
- }
117
92
 
118
- try {
93
+ // V155-015 (HIGH): per-page advisory lock now uses the canonical
94
+ // `withFsLock` primitive instead of an isolated openSync('wx') + manual
95
+ // stale recovery. The prior pattern had a stale-reclaim race — process A
96
+ // holds lock, B reads mtime >60s, B unlinks A's lockfile, B opens its own
97
+ // lock, BOTH then rename their .tmp into pagePath. `withFsLock` inherits
98
+ // heartbeat-refresh (`heartbeatMs` < `staleMs`) so a live A always renews
99
+ // before B's stale check fires; a crashed A's lock still ages out cleanly.
100
+ // The wiki-page lock sorts to LOCK_TIERS' tier-99 fallback (unknown path
101
+ // → tail of canonical order), so it never deadlocks against §3 locks.
102
+ return withFsLock(lockPathFor(pagePath), async () => {
119
103
  // Read existing AFTER acquiring the lock so we see the freshest committed
120
104
  // state (no race with another compile that just landed its rename).
121
- const existing = existsSync(pagePath) ? readFileSync(pagePath, 'utf8') : '';
105
+ let existing = '';
106
+ if (existsSync(pagePath)) {
107
+ // V155-054 (LOW): cap the existing-page size to defend against a
108
+ // poisoned wiki page that would otherwise blow up memory + LLM budget
109
+ // during compile. 2 MB is generous — real wiki pages are <100 KB.
110
+ try {
111
+ const st = statSync(pagePath);
112
+ if (st.size > WIKI_PAGE_MAX_BYTES) {
113
+ return {
114
+ ok: false, error: 'page-too-large', pagePath,
115
+ sizeBytes: st.size, maxBytes: WIKI_PAGE_MAX_BYTES,
116
+ };
117
+ }
118
+ } catch { /* statSync race — proceed with read */ }
119
+ existing = readFileSync(pagePath, 'utf8');
120
+ }
122
121
  const facts = queryFacts(db, subject);
123
122
  const history = getHistoryWindow(db, subject, null, { limit: 50 });
124
123
  const backlinks = queryBacklinks(db, slug);
@@ -137,8 +136,5 @@ export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
137
136
  renameSync(tmp, pagePath);
138
137
 
139
138
  return { ok: true, pagePath, factsCount: facts.length, historyRows: history.rows.length };
140
- } finally {
141
- try { closeSync(lockFd); } catch {}
142
- try { unlinkSync(lockPath); } catch {}
143
- }
139
+ });
144
140
  }
@@ -1,5 +1,8 @@
1
- import { existsSync, mkdirSync, readFileSync, realpathSync } from 'node:fs';
2
- import { isAbsolute, join, relative, resolve } from 'node:path';
1
+ import {
2
+ existsSync, mkdirSync, readFileSync, realpathSync,
3
+ readdirSync, unlinkSync,
4
+ } from 'node:fs';
5
+ import { basename, isAbsolute, join, relative, resolve } from 'node:path';
3
6
  import { writeAtomic } from './lib/atomic-io.js';
4
7
 
5
8
  // F-FUN-7 (audit-MED-teams-#8): role-type → Codex tool allowlist. Honors
@@ -102,12 +105,32 @@ export function syncCodexAgents(projectRoot = process.cwd(), options = {}) {
102
105
  agentFiles.push(agentPath);
103
106
  }
104
107
 
108
+ // V155-063 (v1.5.5): garbage-collect TOML files for roles that no longer
109
+ // exist in the current charter. Without this sweep, charter A (role X+Y)
110
+ // → charter B (role X only) leaves `<safeAgentsDir>/y.toml` on disk and
111
+ // Codex continues to pick up the phantom agent. We compare the dir
112
+ // listing against the set of just-written agentFiles and unlink any
113
+ // `.toml` whose basename isn't claimed.
114
+ const removed = [];
115
+ const expected = new Set(agentFiles.map((p) => basename(p)));
116
+ let entries = [];
117
+ try { entries = readdirSync(safeAgentsDir); } catch { /* best-effort */ }
118
+ for (const name of entries) {
119
+ if (!name.endsWith('.toml')) continue;
120
+ if (expected.has(name)) continue;
121
+ const p = join(safeAgentsDir, name);
122
+ try { unlinkSync(p); removed.push(p); } catch { /* best-effort */ }
123
+ }
124
+
105
125
  return {
106
126
  ok: true,
107
127
  agentsDir: safeAgentsDir,
108
128
  agentFiles,
109
129
  count: agentFiles.length,
110
130
  skipped,
131
+ // Only surface `removed` when something was GCed so the common path stays
132
+ // terse.
133
+ ...(removed.length > 0 ? { removed } : {}),
111
134
  };
112
135
  }
113
136