@ijfw/memory-server 1.5.4 → 1.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +15 -1
- package/src/brain/dream-pipeline.js +77 -14
- package/src/brain/dump-ingest.js +32 -0
- package/src/brain/entity-collapse.js +2 -2
- package/src/brain/export.js +60 -6
- package/src/brain/extractors/markdown.js +28 -2
- package/src/brain/layout-sentinel.js +19 -14
- package/src/brain/path-guard.js +17 -0
- package/src/brain/wiki-compiler.js +35 -39
- package/src/codex-agents.js +25 -2
- package/src/cross-orchestrator-cli.js +176 -18
- package/src/dashboard-server.js +36 -3
- package/src/dispatch/override.js +18 -2
- package/src/dispatch/signer-cli.js +14 -9
- package/src/dream/stage-runner.js +17 -0
- package/src/dream/state-file.js +15 -1
- package/src/extension-installer.js +91 -2
- package/src/extension-registry.js +15 -4
- package/src/handlers/brain-handler.js +44 -5
- package/src/lib/atomic-io.js +69 -12
- package/src/lib/shasum-verify.js +46 -22
- package/src/lib/ui-review-runner.js +7 -2
- package/src/lib/uispec-drift.js +8 -3
- package/src/lib/uispec-intake.js +5 -2
- package/src/memory/layout-migrations/001-visible-layer.js +71 -7
- package/src/memory/reader.js +111 -58
- package/src/orchestrator/merge-block-aware.js +75 -37
- package/src/orchestrator/post-done-runner.js +6 -1
- package/src/orchestrator/state-sdk.js +242 -14
- package/src/orchestrator/wave-state.js +22 -69
- package/src/recovery/checkpoint.js +30 -6
- package/src/recovery/code-fixer.js +52 -7
- package/src/runtime-mediator.js +2 -2
- package/src/server.js +57 -8
- package/src/swarm/planner.js +46 -1
- package/src/update-apply.js +27 -35
- package/src/update-check.js +6 -2
package/package.json
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ijfw/memory-server",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.6",
|
|
4
4
|
"description": "Cross-platform persistent memory server for IJFW. 14 MCP tools (memory + admin/update + brain). Works with 15 platforms: 14 via MCP (Claude Code, Codex, Gemini CLI, Cursor, Windsurf, Copilot, Hermes, Wayland, OpenCode, QwenCode, Cline, KimiCode, OpenClaw, Antigravity) plus Aider via the rules-only tier.",
|
|
5
5
|
"author": "Sean Donahoe",
|
|
6
|
+
"contributors": [
|
|
7
|
+
{
|
|
8
|
+
"name": "Ferrox Labs",
|
|
9
|
+
"url": "https://ferroxlabs.com"
|
|
10
|
+
}
|
|
11
|
+
],
|
|
6
12
|
"license": "MIT",
|
|
13
|
+
"homepage": "https://github.com/FerroxLabs/ijfw",
|
|
14
|
+
"bugs": {
|
|
15
|
+
"url": "https://github.com/FerroxLabs/ijfw/issues"
|
|
16
|
+
},
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/FerroxLabs/ijfw.git"
|
|
20
|
+
},
|
|
7
21
|
"type": "module",
|
|
8
22
|
"main": "src/server.js",
|
|
9
23
|
"bin": {
|
|
@@ -16,8 +16,10 @@
|
|
|
16
16
|
// Budget: every LLM call goes through BudgetGuard. budgetExhausted=true
|
|
17
17
|
// signals the cycle stopped voluntarily (not crashed).
|
|
18
18
|
|
|
19
|
-
import {
|
|
20
|
-
|
|
19
|
+
import {
|
|
20
|
+
mkdirSync, existsSync, openSync, writeSync, closeSync, constants as fsConstants,
|
|
21
|
+
} from 'node:fs';
|
|
22
|
+
import { join, dirname } from 'node:path';
|
|
21
23
|
import { resolveBrainPaths } from './paths.js';
|
|
22
24
|
import { scanInbox, writeManifest, commitProcessed, isProcessed } from './dump-ingest.js';
|
|
23
25
|
import { extractFile } from './extractors/index.js';
|
|
@@ -25,6 +27,7 @@ import { BudgetGuard } from './budget-guard.js';
|
|
|
25
27
|
import { compileWikiPage } from './wiki-compiler.js';
|
|
26
28
|
import { callTiered } from './tiered-llm.js';
|
|
27
29
|
import { validateSafeRepoPath } from './path-guard.js';
|
|
30
|
+
import { withFsLock, lockPathFor } from '../fs-lock.js';
|
|
28
31
|
|
|
29
32
|
function ensureFactsTable(db) {
|
|
30
33
|
// Idempotent: matches the schema downstream consumers expect.
|
|
@@ -41,18 +44,33 @@ function ensureFactsTable(db) {
|
|
|
41
44
|
function appendLog(wikiLogPath, line, repoRoot) {
|
|
42
45
|
try {
|
|
43
46
|
// F-LENS2-05/11: refuse to follow a symlinked log path out of the repo.
|
|
44
|
-
// lstat (NOT stat) so we see the symlink itself; if the path is a
|
|
45
|
-
// symlink, drop the append rather than write through it.
|
|
46
47
|
if (repoRoot) {
|
|
47
48
|
const guard = validateSafeRepoPath(repoRoot, wikiLogPath);
|
|
48
49
|
if (!guard.ok) return;
|
|
49
50
|
}
|
|
50
|
-
try {
|
|
51
|
-
const lst = lstatSync(wikiLogPath);
|
|
52
|
-
if (lst.isSymbolicLink()) return; // F-LENS2-11: refuse symlink follow
|
|
53
|
-
} catch { /* file doesn't exist yet — ok to create */ }
|
|
54
51
|
mkdirSync(dirname(wikiLogPath), { recursive: true });
|
|
55
|
-
|
|
52
|
+
// V155-039 (MED): the prior `lstatSync` → `appendFileSync` sequence had
|
|
53
|
+
// a TOCTOU window — an attacker could swap a regular file for a symlink
|
|
54
|
+
// between the two syscalls so the write follows the new symlink (e.g.
|
|
55
|
+
// to /etc/passwd). `O_NOFOLLOW` makes `openSync` fail atomically if the
|
|
56
|
+
// FINAL path component is a symlink, closing the race with a single
|
|
57
|
+
// syscall. `O_APPEND` keeps append semantics; `O_CREAT` creates on
|
|
58
|
+
// first write. Mode 0o600 keeps log private.
|
|
59
|
+
let fd;
|
|
60
|
+
try {
|
|
61
|
+
// eslint-disable-next-line no-bitwise
|
|
62
|
+
fd = openSync(
|
|
63
|
+
wikiLogPath,
|
|
64
|
+
fsConstants.O_APPEND | fsConstants.O_CREAT | fsConstants.O_WRONLY | fsConstants.O_NOFOLLOW,
|
|
65
|
+
0o600,
|
|
66
|
+
);
|
|
67
|
+
} catch {
|
|
68
|
+
// ELOOP / EACCES → refuse to write. No fallback path — logging is
|
|
69
|
+
// best-effort and a symlinked log path is a refusal, not a degrade.
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
try { writeSync(fd, line + '\n'); }
|
|
73
|
+
finally { try { closeSync(fd); } catch { /* best-effort */ } }
|
|
56
74
|
} catch { /* logging is best-effort */ }
|
|
57
75
|
}
|
|
58
76
|
|
|
@@ -179,6 +197,23 @@ export async function defaultExtractFacts({ file: _file, text, chunks, env, guar
|
|
|
179
197
|
|
|
180
198
|
function nowIso() { return new Date().toISOString(); }
|
|
181
199
|
|
|
200
|
+
// V155-020 (HIGH): facts-table source check. `isProcessed(processed, name)`
|
|
201
|
+
// only inspects the manifest file — `rm -rf <processed>/` looks like a
|
|
202
|
+
// clean slate to the gate, so re-ingest doubles every triple (cycle 2 = 2x,
|
|
203
|
+
// cycle 3 = 3x, …). Cross-check the facts table: if ANY fact already
|
|
204
|
+
// references this file as its `source`, treat it as processed regardless
|
|
205
|
+
// of manifest presence. Cleaner than adding a UNIQUE constraint (no
|
|
206
|
+
// migration required); the source column already stores file.name.
|
|
207
|
+
function isProcessedDouble(db, processedDir, fileName) {
|
|
208
|
+
if (isProcessed(processedDir, fileName)) return true;
|
|
209
|
+
try {
|
|
210
|
+
const row = db.prepare(
|
|
211
|
+
'SELECT 1 FROM facts WHERE source = ? LIMIT 1'
|
|
212
|
+
).get(fileName);
|
|
213
|
+
return !!row;
|
|
214
|
+
} catch { return false; }
|
|
215
|
+
}
|
|
216
|
+
|
|
182
217
|
export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId, extractFacts } = {}) {
|
|
183
218
|
if (!db) throw new Error('dream-pipeline: db required');
|
|
184
219
|
if (!repoRoot) throw new Error('dream-pipeline: repoRoot required');
|
|
@@ -195,6 +230,32 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
|
|
|
195
230
|
const guard = BudgetGuard(guardOpts);
|
|
196
231
|
const extractor = extractFacts || defaultExtractFacts;
|
|
197
232
|
|
|
233
|
+
// V155-064 (LOW): gate the mkdirSyncs on existsSync so the syscalls don't
|
|
234
|
+
// fire on every cycle (cheap-and-correct today, but if a future watcher
|
|
235
|
+
// layers fs-events on those dirs the mkdir wakeups become hot-path noise).
|
|
236
|
+
// `mkdir … recursive:true` is idempotent at the kernel layer; this is a
|
|
237
|
+
// userspace short-circuit.
|
|
238
|
+
if (!existsSync(paths.dumpInbox)) mkdirSync(paths.dumpInbox, { recursive: true });
|
|
239
|
+
if (!existsSync(paths.dumpProcessed)) mkdirSync(paths.dumpProcessed, { recursive: true });
|
|
240
|
+
|
|
241
|
+
// V155-038 (MED): project-scope lock around the entire run. Two
|
|
242
|
+
// simultaneous `ijfw_brain dream` triggers (manual + chokidar; or two
|
|
243
|
+
// operators on the same repo) would otherwise BOTH scan the same inbox,
|
|
244
|
+
// BOTH call the (paid) LLM extractor, and BOTH run the rename race for
|
|
245
|
+
// `commitProcessed` — the loser silently catches ENOENT but its facts
|
|
246
|
+
// already landed (no UNIQUE constraint on facts). Net: double-extract
|
|
247
|
+
// cost + duplicate facts + single committed file. The lock makes a
|
|
248
|
+
// concurrent invocation wait for the first to finish; the second then
|
|
249
|
+
// sees the manifests (and V155-020 facts-table cross-check) and exits
|
|
250
|
+
// with zero candidates.
|
|
251
|
+
const lockTarget = join(repoRoot, '.ijfw', 'state', '.dream-cycle.lock');
|
|
252
|
+
if (!existsSync(dirname(lockTarget))) mkdirSync(dirname(lockTarget), { recursive: true });
|
|
253
|
+
return withFsLock(lockPathFor(lockTarget), async () => runDreamCycleLocked({
|
|
254
|
+
db, repoRoot, env, paths, cid, guard, extractor,
|
|
255
|
+
}));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function runDreamCycleLocked({ db, repoRoot, env, paths, cid, guard, extractor }) {
|
|
198
259
|
let processed = 0;
|
|
199
260
|
let factsInserted = 0;
|
|
200
261
|
let pagesCompiled = 0;
|
|
@@ -202,11 +263,11 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
|
|
|
202
263
|
const touchedSubjects = new Set();
|
|
203
264
|
const errors = [];
|
|
204
265
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
266
|
+
// V155-020: cross-check the facts table so a `rm -rf <processed>/` cannot
|
|
267
|
+
// trigger re-ingest doubling. The manifest gate stays as the cheap first
|
|
268
|
+
// check; only filename-collision survivors get the DB query.
|
|
208
269
|
const candidates = scanInbox(paths.dumpInbox).filter(
|
|
209
|
-
(f) => !
|
|
270
|
+
(f) => !isProcessedDouble(db, paths.dumpProcessed, f.name)
|
|
210
271
|
);
|
|
211
272
|
|
|
212
273
|
for (const file of candidates) {
|
|
@@ -310,8 +371,10 @@ export async function runDreamCycle({ db, repoRoot, env = process.env, cycleId,
|
|
|
310
371
|
}
|
|
311
372
|
|
|
312
373
|
// Compile pages for touched subjects. Failures are logged, not fatal.
|
|
374
|
+
// V155-015: compileWikiPage is now async (withFsLock primitive).
|
|
313
375
|
for (const subject of touchedSubjects) {
|
|
314
|
-
|
|
376
|
+
// eslint-disable-next-line no-await-in-loop
|
|
377
|
+
const r = await compileWikiPage(db, { repoRoot, type: 'entity', subject });
|
|
315
378
|
if (r.ok) {
|
|
316
379
|
pagesCompiled += 1;
|
|
317
380
|
appendLog(paths.wikiLog, `${nowIso()} compile entity ${subject} facts=${r.factsCount}`, repoRoot);
|
package/src/brain/dump-ingest.js
CHANGED
|
@@ -79,10 +79,42 @@ export function isProcessed(processedDir, fileName) {
|
|
|
79
79
|
return existsSync(manifestPath(processedDir, fileName));
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
// readManifest: legacy behaviour — returns the parsed JSON payload directly,
|
|
83
|
+
// throws on missing/unreadable/parse-fail. Callers in tests + E2E paths rely
|
|
84
|
+
// on the direct round-trip shape; do NOT break the contract.
|
|
85
|
+
//
|
|
86
|
+
// V155-068 (v1.5.5): added `readManifestSafe` (below) for callers — like the
|
|
87
|
+
// dream cycle's manifest-reprocess logic — that need to treat
|
|
88
|
+
// corrupt/absent as a graceful "needs reprocess" signal rather than letting
|
|
89
|
+
// JSON.parse abort the whole cycle.
|
|
82
90
|
export function readManifest(processedDir, fileName) {
|
|
83
91
|
return JSON.parse(readFileSync(manifestPath(processedDir, fileName), 'utf8'));
|
|
84
92
|
}
|
|
85
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Tolerant variant of readManifest for callers that need to distinguish:
|
|
96
|
+
* - { ok: true, data } — clean read
|
|
97
|
+
* - { ok: false, code: 'enoent' } — manifest absent
|
|
98
|
+
* - { ok: false, code: 'unreadable', message } — readFile threw
|
|
99
|
+
* - { ok: false, code: 'parse-fail', message, path } — JSON.parse threw
|
|
100
|
+
*
|
|
101
|
+
* V155-068 (v1.5.5): closes the dream-cycle crash on tampered manifests.
|
|
102
|
+
*/
|
|
103
|
+
export function readManifestSafe(processedDir, fileName) {
|
|
104
|
+
const p = manifestPath(processedDir, fileName);
|
|
105
|
+
if (!existsSync(p)) return { ok: false, code: 'enoent' };
|
|
106
|
+
let raw;
|
|
107
|
+
try { raw = readFileSync(p, 'utf8'); }
|
|
108
|
+
catch (e) {
|
|
109
|
+
return { ok: false, code: 'unreadable', message: e?.message || String(e) };
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
return { ok: true, data: JSON.parse(raw) };
|
|
113
|
+
} catch (e) {
|
|
114
|
+
return { ok: false, code: 'parse-fail', message: e?.message || String(e), path: p };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
86
118
|
function manifestPath(processedDir, fileName) {
|
|
87
119
|
return join(processedDir, `${fileName}.manifest.json`);
|
|
88
120
|
}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
//
|
|
3
3
|
// canonicalize(s): lowercase + collapsed-whitespace + trim. findCandidateMerges(db)
|
|
4
4
|
// surfaces groups of distinct stored subject strings that normalize to the same
|
|
5
|
-
// form (e.g. "
|
|
6
|
-
// to "
|
|
5
|
+
// form (e.g. "Jane Doe" / "jane doe" / " Jane Doe " all collapse
|
|
6
|
+
// to "jane doe"). Promotion (actual merge) is operator-confirmed -- this
|
|
7
7
|
// module only surfaces candidates.
|
|
8
8
|
|
|
9
9
|
export function canonicalize(s) {
|
package/src/brain/export.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// Both functions tolerate the v1->v2 layout transition: they consult the
|
|
13
13
|
// layout sentinel via resolveBrainPaths to pick the right wiki location.
|
|
14
14
|
|
|
15
|
-
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
15
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from 'node:fs';
|
|
16
16
|
import { dirname, join } from 'node:path';
|
|
17
17
|
import { resolveBrainPaths } from './paths.js';
|
|
18
18
|
import { validateSafeRepoPath } from './path-guard.js';
|
|
@@ -47,6 +47,34 @@ function parseWikilinks(md) {
|
|
|
47
47
|
return [...out];
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
// V155-034 (v1.5.5): export size caps. Wiki pages are written by the dream
|
|
51
|
+
// pipeline from user-droppable `dump/inbox/` content, so a poisoned dump can
|
|
52
|
+
// produce a page with `[[link1]]...[[linkN]]` plus a few 100MB linked pages.
|
|
53
|
+
// Without caps, `wiki.export` concatenates everything into memory and OOMs the
|
|
54
|
+
// MCP server. The caps bound:
|
|
55
|
+
// - any single page (root or linked) at MAX_PAGE_BYTES
|
|
56
|
+
// - the total bundle at MAX_BUNDLE_BYTES
|
|
57
|
+
// Truncated pages are replaced with a sentinel header so the consumer can
|
|
58
|
+
// tell the export was capped, not silently empty.
|
|
59
|
+
const MAX_PAGE_BYTES = 2 * 1024 * 1024; // 2 MB per page
|
|
60
|
+
const MAX_BUNDLE_BYTES = 10 * 1024 * 1024; // 10 MB per bundle
|
|
61
|
+
|
|
62
|
+
function readPageCapped(filePath) {
|
|
63
|
+
try {
|
|
64
|
+
const st = statSync(filePath);
|
|
65
|
+
if (st.size > MAX_PAGE_BYTES) {
|
|
66
|
+
return {
|
|
67
|
+
truncated: true,
|
|
68
|
+
sizeBytes: st.size,
|
|
69
|
+
body: `[... truncated — page is ${(st.size / 1024 / 1024).toFixed(2)} MB; export cap is ${(MAX_PAGE_BYTES / 1024 / 1024).toFixed(0)} MB ...]`,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return { truncated: false, sizeBytes: st.size, body: readFileSync(filePath, 'utf8') };
|
|
73
|
+
} catch (e) {
|
|
74
|
+
return { truncated: false, error: e?.message || String(e), body: null };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
50
78
|
export function exportPageBundle(repoRoot, slug, outFile) {
|
|
51
79
|
// F-LENS2-05 defense-in-depth: brain-handler already validates outFile via
|
|
52
80
|
// validateSafeRepoPath, but exportPageBundle is also exported and may be
|
|
@@ -57,26 +85,52 @@ export function exportPageBundle(repoRoot, slug, outFile) {
|
|
|
57
85
|
const paths = resolveBrainPaths(repoRoot);
|
|
58
86
|
const root = findPage(paths.wikiDir, slug);
|
|
59
87
|
if (!root) return { error: 'page-not-found', slug };
|
|
60
|
-
const
|
|
88
|
+
const rootRead = readPageCapped(root.path);
|
|
89
|
+
if (rootRead.body == null) {
|
|
90
|
+
return { error: 'root-page-unreadable', slug, message: rootRead.error };
|
|
91
|
+
}
|
|
92
|
+
const rootBody = rootRead.body;
|
|
61
93
|
const linked = parseWikilinks(rootBody);
|
|
62
94
|
|
|
63
95
|
const parts = [
|
|
64
96
|
`# Export: ${slug}\n\n_Generated by IJFW brain export from ${root.type}/${slug}.md._\n`,
|
|
65
97
|
`## ${slug}\n\n${rootBody.trim()}`,
|
|
66
98
|
];
|
|
99
|
+
let runningBytes = Buffer.byteLength(parts.join('\n\n'), 'utf8');
|
|
67
100
|
const includedLinks = [];
|
|
101
|
+
const truncatedLinks = [];
|
|
102
|
+
const skippedLinks = [];
|
|
103
|
+
let bundleTruncated = false;
|
|
68
104
|
for (const target of linked) {
|
|
105
|
+
if (runningBytes >= MAX_BUNDLE_BYTES) {
|
|
106
|
+
bundleTruncated = true;
|
|
107
|
+
skippedLinks.push(target);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
69
110
|
const found = findPage(paths.wikiDir, target);
|
|
70
111
|
if (!found) continue;
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
112
|
+
const r = readPageCapped(found.path);
|
|
113
|
+
if (r.body == null) continue;
|
|
114
|
+
const block = `### ${target}\n\n${r.body.trim()}`;
|
|
115
|
+
runningBytes += Buffer.byteLength(block, 'utf8') + 2; // approx '\n\n'
|
|
116
|
+
parts.push(block);
|
|
74
117
|
includedLinks.push(target);
|
|
118
|
+
if (r.truncated) truncatedLinks.push(target);
|
|
75
119
|
}
|
|
120
|
+
if (rootRead.truncated && !truncatedLinks.includes(slug)) truncatedLinks.unshift(slug);
|
|
121
|
+
|
|
76
122
|
const out = parts.join('\n\n') + '\n';
|
|
77
123
|
mkdirSync(dirname(outFile), { recursive: true });
|
|
78
124
|
writeFileSync(outFile, out);
|
|
79
|
-
return {
|
|
125
|
+
return {
|
|
126
|
+
outFile,
|
|
127
|
+
bytes: Buffer.byteLength(out, 'utf8'),
|
|
128
|
+
linkedPagesIncluded: includedLinks,
|
|
129
|
+
// Diagnostic fields — only present when caps actually fired so the common
|
|
130
|
+
// case stays terse.
|
|
131
|
+
...(truncatedLinks.length > 0 ? { truncatedPages: truncatedLinks } : {}),
|
|
132
|
+
...(bundleTruncated ? { bundleTruncated: true, skippedLinks } : {}),
|
|
133
|
+
};
|
|
80
134
|
}
|
|
81
135
|
|
|
82
136
|
const SHARE_README = `# Your IJFW Brain
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
// markdown.js -- markdown + text extractor (chunk on blank-line boundaries).
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { readFileSync, statSync } from 'node:fs';
|
|
3
3
|
|
|
4
4
|
const DEFAULT_MAX_CHARS = 3000;
|
|
5
|
+
// V155-067 (v1.5.5): default file-size cap. Inbox files come from
|
|
6
|
+
// user-droppable `dump/inbox/`; a multi-GB drop would OOM the dream cycle
|
|
7
|
+
// during `readFileSync('utf8')`. Cap at 10 MB — well above any reasonable
|
|
8
|
+
// hand-written brief or transcript. Caller can override (and `scanInbox`
|
|
9
|
+
// already records `sizeBytes` so the candidate filter can pre-skip).
|
|
10
|
+
export const DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
5
11
|
|
|
6
12
|
export function chunkAtBlankLines(text, maxChars = DEFAULT_MAX_CHARS) {
|
|
7
13
|
if (text.length <= maxChars) return [text];
|
|
@@ -21,7 +27,27 @@ export function chunkAtBlankLines(text, maxChars = DEFAULT_MAX_CHARS) {
|
|
|
21
27
|
return chunks;
|
|
22
28
|
}
|
|
23
29
|
|
|
24
|
-
export function extractMarkdown(
|
|
30
|
+
export function extractMarkdown(
|
|
31
|
+
filePath,
|
|
32
|
+
{ maxChars = DEFAULT_MAX_CHARS, maxFileBytes = DEFAULT_MAX_FILE_BYTES } = {},
|
|
33
|
+
) {
|
|
34
|
+
// V155-067 (v1.5.5): pre-stat + size cap. statSync is cheap and lets us
|
|
35
|
+
// surface a structured `error` for oversized files instead of hitting the
|
|
36
|
+
// OOM during readFileSync.
|
|
37
|
+
let st;
|
|
38
|
+
try { st = statSync(filePath); }
|
|
39
|
+
catch (e) {
|
|
40
|
+
return { text: '', chunks: [], error: 'stat-failed', message: e?.message || String(e) };
|
|
41
|
+
}
|
|
42
|
+
if (typeof maxFileBytes === 'number' && st.size > maxFileBytes) {
|
|
43
|
+
return {
|
|
44
|
+
text: '',
|
|
45
|
+
chunks: [],
|
|
46
|
+
error: 'file-too-large',
|
|
47
|
+
sizeBytes: st.size,
|
|
48
|
+
capBytes: maxFileBytes,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
25
51
|
const text = readFileSync(filePath, 'utf8');
|
|
26
52
|
return { text, chunks: chunkAtBlankLines(text, maxChars) };
|
|
27
53
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, existsSync
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { withFsLock, lockPathFor } from '../fs-lock.js';
|
|
3
4
|
|
|
4
5
|
export function readLayoutVersion(repoRoot) {
|
|
5
6
|
const p = join(repoRoot, '.ijfw', '.layout-version');
|
|
@@ -12,18 +13,22 @@ export function writeLayoutVersion(repoRoot, version) {
|
|
|
12
13
|
writeFileSync(join(repoRoot, '.ijfw', '.layout-version'), `${version}\n`);
|
|
13
14
|
}
|
|
14
15
|
|
|
16
|
+
// V155-016 (HIGH): replace the bespoke `openSync('wx')` mutex with the
|
|
17
|
+
// canonical `withFsLock` primitive. The prior implementation had no stale
|
|
18
|
+
// recovery — on SIGKILL between `openSync` and the `finally` unlink, the
|
|
19
|
+
// lockfile orphaned forever; next server startup would throw after 5s and
|
|
20
|
+
// the operator had to manually `rm .ijfw/.migrate.lock`. `withFsLock`
|
|
21
|
+
// inherits canonical heartbeat-refreshed stale recovery (live processes
|
|
22
|
+
// always renew before staleMs fires; crashed holders age out cleanly).
|
|
23
|
+
// The .migrate.lock path sorts to LOCK_TIERS' tier-99 fallback (unknown
|
|
24
|
+
// path → tail of canonical order) so it cannot deadlock against numbered
|
|
25
|
+
// §3 locks. `timeoutMs` is preserved as `acquireTimeoutMs` so the documented
|
|
26
|
+
// startup-timeout behaviour is unchanged.
|
|
15
27
|
export async function withLayoutLock(repoRoot, fn, { timeoutMs = 5000 } = {}) {
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
if (e.code !== 'EEXIST') throw e;
|
|
23
|
-
if (Date.now() - start > timeoutMs) throw new Error(`layout-sentinel: locked > ${timeoutMs}ms`);
|
|
24
|
-
await new Promise(r => setTimeout(r, 25));
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
try { return await fn(); }
|
|
28
|
-
finally { try { closeSync(fd); } catch {} try { unlinkSync(lockPath); } catch {} }
|
|
28
|
+
const lockTarget = join(repoRoot, '.ijfw', '.migrate.lock');
|
|
29
|
+
return withFsLock(lockPathFor(lockTarget), fn, {
|
|
30
|
+
staleMs: 60_000,
|
|
31
|
+
heartbeatMs: 2_000,
|
|
32
|
+
acquireTimeoutMs: timeoutMs,
|
|
33
|
+
});
|
|
29
34
|
}
|
package/src/brain/path-guard.js
CHANGED
|
@@ -89,11 +89,28 @@ export function validateSafeRepoPath(repoRoot, targetPath) {
|
|
|
89
89
|
// fine — writers either overwrite or 'wx'-fail on their own. The residual
|
|
90
90
|
// TOCTOU window between this check and the writer's open is microseconds
|
|
91
91
|
// and requires a same-uid attacker; documented threat model.
|
|
92
|
+
//
|
|
93
|
+
// V155-055 (v1.5.5): hardlink hardening. isSymbolicLink() returns false for
|
|
94
|
+
// hardlinks, so a same-uid attacker can pre-create
|
|
95
|
+
// `<repoRoot>/.ijfw/wiki/concepts/foo.md` as a hardlink to
|
|
96
|
+
// `~/.ssh/authorized_keys`. writeAtomic via rename severs the hardlink
|
|
97
|
+
// (safe), but appendFileSync callers (budget-guard, dream-pipeline) write
|
|
98
|
+
// through the hardlink. We refuse any target file with nlink > 1 — under
|
|
99
|
+
// the brain's content tree, every file should be IJFW-owned and have nlink
|
|
100
|
+
// exactly 1.
|
|
92
101
|
try {
|
|
93
102
|
const st = lstatSync(resolved);
|
|
94
103
|
if (st.isSymbolicLink()) {
|
|
95
104
|
return { ok: false, error: 'outFile-is-symlink', resolved };
|
|
96
105
|
}
|
|
106
|
+
if (st.isFile() && typeof st.nlink === 'number' && st.nlink > 1) {
|
|
107
|
+
return {
|
|
108
|
+
ok: false,
|
|
109
|
+
error: 'outFile-is-hardlink',
|
|
110
|
+
resolved,
|
|
111
|
+
nlink: st.nlink,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
97
114
|
} catch {
|
|
98
115
|
// ENOENT — target doesn't exist yet; the most common case. OK to proceed.
|
|
99
116
|
}
|
|
@@ -9,15 +9,20 @@
|
|
|
9
9
|
// Wiki path: <ijfw|.ijfw>/wiki/<type>s/<slug>.md per the layout sentinel.
|
|
10
10
|
// Slug is the subject lowercased + non-alphanum -> '-'.
|
|
11
11
|
|
|
12
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync,
|
|
12
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, statSync } from 'node:fs';
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
// V155-054 (LOW): a poisoned wiki page on disk could blow up memory during
|
|
16
|
+
// compile via the existing-page readFileSync. Refuse compile if the prior
|
|
17
|
+
// page is larger than this cap.
|
|
18
|
+
const WIKI_PAGE_MAX_BYTES = 2 * 1024 * 1024; // 2 MB
|
|
19
|
+
|
|
16
20
|
import { resolveBrainPaths } from './paths.js';
|
|
17
21
|
import { applyTemplate } from './wiki-templates.js';
|
|
18
22
|
import { resolveCitations } from './citation-resolver.js';
|
|
19
23
|
import { getHistoryWindow } from '../memory/temporal.js';
|
|
20
24
|
import { validateSafeRepoPath } from './path-guard.js';
|
|
25
|
+
import { withFsLock, lockPathFor } from '../fs-lock.js';
|
|
21
26
|
|
|
22
27
|
export function slugify(s) {
|
|
23
28
|
return String(s || '').toLowerCase().trim()
|
|
@@ -68,7 +73,7 @@ function querySources(db, subject) {
|
|
|
68
73
|
} catch { return []; }
|
|
69
74
|
}
|
|
70
75
|
|
|
71
|
-
export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
|
|
76
|
+
export async function compileWikiPage(db, { repoRoot, type, subject } = {}) {
|
|
72
77
|
if (!subject) return { ok: false, error: 'missing-subject' };
|
|
73
78
|
const paths = resolveBrainPaths(repoRoot);
|
|
74
79
|
const slug = slugify(subject);
|
|
@@ -83,42 +88,36 @@ export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
|
|
|
83
88
|
const guard = validateSafeRepoPath(repoRoot, pagePath);
|
|
84
89
|
if (!guard.ok) return guard;
|
|
85
90
|
|
|
86
|
-
// F8: per-page advisory lock prevents two concurrent compiles for the
|
|
87
|
-
// same subject from interleaving (both read existing → both render →
|
|
88
|
-
// both rename → operator NOTES from the intermediate state could be
|
|
89
|
-
// lost). EEXIST-based exclusive open is portable + atomic.
|
|
90
91
|
mkdirSync(pageDir, { recursive: true });
|
|
91
|
-
const lockPath = pagePath + '.lock';
|
|
92
|
-
let lockFd;
|
|
93
|
-
try {
|
|
94
|
-
lockFd = openSync(lockPath, 'wx');
|
|
95
|
-
} catch (e) {
|
|
96
|
-
if (e.code === 'EEXIST') {
|
|
97
|
-
// FLAG-8: stale-lock recovery. If a prior compile process was SIGKILL'd
|
|
98
|
-
// between acquire and finally, the lockfile orphans and every subsequent
|
|
99
|
-
// compile fails. Check the lockfile's age; if older than STALE_LOCK_MS,
|
|
100
|
-
// assume it's a dead-process orphan and reclaim.
|
|
101
|
-
let stale = false;
|
|
102
|
-
try {
|
|
103
|
-
const age = Date.now() - statSync(lockPath).mtimeMs;
|
|
104
|
-
if (age > STALE_LOCK_MS) stale = true;
|
|
105
|
-
} catch { /* lockfile vanished while we checked — race won by us */ stale = true; }
|
|
106
|
-
if (stale) {
|
|
107
|
-
try { unlinkSync(lockPath); } catch {}
|
|
108
|
-
try { lockFd = openSync(lockPath, 'wx'); }
|
|
109
|
-
catch (e2) { return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath, staleReclaimFailed: e2.code }; }
|
|
110
|
-
} else {
|
|
111
|
-
return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath };
|
|
112
|
-
}
|
|
113
|
-
} else {
|
|
114
|
-
throw e;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
92
|
|
|
118
|
-
|
|
93
|
+
// V155-015 (HIGH): per-page advisory lock now uses the canonical
|
|
94
|
+
// `withFsLock` primitive instead of an isolated openSync('wx') + manual
|
|
95
|
+
// stale recovery. The prior pattern had a stale-reclaim race — process A
|
|
96
|
+
// holds lock, B reads mtime >60s, B unlinks A's lockfile, B opens its own
|
|
97
|
+
// lock, BOTH then rename their .tmp into pagePath. `withFsLock` inherits
|
|
98
|
+
// heartbeat-refresh (`heartbeatMs` < `staleMs`) so a live A always renews
|
|
99
|
+
// before B's stale check fires; a crashed A's lock still ages out cleanly.
|
|
100
|
+
// The wiki-page lock sorts to LOCK_TIERS' tier-99 fallback (unknown path
|
|
101
|
+
// → tail of canonical order), so it never deadlocks against §3 locks.
|
|
102
|
+
return withFsLock(lockPathFor(pagePath), async () => {
|
|
119
103
|
// Read existing AFTER acquiring the lock so we see the freshest committed
|
|
120
104
|
// state (no race with another compile that just landed its rename).
|
|
121
|
-
|
|
105
|
+
let existing = '';
|
|
106
|
+
if (existsSync(pagePath)) {
|
|
107
|
+
// V155-054 (LOW): cap the existing-page size to defend against a
|
|
108
|
+
// poisoned wiki page that would otherwise blow up memory + LLM budget
|
|
109
|
+
// during compile. 2 MB is generous — real wiki pages are <100 KB.
|
|
110
|
+
try {
|
|
111
|
+
const st = statSync(pagePath);
|
|
112
|
+
if (st.size > WIKI_PAGE_MAX_BYTES) {
|
|
113
|
+
return {
|
|
114
|
+
ok: false, error: 'page-too-large', pagePath,
|
|
115
|
+
sizeBytes: st.size, maxBytes: WIKI_PAGE_MAX_BYTES,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
} catch { /* statSync race — proceed with read */ }
|
|
119
|
+
existing = readFileSync(pagePath, 'utf8');
|
|
120
|
+
}
|
|
122
121
|
const facts = queryFacts(db, subject);
|
|
123
122
|
const history = getHistoryWindow(db, subject, null, { limit: 50 });
|
|
124
123
|
const backlinks = queryBacklinks(db, slug);
|
|
@@ -137,8 +136,5 @@ export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
|
|
|
137
136
|
renameSync(tmp, pagePath);
|
|
138
137
|
|
|
139
138
|
return { ok: true, pagePath, factsCount: facts.length, historyRows: history.rows.length };
|
|
140
|
-
}
|
|
141
|
-
try { closeSync(lockFd); } catch {}
|
|
142
|
-
try { unlinkSync(lockPath); } catch {}
|
|
143
|
-
}
|
|
139
|
+
});
|
|
144
140
|
}
|
package/src/codex-agents.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
existsSync, mkdirSync, readFileSync, realpathSync,
|
|
3
|
+
readdirSync, unlinkSync,
|
|
4
|
+
} from 'node:fs';
|
|
5
|
+
import { basename, isAbsolute, join, relative, resolve } from 'node:path';
|
|
3
6
|
import { writeAtomic } from './lib/atomic-io.js';
|
|
4
7
|
|
|
5
8
|
// F-FUN-7 (audit-MED-teams-#8): role-type → Codex tool allowlist. Honors
|
|
@@ -102,12 +105,32 @@ export function syncCodexAgents(projectRoot = process.cwd(), options = {}) {
|
|
|
102
105
|
agentFiles.push(agentPath);
|
|
103
106
|
}
|
|
104
107
|
|
|
108
|
+
// V155-063 (v1.5.5): garbage-collect TOML files for roles that no longer
|
|
109
|
+
// exist in the current charter. Without this sweep, charter A (role X+Y)
|
|
110
|
+
// → charter B (role X only) leaves `<safeAgentsDir>/y.toml` on disk and
|
|
111
|
+
// Codex continues to pick up the phantom agent. We compare the dir
|
|
112
|
+
// listing against the set of just-written agentFiles and unlink any
|
|
113
|
+
// `.toml` whose basename isn't claimed.
|
|
114
|
+
const removed = [];
|
|
115
|
+
const expected = new Set(agentFiles.map((p) => basename(p)));
|
|
116
|
+
let entries = [];
|
|
117
|
+
try { entries = readdirSync(safeAgentsDir); } catch { /* best-effort */ }
|
|
118
|
+
for (const name of entries) {
|
|
119
|
+
if (!name.endsWith('.toml')) continue;
|
|
120
|
+
if (expected.has(name)) continue;
|
|
121
|
+
const p = join(safeAgentsDir, name);
|
|
122
|
+
try { unlinkSync(p); removed.push(p); } catch { /* best-effort */ }
|
|
123
|
+
}
|
|
124
|
+
|
|
105
125
|
return {
|
|
106
126
|
ok: true,
|
|
107
127
|
agentsDir: safeAgentsDir,
|
|
108
128
|
agentFiles,
|
|
109
129
|
count: agentFiles.length,
|
|
110
130
|
skipped,
|
|
131
|
+
// Only surface `removed` when something was GCed so the common path stays
|
|
132
|
+
// terse.
|
|
133
|
+
...(removed.length > 0 ? { removed } : {}),
|
|
111
134
|
};
|
|
112
135
|
}
|
|
113
136
|
|