@ijfw/memory-server 1.5.1 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -5
- package/src/brain/budget-guard.js +86 -0
- package/src/brain/citation-resolver.js +41 -0
- package/src/brain/context-injection.js +69 -0
- package/src/brain/discovery.js +83 -0
- package/src/brain/dream-pipeline.js +324 -0
- package/src/brain/dump-ingest.js +88 -0
- package/src/brain/entity-collapse.js +28 -0
- package/src/brain/export.js +121 -0
- package/src/brain/extractors/index.js +24 -0
- package/src/brain/extractors/markdown.js +27 -0
- package/src/brain/extractors/pdf.js +31 -0
- package/src/brain/extractors/transcript.js +38 -0
- package/src/brain/first-run-scan.js +61 -0
- package/src/brain/index.js +1 -0
- package/src/brain/layout-sentinel.js +29 -0
- package/src/brain/migrate-facts-internal-once.js +87 -0
- package/src/brain/path-guard.js +103 -0
- package/src/brain/paths.js +26 -0
- package/src/brain/promotion-suggester.js +41 -0
- package/src/brain/stub-detector.js +33 -0
- package/src/brain/tiered-llm.js +83 -0
- package/src/brain/wiki-compiler.js +144 -0
- package/src/brain/wiki-sentinels.js +45 -0
- package/src/brain/wiki-templates.js +94 -0
- package/src/cross-orchestrator-cli.js +132 -5
- package/src/cross-orchestrator.js +2 -2
- package/src/dashboard-server.js +1 -1
- package/src/dream/runner.mjs +21 -0
- package/src/extension-registry.js +2 -2
- package/src/handlers/brain-handler.js +319 -0
- package/src/memory/auto-linker.js +5 -1
- package/src/memory/benchmark.js +4 -3
- package/src/memory/layout-migrations/001-visible-layer.js +131 -0
- package/src/memory/layout-migrations/index.js +50 -0
- package/src/memory/migration-runner.js +31 -2
- package/src/memory/obsidian-parser.js +3 -1
- package/src/memory/reader.js +2 -1
- package/src/memory/search.js +144 -16
- package/src/memory/temporal.js +40 -1
- package/src/orchestrator/agents-md-blackboard.js +114 -1
- package/src/orchestrator/discipline-selector.js +259 -0
- package/src/orchestrator/merge-block-aware.js +15 -5
- package/src/orchestrator/state-sdk.js +42 -4
- package/src/orchestrator/wave-state.js +38 -0
- package/src/recovery/code-fixer.js +1 -1
- package/src/server.js +290 -75
- package/src/update-apply.js +1 -1
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// IJFW v1.5.2 -- brain dump inbox scanner.
|
|
2
|
+
//
|
|
3
|
+
// scanInbox(dir) walks the inbox dir at depth-0 ONLY (no recursion into
|
|
4
|
+
// subdirs, no dotfiles), classifies each regular file into one of the
|
|
5
|
+
// supported kinds, and returns metadata for the dream-cycle to extract.
|
|
6
|
+
//
|
|
7
|
+
// Subdirs are reserved for future use (e.g., per-source folders); the dump
|
|
8
|
+
// pipeline treats them as opaque and skips them.
|
|
9
|
+
|
|
10
|
+
import { readdirSync, statSync, writeFileSync, readFileSync, renameSync, existsSync } from 'node:fs';
|
|
11
|
+
import { join, extname } from 'node:path';
|
|
12
|
+
|
|
13
|
+
function classify(name) {
|
|
14
|
+
if (name.includes('.transcript.')) return 'transcript';
|
|
15
|
+
const ext = extname(name).toLowerCase();
|
|
16
|
+
if (ext === '.md' || ext === '.markdown') return 'markdown';
|
|
17
|
+
if (ext === '.pdf') return 'pdf';
|
|
18
|
+
if (ext === '.txt') return 'text';
|
|
19
|
+
return 'unknown';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function scanInbox(inboxDir) {
|
|
23
|
+
let entries;
|
|
24
|
+
try {
|
|
25
|
+
entries = readdirSync(inboxDir, { withFileTypes: true });
|
|
26
|
+
} catch (e) {
|
|
27
|
+
if (e.code === 'ENOENT') return [];
|
|
28
|
+
throw e;
|
|
29
|
+
}
|
|
30
|
+
return entries
|
|
31
|
+
.filter((e) => e.isFile() && !e.name.startsWith('.'))
|
|
32
|
+
.map((e) => {
|
|
33
|
+
const p = join(inboxDir, e.name);
|
|
34
|
+
const s = statSync(p);
|
|
35
|
+
return {
|
|
36
|
+
path: p,
|
|
37
|
+
name: e.name,
|
|
38
|
+
kind: classify(e.name),
|
|
39
|
+
sizeBytes: s.size,
|
|
40
|
+
mtimeMs: s.mtimeMs,
|
|
41
|
+
};
|
|
42
|
+
})
|
|
43
|
+
.filter((f) => f.kind !== 'unknown');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export { classify };
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* writeManifest -- write the per-file receipt as <processed>/<name>.manifest.json
|
|
50
|
+
* atomically via .tmp+rename. Must be called BEFORE commitProcessed so a crash
|
|
51
|
+
* between manifest-write and inbox-rename leaves the file as an orphan in
|
|
52
|
+
* inbox/ (reprocessable on next cycle) -- never a half-state.
|
|
53
|
+
*/
|
|
54
|
+
export function writeManifest(processedDir, fileName, payload) {
|
|
55
|
+
const final = manifestPath(processedDir, fileName);
|
|
56
|
+
const tmp = final + '.tmp';
|
|
57
|
+
writeFileSync(tmp, JSON.stringify(payload, null, 2));
|
|
58
|
+
renameSync(tmp, final);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* commitProcessed -- atomically move <inbox>/<name> -> <processed>/<name>.
|
|
63
|
+
* Same-filesystem rename = atomic. Idempotent: silently succeeds if file
|
|
64
|
+
* already at destination and source is gone.
|
|
65
|
+
*/
|
|
66
|
+
export function commitProcessed(inboxDir, processedDir, fileName) {
|
|
67
|
+
const src = join(inboxDir, fileName);
|
|
68
|
+
const dst = join(processedDir, fileName);
|
|
69
|
+
if (!existsSync(src) && existsSync(dst)) return; // already committed (recovery no-op)
|
|
70
|
+
renameSync(src, dst);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* isProcessed -- true iff the manifest exists for this file. The manifest is
|
|
75
|
+
* the source of truth for "this file completed" -- the inbox->processed rename
|
|
76
|
+
* is a cleanup step after the manifest is written.
|
|
77
|
+
*/
|
|
78
|
+
export function isProcessed(processedDir, fileName) {
|
|
79
|
+
return existsSync(manifestPath(processedDir, fileName));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function readManifest(processedDir, fileName) {
|
|
83
|
+
return JSON.parse(readFileSync(manifestPath(processedDir, fileName), 'utf8'));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function manifestPath(processedDir, fileName) {
|
|
87
|
+
return join(processedDir, `${fileName}.manifest.json`);
|
|
88
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// IJFW v1.5.2 -- brain entity collapse.
|
|
2
|
+
//
|
|
3
|
+
// canonicalize(s): lowercase + collapsed-whitespace + trim. findCandidateMerges(db)
|
|
4
|
+
// surfaces groups of distinct stored subject strings that normalize to the same
|
|
5
|
+
// form (e.g. "Sean Donahoe" / "sean donahoe" / " Sean Donahoe " all collapse
|
|
6
|
+
// to "sean donahoe"). Promotion (actual merge) is operator-confirmed -- this
|
|
7
|
+
// module only surfaces candidates.
|
|
8
|
+
|
|
9
|
+
export function canonicalize(s) {
|
|
10
|
+
return String(s == null ? '' : s).toLowerCase().trim().replace(/\s+/g, ' ');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function findCandidateMerges(db) {
|
|
14
|
+
const rows = db.prepare('SELECT DISTINCT subject FROM facts').all();
|
|
15
|
+
const groups = new Map();
|
|
16
|
+
for (const { subject } of rows) {
|
|
17
|
+
if (subject == null) continue;
|
|
18
|
+
const c = canonicalize(subject);
|
|
19
|
+
if (!c) continue;
|
|
20
|
+
if (!groups.has(c)) groups.set(c, new Set());
|
|
21
|
+
groups.get(c).add(subject);
|
|
22
|
+
}
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const [canonical, set] of groups) {
|
|
25
|
+
if (set.size > 1) out.push({ canonical, variants: [...set].sort() });
|
|
26
|
+
}
|
|
27
|
+
return out.sort((a, b) => a.canonical.localeCompare(b.canonical));
|
|
28
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// IJFW v1.5.2 -- brain export + share helpers.
|
|
2
|
+
//
|
|
3
|
+
// exportPageBundle(repoRoot, slug, outFile)
|
|
4
|
+
// Reads <slug>.md from any of the 4 wiki type dirs, finds [[linked]]
|
|
5
|
+
// wikilinks in the content, inlines each linked page as an "### slug"
|
|
6
|
+
// section after the root. Returns { outFile, bytes, linkedPagesIncluded }.
|
|
7
|
+
//
|
|
8
|
+
// writeShareReadme(repoRoot)
|
|
9
|
+
// Writes <repoRoot>/ijfw/README.md with team-share instructions
|
|
10
|
+
// (commit ijfw/ to git, teammates clone + `ijfw memory reindex`).
|
|
11
|
+
//
|
|
12
|
+
// Both functions tolerate the v1->v2 layout transition: they consult the
|
|
13
|
+
// layout sentinel via resolveBrainPaths to pick the right wiki location.
|
|
14
|
+
|
|
15
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
16
|
+
import { dirname, join } from 'node:path';
|
|
17
|
+
import { resolveBrainPaths } from './paths.js';
|
|
18
|
+
import { validateSafeRepoPath } from './path-guard.js';
|
|
19
|
+
|
|
20
|
+
const WIKI_TYPES = ['concepts', 'entities', 'decisions', 'milestones'];
|
|
21
|
+
// Match the entire bracket-pair content as a single negated character class.
|
|
22
|
+
// The slug (left of optional `|`) is extracted in JS after the regex match
|
|
23
|
+
// instead of via a nested optional group -- safe-regex flagged the original
|
|
24
|
+
// pattern (`[^\]\n|]+(?:\|[^\]\n]*)?`) as potentially unsafe even though it
|
|
25
|
+
// can't backtrack catastrophically. This single-class form is provably linear.
|
|
26
|
+
const WIKILINK_RE = /\[\[([^\]\n]+)\]\]/g;
|
|
27
|
+
|
|
28
|
+
function findPage(wikiDir, slug) {
|
|
29
|
+
for (const t of WIKI_TYPES) {
|
|
30
|
+
const p = join(wikiDir, t, `${slug}.md`);
|
|
31
|
+
if (existsSync(p)) return { path: p, type: t };
|
|
32
|
+
}
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function parseWikilinks(md) {
|
|
37
|
+
if (!md) return [];
|
|
38
|
+
const out = new Set();
|
|
39
|
+
for (const m of md.matchAll(WIKILINK_RE)) {
|
|
40
|
+
// The capture group now includes the optional `|alias` suffix; the slug
|
|
41
|
+
// is the substring before the first `|` (matches the original semantics).
|
|
42
|
+
const inner = m[1];
|
|
43
|
+
const pipeIdx = inner.indexOf('|');
|
|
44
|
+
const target = (pipeIdx === -1 ? inner : inner.slice(0, pipeIdx)).trim();
|
|
45
|
+
if (target) out.add(target);
|
|
46
|
+
}
|
|
47
|
+
return [...out];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function exportPageBundle(repoRoot, slug, outFile) {
|
|
51
|
+
// F-LENS2-05 defense-in-depth: brain-handler already validates outFile via
|
|
52
|
+
// validateSafeRepoPath, but exportPageBundle is also exported and may be
|
|
53
|
+
// called directly by future callers. Re-validate here so the policy holds
|
|
54
|
+
// at the writer boundary too.
|
|
55
|
+
const guard = validateSafeRepoPath(repoRoot, outFile);
|
|
56
|
+
if (!guard.ok) return { error: guard.error };
|
|
57
|
+
const paths = resolveBrainPaths(repoRoot);
|
|
58
|
+
const root = findPage(paths.wikiDir, slug);
|
|
59
|
+
if (!root) return { error: 'page-not-found', slug };
|
|
60
|
+
const rootBody = readFileSync(root.path, 'utf8');
|
|
61
|
+
const linked = parseWikilinks(rootBody);
|
|
62
|
+
|
|
63
|
+
const parts = [
|
|
64
|
+
`# Export: ${slug}\n\n_Generated by IJFW brain export from ${root.type}/${slug}.md._\n`,
|
|
65
|
+
`## ${slug}\n\n${rootBody.trim()}`,
|
|
66
|
+
];
|
|
67
|
+
const includedLinks = [];
|
|
68
|
+
for (const target of linked) {
|
|
69
|
+
const found = findPage(paths.wikiDir, target);
|
|
70
|
+
if (!found) continue;
|
|
71
|
+
let body;
|
|
72
|
+
try { body = readFileSync(found.path, 'utf8'); } catch { continue; }
|
|
73
|
+
parts.push(`### ${target}\n\n${body.trim()}`);
|
|
74
|
+
includedLinks.push(target);
|
|
75
|
+
}
|
|
76
|
+
const out = parts.join('\n\n') + '\n';
|
|
77
|
+
mkdirSync(dirname(outFile), { recursive: true });
|
|
78
|
+
writeFileSync(outFile, out);
|
|
79
|
+
return { outFile, bytes: Buffer.byteLength(out, 'utf8'), linkedPagesIncluded: includedLinks };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const SHARE_README = `# Your IJFW Brain
|
|
83
|
+
|
|
84
|
+
This folder is **your project brain** — durable, queryable knowledge maintained by IJFW across every CLI you use (Claude Code, Codex, Cursor, Windsurf, Gemini, and more).
|
|
85
|
+
|
|
86
|
+
## Structure
|
|
87
|
+
|
|
88
|
+
\`\`\`
|
|
89
|
+
ijfw/
|
|
90
|
+
├── memory/ # raw memory entries (knowledge.md, handoff.md, journal.md, decisions.md)
|
|
91
|
+
├── sessions/ # readable per-session logs
|
|
92
|
+
├── dump/
|
|
93
|
+
│ ├── inbox/ # drop ANY file here (md/txt/transcript/pdf) — gets ingested next dream cycle
|
|
94
|
+
│ └── processed/ # files move here once ingested, with a .manifest.json receipt
|
|
95
|
+
└── wiki/ # LLM-curated, Obsidian-readable
|
|
96
|
+
├── concepts/, entities/, decisions/, milestones/
|
|
97
|
+
\`\`\`
|
|
98
|
+
|
|
99
|
+
## Share with your team
|
|
100
|
+
|
|
101
|
+
1. Commit \`ijfw/\` to git like any other source folder.
|
|
102
|
+
2. Teammates clone the repo and run \`ijfw memory reindex\` to populate their local index.
|
|
103
|
+
3. The internal index db lives in \`.ijfw/\` (gitignored) — only the content in \`ijfw/\` is shared.
|
|
104
|
+
|
|
105
|
+
## Privacy
|
|
106
|
+
|
|
107
|
+
Everything in \`ijfw/\` is plain markdown / json. Inspect it. Edit it. Diff it. Your brain stays under your version control.
|
|
108
|
+
`;
|
|
109
|
+
|
|
110
|
+
export function writeShareReadme(repoRoot) {
|
|
111
|
+
const paths = resolveBrainPaths(repoRoot);
|
|
112
|
+
const out = join(paths.contentDir, 'README.md');
|
|
113
|
+
// F-LENS2-05: validate the share-readme path before writing. A symlinked
|
|
114
|
+
// ijfw/ directory or contentDir resolution outside the repo would otherwise
|
|
115
|
+
// let writeFileSync clobber an arbitrary path.
|
|
116
|
+
const guard = validateSafeRepoPath(repoRoot, out);
|
|
117
|
+
if (!guard.ok) return { error: guard.error };
|
|
118
|
+
mkdirSync(paths.contentDir, { recursive: true });
|
|
119
|
+
writeFileSync(out, SHARE_README);
|
|
120
|
+
return { outFile: out, bytes: Buffer.byteLength(SHARE_README, 'utf8') };
|
|
121
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// index.js -- dispatch extractFile(file) by kind.
|
|
2
|
+
import { extractMarkdown } from './markdown.js';
|
|
3
|
+
import { extractTranscript } from './transcript.js';
|
|
4
|
+
import { extractPdf } from './pdf.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* file: object from scanInbox -- shape { path, name, kind, sizeBytes, mtimeMs }.
|
|
8
|
+
* Returns Promise<{ text, chunks } | { error, ... }>.
|
|
9
|
+
*/
|
|
10
|
+
export async function extractFile(file) {
|
|
11
|
+
switch (file.kind) {
|
|
12
|
+
case 'markdown':
|
|
13
|
+
case 'text':
|
|
14
|
+
return extractMarkdown(file.path);
|
|
15
|
+
case 'transcript':
|
|
16
|
+
return extractTranscript(file.path);
|
|
17
|
+
case 'pdf':
|
|
18
|
+
return extractPdf(file.path);
|
|
19
|
+
default:
|
|
20
|
+
return { error: 'unsupported-kind', kind: file.kind, path: file.path };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export { extractMarkdown, extractTranscript, extractPdf };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// markdown.js -- markdown + text extractor (chunk on blank-line boundaries).
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
|
|
4
|
+
const DEFAULT_MAX_CHARS = 3000;
|
|
5
|
+
|
|
6
|
+
export function chunkAtBlankLines(text, maxChars = DEFAULT_MAX_CHARS) {
|
|
7
|
+
if (text.length <= maxChars) return [text];
|
|
8
|
+
const paragraphs = text.split(/\n\s*\n/);
|
|
9
|
+
const chunks = [];
|
|
10
|
+
let cur = '';
|
|
11
|
+
for (const p of paragraphs) {
|
|
12
|
+
const candidate = cur ? cur + '\n\n' + p : p;
|
|
13
|
+
if (candidate.length <= maxChars) { cur = candidate; continue; }
|
|
14
|
+
if (cur) chunks.push(cur);
|
|
15
|
+
if (p.length <= maxChars) { cur = p; continue; }
|
|
16
|
+
// single paragraph exceeds maxChars — hard-split on chars
|
|
17
|
+
for (let i = 0; i < p.length; i += maxChars) chunks.push(p.slice(i, i + maxChars));
|
|
18
|
+
cur = '';
|
|
19
|
+
}
|
|
20
|
+
if (cur) chunks.push(cur);
|
|
21
|
+
return chunks;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function extractMarkdown(filePath, { maxChars = DEFAULT_MAX_CHARS } = {}) {
|
|
25
|
+
const text = readFileSync(filePath, 'utf8');
|
|
26
|
+
return { text, chunks: chunkAtBlankLines(text, maxChars) };
|
|
27
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// pdf.js -- lazy pdf-parse wrapper. pdf-parse is NOT a hard dependency;
|
|
2
|
+
// returns { error } gracefully when the dep is missing.
|
|
3
|
+
|
|
4
|
+
import { readFileSync } from 'node:fs';
|
|
5
|
+
|
|
6
|
+
export async function extractPdf(filePath) {
|
|
7
|
+
let pdfParse;
|
|
8
|
+
try {
|
|
9
|
+
// pdf-parse exposes its parser as the default export of its CJS module.
|
|
10
|
+
const mod = await import('pdf-parse');
|
|
11
|
+
pdfParse = mod.default || mod;
|
|
12
|
+
} catch (e) {
|
|
13
|
+
if (e.code === 'ERR_MODULE_NOT_FOUND' || /Cannot find module/i.test(e.message || '')) {
|
|
14
|
+
return { error: 'pdf-parse-not-installed', filePath };
|
|
15
|
+
}
|
|
16
|
+
throw e;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
const buf = readFileSync(filePath);
|
|
20
|
+
const parsed = await pdfParse(buf);
|
|
21
|
+
const text = parsed && typeof parsed.text === 'string' ? parsed.text : '';
|
|
22
|
+
// Chunk on form-feed (PDF page break) when present, else fall back to blank-line chunking.
|
|
23
|
+
const { chunkAtBlankLines } = await import('./markdown.js');
|
|
24
|
+
const pageChunks = text.includes('\f')
|
|
25
|
+
? text.split('\f').map((p) => p.trim()).filter(Boolean)
|
|
26
|
+
: chunkAtBlankLines(text);
|
|
27
|
+
return { text, chunks: pageChunks };
|
|
28
|
+
} catch (e) {
|
|
29
|
+
return { error: 'pdf-parse-failed', message: e.message, filePath };
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// transcript.js -- speaker-turn chunking.
|
|
2
|
+
//
|
|
3
|
+
// Detects turns whose first line starts with `Name:` or `[Name]`. Each turn
|
|
4
|
+
// is one chunk. Sequential lines without a speaker prefix belong to the
|
|
5
|
+
// current speaker's turn.
|
|
6
|
+
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
|
|
9
|
+
// Speaker prefix at line start: "Name:" (alphanumeric + spaces) or "[Name]".
|
|
10
|
+
// Allow up to 60 chars of name to avoid pathological matches on prose.
|
|
11
|
+
const SPEAKER_RE = /^(\[[^\]]{1,60}\]|[A-Za-z][A-Za-z0-9 _.'-]{0,59}:)\s/;
|
|
12
|
+
|
|
13
|
+
export function splitOnSpeakerTurns(text) {
|
|
14
|
+
const lines = text.split('\n');
|
|
15
|
+
const turns = [];
|
|
16
|
+
let current = null;
|
|
17
|
+
for (const line of lines) {
|
|
18
|
+
if (SPEAKER_RE.test(line)) {
|
|
19
|
+
if (current !== null) turns.push(current);
|
|
20
|
+
current = line;
|
|
21
|
+
} else {
|
|
22
|
+
if (current === null) {
|
|
23
|
+
// pre-amble before any speaker — collect into its own chunk
|
|
24
|
+
current = line;
|
|
25
|
+
} else {
|
|
26
|
+
current += '\n' + line;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (current !== null) turns.push(current);
|
|
31
|
+
// trim trailing newlines per turn, drop empty
|
|
32
|
+
return turns.map((t) => t.replace(/\n+$/, '')).filter((t) => t.trim().length > 0);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function extractTranscript(filePath) {
|
|
36
|
+
const text = readFileSync(filePath, 'utf8');
|
|
37
|
+
return { text, chunks: splitOnSpeakerTurns(text) };
|
|
38
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// IJFW v1.5.2 -- brain first-run scan.
|
|
2
|
+
//
|
|
3
|
+
// Powers the dashboard / Wayland portal "wow" cold-start by surfacing the
|
|
4
|
+
// operator's existing CLI memory from known per-CLI homes. Each "source"
|
|
5
|
+
// reports { id, label, path, found, count, projects }. onProgress fires
|
|
6
|
+
// as { stage: "scanning" | "found", id, ... }.
|
|
7
|
+
|
|
8
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
|
|
11
|
+
const SOURCES = [
|
|
12
|
+
{ id: 'claude', label: 'Claude Code', rel: '.claude/projects', kind: 'dir-of-projects' },
|
|
13
|
+
{ id: 'codex', label: 'Codex CLI', rel: '.codex', kind: 'dir' },
|
|
14
|
+
{ id: 'gemini', label: 'Gemini CLI', rel: '.gemini', kind: 'dir' },
|
|
15
|
+
{ id: 'cursor', label: 'Cursor', rel: '.cursor', kind: 'dir' },
|
|
16
|
+
{ id: 'windsurf', label: 'Windsurf', rel: '.windsurf', kind: 'dir' },
|
|
17
|
+
{ id: 'claude-md', label: 'Global CLAUDE.md', rel: 'CLAUDE.md', kind: 'file' },
|
|
18
|
+
{ id: 'agents-md', label: 'Global AGENTS.md', rel: 'AGENTS.md', kind: 'file' },
|
|
19
|
+
{ id: 'gemini-md', label: 'Global GEMINI.md', rel: 'GEMINI.md', kind: 'file' },
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
function safeReaddir(p) { try { return readdirSync(p); } catch { return []; } }
|
|
23
|
+
|
|
24
|
+
function sessionsForClaude(claudeProjectsDir) {
|
|
25
|
+
const projects = safeReaddir(claudeProjectsDir).filter((name) => {
|
|
26
|
+
try { return statSync(join(claudeProjectsDir, name)).isDirectory(); } catch { return false; }
|
|
27
|
+
});
|
|
28
|
+
let totalSessions = 0;
|
|
29
|
+
for (const p of projects) {
|
|
30
|
+
const files = safeReaddir(join(claudeProjectsDir, p)).filter((f) => f.endsWith('.jsonl'));
|
|
31
|
+
totalSessions += files.length;
|
|
32
|
+
}
|
|
33
|
+
return { projects: projects.length, totalSessions };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function firstRunScan({ homeDir, onProgress } = {}) {
|
|
37
|
+
const sources = [];
|
|
38
|
+
let totalSessions = 0;
|
|
39
|
+
for (const src of SOURCES) {
|
|
40
|
+
if (onProgress) onProgress({ stage: 'scanning', id: src.id });
|
|
41
|
+
const p = join(homeDir, src.rel);
|
|
42
|
+
const present = existsSync(p);
|
|
43
|
+
if (!present) {
|
|
44
|
+
sources.push({ id: src.id, label: src.label, path: p, found: false, count: 0, projects: 0 });
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
let count = 0, projects = 0;
|
|
48
|
+
if (src.kind === 'dir-of-projects') {
|
|
49
|
+
const r = sessionsForClaude(p);
|
|
50
|
+
projects = r.projects; count = r.totalSessions; totalSessions += r.totalSessions;
|
|
51
|
+
} else if (src.kind === 'dir') {
|
|
52
|
+
count = safeReaddir(p).length;
|
|
53
|
+
} else if (src.kind === 'file') {
|
|
54
|
+
try { count = statSync(p).size > 0 ? 1 : 0; } catch { count = 0; }
|
|
55
|
+
}
|
|
56
|
+
const entry = { id: src.id, label: src.label, path: p, found: true, count, projects };
|
|
57
|
+
sources.push(entry);
|
|
58
|
+
if (onProgress) onProgress({ stage: 'found', id: src.id, count, projects });
|
|
59
|
+
}
|
|
60
|
+
return { sources, totalSessions };
|
|
61
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const BRAIN_VERSION = 1;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, openSync, closeSync, unlinkSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
export function readLayoutVersion(repoRoot) {
|
|
5
|
+
const p = join(repoRoot, '.ijfw', '.layout-version');
|
|
6
|
+
if (!existsSync(p)) return 1;
|
|
7
|
+
const v = parseInt(readFileSync(p, 'utf8').trim(), 10);
|
|
8
|
+
return Number.isFinite(v) && v >= 1 ? v : 1;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function writeLayoutVersion(repoRoot, version) {
|
|
12
|
+
writeFileSync(join(repoRoot, '.ijfw', '.layout-version'), `${version}\n`);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function withLayoutLock(repoRoot, fn, { timeoutMs = 5000 } = {}) {
|
|
16
|
+
const lockPath = join(repoRoot, '.ijfw', '.migrate.lock');
|
|
17
|
+
const start = Date.now();
|
|
18
|
+
let fd = null;
|
|
19
|
+
while (true) {
|
|
20
|
+
try { fd = openSync(lockPath, 'wx'); break; }
|
|
21
|
+
catch (e) {
|
|
22
|
+
if (e.code !== 'EEXIST') throw e;
|
|
23
|
+
if (Date.now() - start > timeoutMs) throw new Error(`layout-sentinel: locked > ${timeoutMs}ms`);
|
|
24
|
+
await new Promise(r => setTimeout(r, 25));
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
try { return await fn(); }
|
|
28
|
+
finally { try { closeSync(fd); } catch {} try { unlinkSync(lockPath); } catch {} }
|
|
29
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// IJFW v1.5.2 -- one-shot migration: relocate FACTS_FILE + FACTS_DB_FILE to
|
|
2
|
+
// the internal hidden-paths location (Plan A audit F5).
|
|
3
|
+
//
|
|
4
|
+
// Plan A's lazy MEMORY_DIR refactor (Task 5) intentionally kept FACTS_FILE
|
|
5
|
+
// + FACTS_DB_FILE at <contentDir>/memory/ with a deferral comment, because
|
|
6
|
+
// moving them required a data migration outside Task 5's scope. This file
|
|
7
|
+
// IS the F5 migration that closes that deferral:
|
|
8
|
+
//
|
|
9
|
+
// .ijfw/memory/facts.jsonl -> .ijfw/facts.jsonl
|
|
10
|
+
// .ijfw/memory/facts.db -> .ijfw/index/memory.db
|
|
11
|
+
//
|
|
12
|
+
// This matches the design contract from Task 3's paths.js: internal paths
|
|
13
|
+
// (indexDb, factsJsonl, stateDir, metricsDir, receiptsDir) ALWAYS live under
|
|
14
|
+
// .ijfw/, never under the visible ijfw/ content dir. Without this migration
|
|
15
|
+
// the post-Task-10 layout would move FACTS into ijfw/memory/ — the opposite
|
|
16
|
+
// of where they belong.
|
|
17
|
+
//
|
|
18
|
+
// Sync, idempotent, atomic (renameSync is a single syscall on POSIX). Runs at
|
|
19
|
+
// server startup; safe to call repeatedly. Crash mid-migration leaves either
|
|
20
|
+
// the old or the new path populated, never both — operator can re-run
|
|
21
|
+
// safely.
|
|
22
|
+
|
|
23
|
+
import { existsSync, mkdirSync, renameSync } from 'node:fs';
|
|
24
|
+
import { join, dirname } from 'node:path';
|
|
25
|
+
|
|
26
|
+
export function migrateFactsInternalOnce(repoRoot) {
|
|
27
|
+
if (!repoRoot || typeof repoRoot !== 'string') {
|
|
28
|
+
return { skipped: true, reason: 'no-repo-root' };
|
|
29
|
+
}
|
|
30
|
+
const oldJsonl = join(repoRoot, '.ijfw', 'memory', 'facts.jsonl');
|
|
31
|
+
const newJsonl = join(repoRoot, '.ijfw', 'facts.jsonl');
|
|
32
|
+
const oldDb = join(repoRoot, '.ijfw', 'memory', 'facts.db');
|
|
33
|
+
const newDb = join(repoRoot, '.ijfw', 'index', 'memory.db');
|
|
34
|
+
// v1.5.2.1 F3: 010 may have already copied facts.{jsonl,db} into the visible
|
|
35
|
+
// layer at ijfw/memory/ before this migration runs. Those copies are NEVER
|
|
36
|
+
// valid as a runtime source — facts always live at the internal paths
|
|
37
|
+
// (paths().factsJsonl / paths().indexDb), and the visible-layer copies will
|
|
38
|
+
// drift on the first write. Flag them so the operator can prune; do NOT
|
|
39
|
+
// delete automatically (could destroy user data in extremis).
|
|
40
|
+
const visibleJsonlOrphan = join(repoRoot, 'ijfw', 'memory', 'facts.jsonl');
|
|
41
|
+
const visibleDbOrphan = join(repoRoot, 'ijfw', 'memory', 'facts.db');
|
|
42
|
+
|
|
43
|
+
// Helper to extend the orphans list with any visible-layer facts files.
|
|
44
|
+
const collectVisibleOrphans = (orphans) => {
|
|
45
|
+
if (existsSync(visibleJsonlOrphan)) orphans.push(visibleJsonlOrphan);
|
|
46
|
+
if (existsSync(visibleDbOrphan)) orphans.push(visibleDbOrphan);
|
|
47
|
+
return orphans;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
// Idempotent short-circuit: if EITHER new path exists, treat as migrated.
|
|
51
|
+
// (We don't require both because a partial migration that crashed before
|
|
52
|
+
// the second move would still be detectable on retry, but a fully
|
|
53
|
+
// intentional standalone deployment of just one is operator-territory and
|
|
54
|
+
// we don't second-guess it.)
|
|
55
|
+
if (existsSync(newJsonl) || existsSync(newDb)) {
|
|
56
|
+
// FLAG-7: detect orphans — if a legacy path ALSO exists alongside the
|
|
57
|
+
// new path, surface it so the operator can clean up. Don't move
|
|
58
|
+
// automatically (could overwrite real data); just flag. v1.5.2.1 also
|
|
59
|
+
// detects the visible-layer facts files left by an earlier 010 run.
|
|
60
|
+
const orphans = [];
|
|
61
|
+
if (existsSync(oldJsonl)) orphans.push(oldJsonl);
|
|
62
|
+
if (existsSync(oldDb)) orphans.push(oldDb);
|
|
63
|
+
collectVisibleOrphans(orphans);
|
|
64
|
+
return orphans.length > 0
|
|
65
|
+
? { skipped: true, reason: 'already-migrated', orphans }
|
|
66
|
+
: { skipped: true, reason: 'already-migrated' };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const moved = [];
|
|
70
|
+
if (existsSync(oldJsonl)) {
|
|
71
|
+
mkdirSync(dirname(newJsonl), { recursive: true });
|
|
72
|
+
renameSync(oldJsonl, newJsonl);
|
|
73
|
+
moved.push({ from: oldJsonl, to: newJsonl });
|
|
74
|
+
}
|
|
75
|
+
if (existsSync(oldDb)) {
|
|
76
|
+
mkdirSync(dirname(newDb), { recursive: true });
|
|
77
|
+
renameSync(oldDb, newDb);
|
|
78
|
+
moved.push({ from: oldDb, to: newDb });
|
|
79
|
+
}
|
|
80
|
+
// v1.5.2.1 F3: surface visible-layer facts orphans on the success path too,
|
|
81
|
+
// in case 010 ran before 011 on this install. They're still wrong; operator
|
|
82
|
+
// gets the same prune cue.
|
|
83
|
+
const orphans = collectVisibleOrphans([]);
|
|
84
|
+
return orphans.length > 0
|
|
85
|
+
? { skipped: false, moved, orphans }
|
|
86
|
+
: { skipped: false, moved };
|
|
87
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// IJFW v1.5.2.1 -- shared safe-write path guard (F-LENS2-05, F-LENS2-06, F-LENS2-10).
|
|
2
|
+
//
|
|
3
|
+
// validateSafeRepoPath(repoRoot, targetPath) returns:
|
|
4
|
+
// { ok: true, resolved }
|
|
5
|
+
// { ok: false, error: 'repoRoot-missing' | 'outFile-escapes-repo' | 'outFile-reserved-name', ... }
|
|
6
|
+
//
|
|
7
|
+
// Why a shared helper:
|
|
8
|
+
// - F-LENS2-05: wiki.export's guard was the only callsite. wiki.compile,
|
|
9
|
+
// wiki.shareReadme, dream/budget logs all wrote without containment.
|
|
10
|
+
// - F-LENS2-06: the reserved-name regex missed Windows trailing-dot /
|
|
11
|
+
// trailing-space / NTFS-stream variants ("CON ", "CON.", "CON:Stream1")
|
|
12
|
+
// because Windows itself trims those before opening the device.
|
|
13
|
+
// - F-LENS2-10: the previous guard threw if repoRoot didn't exist on disk
|
|
14
|
+
// (realpathSync on a missing dir). Tests sometimes pass a not-yet-created
|
|
15
|
+
// repoRoot — return a structured rejection instead.
|
|
16
|
+
//
|
|
17
|
+
// The reserved-name check normalises basename via the same trim Windows
|
|
18
|
+
// applies: strip trailing dots/whitespace, then drop NTFS alternate-data-
|
|
19
|
+
// stream suffix (everything after the first ':'). The tightened character
|
|
20
|
+
// class is `com[1-9]|lpt[1-9]` (1-9, not 0-9) per Microsoft's documented set.
|
|
21
|
+
|
|
22
|
+
import { realpathSync, lstatSync } from 'node:fs';
|
|
23
|
+
import {
|
|
24
|
+
resolve as pathResolve,
|
|
25
|
+
relative as pathRelative,
|
|
26
|
+
dirname as pathDirname,
|
|
27
|
+
basename as pathBasename,
|
|
28
|
+
isAbsolute as pathIsAbsolute,
|
|
29
|
+
join as pathJoin,
|
|
30
|
+
} from 'node:path';
|
|
31
|
+
|
|
32
|
+
const WIN_RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
|
|
33
|
+
|
|
34
|
+
export function validateSafeRepoPath(repoRoot, targetPath) {
|
|
35
|
+
if (!repoRoot || typeof repoRoot !== 'string') {
|
|
36
|
+
return { ok: false, error: 'repoRoot-missing', repoRoot };
|
|
37
|
+
}
|
|
38
|
+
if (!targetPath || typeof targetPath !== 'string') {
|
|
39
|
+
return { ok: false, error: 'outFile-missing', targetPath };
|
|
40
|
+
}
|
|
41
|
+
let resolvedRoot;
|
|
42
|
+
try {
|
|
43
|
+
resolvedRoot = realpathSync(pathResolve(repoRoot));
|
|
44
|
+
} catch {
|
|
45
|
+
return { ok: false, error: 'repoRoot-missing', repoRoot };
|
|
46
|
+
}
|
|
47
|
+
let resolved = pathResolve(targetPath);
|
|
48
|
+
// Walk UP toward the root to find the closest existing ancestor, realpath
|
|
49
|
+
// THAT, then re-attach the not-yet-created suffix. Why: macOS /tmp is a
|
|
50
|
+
// symlink to /private/tmp; resolvedRoot becomes /private/var/folders/...
|
|
51
|
+
// but pathResolve(targetPath) gives /var/folders/.../ijfw/wiki/... — the
|
|
52
|
+
// bare-lexical containment check would then claim the path "escapes" the
|
|
53
|
+
// repo. Canonicalising the deepest-existing ancestor unmasks the symlink
|
|
54
|
+
// chain symmetrically with resolvedRoot.
|
|
55
|
+
{
|
|
56
|
+
let ancestor = resolved;
|
|
57
|
+
const suffix = [];
|
|
58
|
+
// Bound the walk so a pathological input can't spin: at most 64 levels
|
|
59
|
+
// (far deeper than any realistic project layout).
|
|
60
|
+
for (let i = 0; i < 64; i++) {
|
|
61
|
+
try {
|
|
62
|
+
ancestor = realpathSync(ancestor);
|
|
63
|
+
break;
|
|
64
|
+
} catch {
|
|
65
|
+
const parent = pathDirname(ancestor);
|
|
66
|
+
if (parent === ancestor) break; // reached '/' or drive root
|
|
67
|
+
suffix.unshift(pathBasename(ancestor));
|
|
68
|
+
ancestor = parent;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
resolved = suffix.length ? pathJoin(ancestor, ...suffix) : ancestor;
|
|
72
|
+
}
|
|
73
|
+
const rel = pathRelative(resolvedRoot, resolved);
|
|
74
|
+
if (rel === '' || rel.startsWith('..') || pathIsAbsolute(rel)) {
|
|
75
|
+
return { ok: false, error: 'outFile-escapes-repo', resolved, repoRoot: resolvedRoot };
|
|
76
|
+
}
|
|
77
|
+
// Windows trims trailing dots/whitespace and treats `name:stream` as the
|
|
78
|
+
// bare `name` device. Normalise the basename the same way before checking.
|
|
79
|
+
const baseRaw = pathBasename(resolved);
|
|
80
|
+
const baseTrimmed = baseRaw.replace(/[.\s]+$/, '');
|
|
81
|
+
const stem = baseTrimmed.split(':')[0].split('.')[0];
|
|
82
|
+
if (WIN_RESERVED.test(stem)) {
|
|
83
|
+
return { ok: false, error: 'outFile-reserved-name', resolved };
|
|
84
|
+
}
|
|
85
|
+
// F-LENS2-07: TOCTOU hardening. Even after parent-ancestor canonicalization,
|
|
86
|
+
// an attacker could create the TARGET itself as a symlink between this
|
|
87
|
+
// guard and the writeFileSync. lstatSync (does NOT follow symlinks) catches
|
|
88
|
+
// an existing symlink at the resolved path. Pre-existing regular file is
|
|
89
|
+
// fine — writers either overwrite or 'wx'-fail on their own. The residual
|
|
90
|
+
// TOCTOU window between this check and the writer's open is microseconds
|
|
91
|
+
// and requires a same-uid attacker; documented threat model.
|
|
92
|
+
try {
|
|
93
|
+
const st = lstatSync(resolved);
|
|
94
|
+
if (st.isSymbolicLink()) {
|
|
95
|
+
return { ok: false, error: 'outFile-is-symlink', resolved };
|
|
96
|
+
}
|
|
97
|
+
} catch {
|
|
98
|
+
// ENOENT — target doesn't exist yet; the most common case. OK to proceed.
|
|
99
|
+
}
|
|
100
|
+
return { ok: true, resolved };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export default { validateSafeRepoPath };
|