@ijfw/memory-server 1.5.0 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ijfw-memorize +14 -7
- package/fixtures/team/book.json +6 -6
- package/fixtures/team/business.json +146 -20
- package/fixtures/team/content.json +6 -6
- package/fixtures/team/design.json +148 -20
- package/fixtures/team/mixed.json +206 -27
- package/fixtures/team/research.json +146 -20
- package/fixtures/team/software.json +148 -20
- package/package.json +8 -4
- package/src/brain/budget-guard.js +86 -0
- package/src/brain/citation-resolver.js +41 -0
- package/src/brain/context-injection.js +69 -0
- package/src/brain/discovery.js +83 -0
- package/src/brain/dream-pipeline.js +324 -0
- package/src/brain/dump-ingest.js +88 -0
- package/src/brain/entity-collapse.js +28 -0
- package/src/brain/export.js +112 -0
- package/src/brain/extractors/index.js +24 -0
- package/src/brain/extractors/markdown.js +27 -0
- package/src/brain/extractors/pdf.js +31 -0
- package/src/brain/extractors/transcript.js +38 -0
- package/src/brain/first-run-scan.js +61 -0
- package/src/brain/index.js +1 -0
- package/src/brain/layout-sentinel.js +29 -0
- package/src/brain/migrate-facts-internal-once.js +87 -0
- package/src/brain/path-guard.js +103 -0
- package/src/brain/paths.js +26 -0
- package/src/brain/promotion-suggester.js +41 -0
- package/src/brain/stub-detector.js +33 -0
- package/src/brain/tiered-llm.js +83 -0
- package/src/brain/wiki-compiler.js +144 -0
- package/src/brain/wiki-sentinels.js +45 -0
- package/src/brain/wiki-templates.js +94 -0
- package/src/cross-orchestrator-cli.js +336 -150
- package/src/cross-orchestrator.js +52 -3
- package/src/dashboard-server.js +1 -1
- package/src/dispatch/extension.js +1 -1
- package/src/dream/runner.mjs +21 -0
- package/src/extension-registry.js +2 -2
- package/src/handlers/brain-handler.js +319 -0
- package/src/hardware-signer.js +4 -2
- package/src/lib/ui-review-runner.js +48 -7
- package/src/memory/auto-linker.js +121 -2
- package/src/memory/benchmark.js +4 -3
- package/src/memory/layout-migrations/001-visible-layer.js +131 -0
- package/src/memory/layout-migrations/index.js +50 -0
- package/src/memory/migration-runner.js +37 -3
- package/src/memory/migrations/009-obsidian-backfill.js +50 -0
- package/src/memory/obsidian-parser.js +65 -2
- package/src/memory/reader.js +2 -1
- package/src/memory/search.js +190 -41
- package/src/memory/temporal.js +40 -1
- package/src/orchestrator/agents-md-blackboard.js +114 -1
- package/src/orchestrator/debug-trident-trigger.js +374 -0
- package/src/orchestrator/discipline-selector.js +276 -0
- package/src/orchestrator/merge-block-aware.js +15 -5
- package/src/orchestrator/post-done-runner.js +36 -8
- package/src/orchestrator/state-sdk.js +216 -10
- package/src/orchestrator/subagent-telemetry.js +19 -0
- package/src/orchestrator/wave-state.js +38 -0
- package/src/override-resolver.js +5 -3
- package/src/recovery/code-fixer.js +311 -6
- package/src/runtime-mediator.js +0 -1
- package/src/server.js +486 -132
- package/src/swarm-config.js +30 -22
- package/src/team/domain-templates/business.json +4 -1
- package/src/team/domain-templates/research.json +4 -1
- package/src/team/generator.js +162 -0
- package/src/update-apply.js +1 -1
- package/src/dashboard-charts.js +0 -239
- package/src/orchestrator/runtime-loop.js +0 -430
|
@@ -101,7 +101,11 @@ function applyProposal(db, entry, proposal) {
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
});
|
|
104
|
-
|
|
104
|
+
// F-LENS2-01: use IMMEDIATE so all facts-table sister-writers acquire the
|
|
105
|
+
// RESERVED lock in the same order — no DEFERRED→IMMEDIATE upgrade collision
|
|
106
|
+
// when another writer (storeFactBitemporal, dream-pipeline, obsidian-parser)
|
|
107
|
+
// holds the lock.
|
|
108
|
+
tx.immediate();
|
|
105
109
|
return { links_added: linksAdded, neighbor_tags_added: neighborTagsAdded };
|
|
106
110
|
}
|
|
107
111
|
|
|
@@ -149,4 +153,119 @@ export async function autoLink(db, entry, opts = {}) {
|
|
|
149
153
|
return { skipped: false, neighbors, proposal, applied };
|
|
150
154
|
}
|
|
151
155
|
|
|
152
|
-
|
|
156
|
+
// v1.5.1 R5-1.2 -- one-time M2 (A-Mem auto-link) backfill for memory written
|
|
157
|
+
// during v1.5.0, when autoLink was NOT wired into the production write path.
|
|
158
|
+
//
|
|
159
|
+
// UNLIKE the M1 backfill (free, always-on), M2 backfill makes one LLM call
|
|
160
|
+
// per row -- backfilling over a large memory can cost real money. So M2
|
|
161
|
+
// backfill is OPT-IN and budget-gated:
|
|
162
|
+
//
|
|
163
|
+
// - IJFW_AUTOLINK_OFF=1 -> backfill is a no-op (kill switch)
|
|
164
|
+
// - IJFW_AUTOLINK_BACKFILL!=1 -> backfill is a no-op by default
|
|
165
|
+
// (M1-always, M2-opt-in is the safe
|
|
166
|
+
// default per R5-1.2)
|
|
167
|
+
// - IJFW_AUTOLINK_BUDGET_USD unset -> backfill is a no-op. A budget MUST be
|
|
168
|
+
// OR <= 0 explicitly configured. The per-call
|
|
169
|
+
// llm-call.js path treats an unset
|
|
170
|
+
// budget as "uncapped"; for a bulk
|
|
171
|
+
// backfill that is unsafe -- a large
|
|
172
|
+
// memory could spend without bound. So
|
|
173
|
+
// the backfill REQUIRES a positive cap.
|
|
174
|
+
// - no API key -> backfill is a no-op (autoLink skips)
|
|
175
|
+
//
|
|
176
|
+
// The per-row autoLink call independently re-checks the SAME env gates (off /
|
|
177
|
+
// budget / key), so even mid-run the backfill respects a budget that drops to
|
|
178
|
+
// zero or a kill switch that flips. Returns aggregate counts.
|
|
179
|
+
export async function backfillAutoLink(db, opts = {}) {
|
|
180
|
+
if (!db || typeof db.prepare !== 'function') {
|
|
181
|
+
throw new Error('backfillAutoLink: db handle is invalid.');
|
|
182
|
+
}
|
|
183
|
+
const force = opts.force === true;
|
|
184
|
+
// Opt-in gate: M2 backfill only runs when explicitly enabled. M1 backfill
|
|
185
|
+
// (obsidian-parser.js) is the always-on default; M2 costs money so it is
|
|
186
|
+
// off unless the operator opts in via IJFW_AUTOLINK_BACKFILL=1.
|
|
187
|
+
if (!force && process.env.IJFW_AUTOLINK_BACKFILL !== '1') {
|
|
188
|
+
return { skipped: true, reason: 'backfill_not_enabled', rows: 0 };
|
|
189
|
+
}
|
|
190
|
+
if (process.env.IJFW_AUTOLINK_OFF === '1') {
|
|
191
|
+
return { skipped: true, reason: 'autolink_off', rows: 0 };
|
|
192
|
+
}
|
|
193
|
+
// Budget cap is MANDATORY for the backfill. An unset budget means
|
|
194
|
+
// llm-call.js runs uncapped -- fine for one-off write-time autoLink, but a
|
|
195
|
+
// bulk backfill over thousands of rows would spend without bound. Refuse
|
|
196
|
+
// unless the operator has set a positive IJFW_AUTOLINK_BUDGET_USD.
|
|
197
|
+
const budget = process.env.IJFW_AUTOLINK_BUDGET_USD;
|
|
198
|
+
if (budget === undefined || !(Number(budget) > 0)) {
|
|
199
|
+
return {
|
|
200
|
+
skipped: true,
|
|
201
|
+
reason: budget === undefined ? 'budget_not_set' : 'budget_exhausted',
|
|
202
|
+
rows: 0,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
const hasKey = !!(process.env.IJFW_AUTOLINK_API_KEY || process.env.ANTHROPIC_API_KEY);
|
|
206
|
+
if (!hasKey) {
|
|
207
|
+
return { skipped: true, reason: 'no_key', rows: 0 };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const batchSize = Math.max(1, opts.batchSize || 200);
|
|
211
|
+
const result = {
|
|
212
|
+
skipped: false, rows: 0, linked: 0, links_added: 0,
|
|
213
|
+
neighbor_tags_added: 0, stopped_early: false,
|
|
214
|
+
};
|
|
215
|
+
let lastId = 0;
|
|
216
|
+
// eslint-disable-next-line no-constant-condition
|
|
217
|
+
while (true) {
|
|
218
|
+
let batch;
|
|
219
|
+
try {
|
|
220
|
+
batch = db
|
|
221
|
+
.prepare(
|
|
222
|
+
'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
|
|
223
|
+
)
|
|
224
|
+
.all(lastId, batchSize);
|
|
225
|
+
} catch {
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
if (!batch || batch.length === 0) break;
|
|
229
|
+
for (const row of batch) {
|
|
230
|
+
lastId = row.id;
|
|
231
|
+
if (typeof row.body !== 'string' || row.body.length === 0) continue;
|
|
232
|
+
// Per-row re-check: a budget that drops to zero or a kill switch that
|
|
233
|
+
// flips mid-run stops the backfill before the next paid call. autoLink
|
|
234
|
+
// itself ALSO re-checks, but stopping here avoids the wasted SELECT.
|
|
235
|
+
if (process.env.IJFW_AUTOLINK_OFF === '1') {
|
|
236
|
+
result.stopped_early = true;
|
|
237
|
+
return result;
|
|
238
|
+
}
|
|
239
|
+
const b = process.env.IJFW_AUTOLINK_BUDGET_USD;
|
|
240
|
+
if (b === undefined || !(Number(b) > 0)) {
|
|
241
|
+
result.stopped_early = true;
|
|
242
|
+
return result;
|
|
243
|
+
}
|
|
244
|
+
result.rows += 1;
|
|
245
|
+
let res;
|
|
246
|
+
try {
|
|
247
|
+
res = await autoLink(db, { id: row.id, body: row.body });
|
|
248
|
+
} catch {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if (res && res.skipped) {
|
|
252
|
+
// autoLink skipped (budget exhausted / off / no key / parse fail).
|
|
253
|
+
// budget_exhausted + autolink_off mean stop the whole run.
|
|
254
|
+
if (res.reason === 'budget_exhausted' || res.reason === 'autolink_off') {
|
|
255
|
+
result.stopped_early = true;
|
|
256
|
+
return result;
|
|
257
|
+
}
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
result.linked += 1;
|
|
261
|
+
if (res && res.applied) {
|
|
262
|
+
result.links_added += res.applied.links_added || 0;
|
|
263
|
+
result.neighbor_tags_added += res.applied.neighbor_tags_added || 0;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (batch.length < batchSize) break;
|
|
267
|
+
}
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export default { autoLink, backfillAutoLink };
|
package/src/memory/benchmark.js
CHANGED
|
@@ -37,10 +37,11 @@
|
|
|
37
37
|
// stale_visible_with_flag: bool }
|
|
38
38
|
// sanity proof the warm filter still gates.
|
|
39
39
|
//
|
|
40
|
-
// What this harness does NOT do (yet --
|
|
40
|
+
// What this harness does NOT do (yet -- folded into the v1.6.0 IJFW Brain
|
|
41
|
+
// workstream alongside the cold-tier vector index):
|
|
41
42
|
// - cross-tier promotion timing (hot->warm happens at first search; warm
|
|
42
|
-
// never promotes to cold without a model).
|
|
43
|
-
// bi-temporal + decay-on-retrieval axes.
|
|
43
|
+
// never promotes to cold without a model). The v1.6.0 Brain work owns
|
|
44
|
+
// the bi-temporal + decay-on-retrieval axes.
|
|
44
45
|
// - multi-writer throughput. Single-writer is the published norm because
|
|
45
46
|
// SQLite's BEGIN IMMEDIATE queue dominates; that's already covered by
|
|
46
47
|
// test-memory-fts5.js's concurrent-writers test.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// IJFW v1.5.2.1 -- fs-layout migration 001: visible ijfw/ layer.
|
|
2
|
+
//
|
|
3
|
+
// Lives in src/memory/layout-migrations/ (NOT src/memory/migrations/). This
|
|
4
|
+
// directory is reserved for filesystem-layout migrations — they reshape on-disk
|
|
5
|
+
// directory layout and track version via sentinel files (see
|
|
6
|
+
// brain/layout-sentinel.js), NOT via SQLite user_version. The SQL
|
|
7
|
+
// migration-runner deliberately rejects files declaring SQL=false (F3 root
|
|
8
|
+
// cause: when SQL and fs-layout migrations coexist, an accidental copy-paste
|
|
9
|
+
// can brick schema migrations). These files are statically registered in
|
|
10
|
+
// layout-migrations/index.js and invoked by server.js at startup.
|
|
11
|
+
//
|
|
12
|
+
// Trident F-B3 safety:
|
|
13
|
+
// - acquires withLayoutLock (serializes concurrent migrations)
|
|
14
|
+
// - freshness gate refuses if any .md mtime < 30s old (concurrent writer)
|
|
15
|
+
// - sentinel flipped LAST so a crash mid-copy leaves v1 + a recoverable retry
|
|
16
|
+
// - copy-not-move keeps legacy .ijfw/ paths intact for one-version fallback
|
|
17
|
+
|
|
18
|
+
import {
|
|
19
|
+
existsSync, mkdirSync, statSync, readdirSync, cpSync,
|
|
20
|
+
} from 'node:fs';
|
|
21
|
+
import { join } from 'node:path';
|
|
22
|
+
import {
|
|
23
|
+
readLayoutVersion, writeLayoutVersion, withLayoutLock,
|
|
24
|
+
} from '../../brain/layout-sentinel.js';
|
|
25
|
+
|
|
26
|
+
const FRESHNESS_MS = 30_000;
|
|
27
|
+
|
|
28
|
+
const SCAFFOLD_DIRS = [
|
|
29
|
+
['ijfw', 'dump', 'inbox'],
|
|
30
|
+
['ijfw', 'dump', 'processed'],
|
|
31
|
+
['ijfw', 'wiki', 'concepts'],
|
|
32
|
+
['ijfw', 'wiki', 'entities'],
|
|
33
|
+
['ijfw', 'wiki', 'decisions'],
|
|
34
|
+
['ijfw', 'wiki', 'milestones'],
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
function walkMd(dir) {
|
|
38
|
+
if (!existsSync(dir)) return [];
|
|
39
|
+
const out = [];
|
|
40
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
41
|
+
const p = join(dir, entry.name);
|
|
42
|
+
if (entry.isDirectory()) out.push(...walkMd(p));
|
|
43
|
+
else if (entry.isFile() && entry.name.endsWith('.md')) out.push(p);
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function findFreshFiles(repoRoot) {
|
|
49
|
+
const cutoff = Date.now() - FRESHNESS_MS;
|
|
50
|
+
const candidates = [
|
|
51
|
+
...walkMd(join(repoRoot, '.ijfw', 'memory')),
|
|
52
|
+
...walkMd(join(repoRoot, '.ijfw', 'sessions')),
|
|
53
|
+
];
|
|
54
|
+
return candidates.filter((p) => statSync(p).mtimeMs >= cutoff);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const DESCRIPTION =
|
|
58
|
+
'fs-layout v2 -- visible ijfw/ + scaffolded dump/wiki dirs (NOT a SQL migration)';
|
|
59
|
+
|
|
60
|
+
export async function up(repoRoot) {
|
|
61
|
+
if (readLayoutVersion(repoRoot) >= 2) {
|
|
62
|
+
return { skipped: true, reason: 'already-migrated' };
|
|
63
|
+
}
|
|
64
|
+
return await withLayoutLock(repoRoot, async () => {
|
|
65
|
+
if (readLayoutVersion(repoRoot) >= 2) {
|
|
66
|
+
return { skipped: true, reason: 'already-migrated' };
|
|
67
|
+
}
|
|
68
|
+
// F4: freshness gate runs INSIDE the lock so a writer cannot sneak in
|
|
69
|
+
// between gate-pass and lock-acquire. The lock holds the freshness
|
|
70
|
+
// contract for the entire copy phase.
|
|
71
|
+
const freshFiles = findFreshFiles(repoRoot);
|
|
72
|
+
if (freshFiles.length > 0) {
|
|
73
|
+
// v1.5.2.1 F2: observability — surface the deferral to stderr so an
|
|
74
|
+
// operator running `ijfw doctor` or watching server logs can see why
|
|
75
|
+
// the visible layer hasn't materialised yet. Silent skip leaves the
|
|
76
|
+
// operator wondering what happened.
|
|
77
|
+
try {
|
|
78
|
+
process.stderr.write(
|
|
79
|
+
`[ijfw layout-migrate] deferred: ${freshFiles.length} file(s) written ` +
|
|
80
|
+
`< ${FRESHNESS_MS}ms ago in .ijfw/memory or .ijfw/sessions; ` +
|
|
81
|
+
`will retry on next server start\n`
|
|
82
|
+
);
|
|
83
|
+
} catch { /* stderr may be detached */ }
|
|
84
|
+
return { skipped: true, reason: 'fresh-writes-detected', freshFiles };
|
|
85
|
+
}
|
|
86
|
+
// v1.5.2.1 F4: detect operator downgrade. If sentinel is 1 but the visible
|
|
87
|
+
// layer destination already has .md content, the operator probably flipped
|
|
88
|
+
// .ijfw/.layout-version back to 1 manually (e.g. attempting downgrade to
|
|
89
|
+
// v1.5.1). cpSync({force:false}) would silently skip the existing files
|
|
90
|
+
// and leave drift between the two layers. Refuse, log, keep sentinel at 1
|
|
91
|
+
// so the operator resolves the conflict manually.
|
|
92
|
+
const memoryDst = join(repoRoot, 'ijfw', 'memory');
|
|
93
|
+
const sessionsDst = join(repoRoot, 'ijfw', 'sessions');
|
|
94
|
+
if (walkMd(memoryDst).length > 0 || walkMd(sessionsDst).length > 0) {
|
|
95
|
+
try {
|
|
96
|
+
process.stderr.write(
|
|
97
|
+
`[ijfw layout-migrate] aborted: visible layer (ijfw/memory or ijfw/sessions) ` +
|
|
98
|
+
`already populated but sentinel=1 (downgrade detected). Resolve manually, ` +
|
|
99
|
+
`then set .ijfw/.layout-version to 2.\n`
|
|
100
|
+
);
|
|
101
|
+
} catch { /* stderr may be detached */ }
|
|
102
|
+
return { skipped: true, reason: 'downgrade-conflict' };
|
|
103
|
+
}
|
|
104
|
+
let copiedFiles = 0;
|
|
105
|
+
// FLAG-4: force:false preserves any user-authored content already at the
|
|
106
|
+
// visible-layer destination (e.g. operator following the README's
|
|
107
|
+
// "commit ijfw/ to git" advice before migration runs). errorOnExist:false
|
|
108
|
+
// means existing destination files cause the COPY of THAT file to skip
|
|
109
|
+
// silently, but the rest of the tree still copies. Behaviour: union of
|
|
110
|
+
// existing visible files (winner) + legacy hidden files (filler).
|
|
111
|
+
const memorySrc = join(repoRoot, '.ijfw', 'memory');
|
|
112
|
+
if (existsSync(memorySrc)) {
|
|
113
|
+
cpSync(memorySrc, memoryDst,
|
|
114
|
+
{ recursive: true, force: false, errorOnExist: false });
|
|
115
|
+
copiedFiles += walkMd(memorySrc).length;
|
|
116
|
+
}
|
|
117
|
+
const sessionsSrc = join(repoRoot, '.ijfw', 'sessions');
|
|
118
|
+
if (existsSync(sessionsSrc)) {
|
|
119
|
+
cpSync(sessionsSrc, sessionsDst,
|
|
120
|
+
{ recursive: true, force: false, errorOnExist: false });
|
|
121
|
+
copiedFiles += walkMd(sessionsSrc).length;
|
|
122
|
+
}
|
|
123
|
+
for (const parts of SCAFFOLD_DIRS) {
|
|
124
|
+
mkdirSync(join(repoRoot, ...parts), { recursive: true });
|
|
125
|
+
}
|
|
126
|
+
writeLayoutVersion(repoRoot, 2);
|
|
127
|
+
return { skipped: false, version: 2, copiedFiles, scaffoldedDirs: SCAFFOLD_DIRS.length };
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export default { description: DESCRIPTION, up };
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// mcp-server/src/memory/layout-migrations/index.js
|
|
2
|
+
//
|
|
3
|
+
// Filesystem-layout migrations. These are NOT SQL migrations — they reshape
|
|
4
|
+
// on-disk directory layout (e.g., copying internal .ijfw/memory/ to visible
|
|
5
|
+
// ijfw/memory/) and track their version via sentinel files, not SQLite
|
|
6
|
+
// user_version. They live in a sibling directory to migrations/ so the SQL
|
|
7
|
+
// migration runner cannot accidentally load them (root cause of F1 in v1.5.2.1).
|
|
8
|
+
//
|
|
9
|
+
// To add a new fs-layout migration:
|
|
10
|
+
// 1. Create NNN-foo.js in this directory exporting DESCRIPTION + up(repoRoot).
|
|
11
|
+
// 2. Import it below and add it to LAYOUT_MIGRATIONS in version order.
|
|
12
|
+
// 3. The server entry-point invokes runLayoutMigrations(repoRoot) at startup.
|
|
13
|
+
|
|
14
|
+
import * as visibleLayer from './001-visible-layer.js';
|
|
15
|
+
|
|
16
|
+
// L-1 (Lens 1): defense-in-depth deep freeze. ESM namespace objects ARE
|
|
17
|
+
// already frozen by spec (Node throws TypeError if you Object.freeze them),
|
|
18
|
+
// so we rewrap each namespace import in a plain object literal exposing the
|
|
19
|
+
// minimal contract, then freeze that. Future refactors that swap a
|
|
20
|
+
// namespace import for a class instance or factory keep the immutability
|
|
21
|
+
// invariant rather than silently going mutable.
|
|
22
|
+
function wrap(ns) {
|
|
23
|
+
return Object.freeze({
|
|
24
|
+
DESCRIPTION: ns.DESCRIPTION,
|
|
25
|
+
up: ns.up,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
export const LAYOUT_MIGRATIONS = Object.freeze([wrap(visibleLayer)]);
|
|
29
|
+
|
|
30
|
+
// L-2 (Lens 1): per-migration try/catch so one failing migration cannot
|
|
31
|
+
// abort subsequent independent ones. Caller decides whether to bail on
|
|
32
|
+
// failures (server.js __mainEntryPoint logs each failure to stderr).
|
|
33
|
+
// Returns an array of { description, ok, result?, error? } in invocation
|
|
34
|
+
// order — preserved even on partial failure for debugging.
|
|
35
|
+
export async function runLayoutMigrations(repoRoot) {
|
|
36
|
+
const results = [];
|
|
37
|
+
for (const m of LAYOUT_MIGRATIONS) {
|
|
38
|
+
try {
|
|
39
|
+
const result = await m.up(repoRoot);
|
|
40
|
+
results.push({ description: m.DESCRIPTION, ok: true, result });
|
|
41
|
+
} catch (err) {
|
|
42
|
+
results.push({
|
|
43
|
+
description: m.DESCRIPTION,
|
|
44
|
+
ok: false,
|
|
45
|
+
error: err && err.message ? err.message : String(err),
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return results;
|
|
50
|
+
}
|
|
@@ -27,9 +27,22 @@ export class SchemaVersionError extends Error {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
// Discover and load every migration module under ./migrations/, sorted by
|
|
30
|
-
// numeric prefix ascending. Each module
|
|
31
|
-
// DESCRIPTION (string), and up(db) (function).
|
|
32
|
-
|
|
30
|
+
// numeric prefix ascending. Each module MUST export VERSION (number),
|
|
31
|
+
// DESCRIPTION (string), and up(db) (function). The filename's numeric prefix
|
|
32
|
+
// MUST equal the exported VERSION (enforced below).
|
|
33
|
+
//
|
|
34
|
+
// SQL=false is REJECTED — fs-layout migrations live in ../layout-migrations/
|
|
35
|
+
// (see v1.5.2.1 F3). The runner used to silently skip SQL=false; that escape
|
|
36
|
+
// hatch was a runtime workaround for a directory-structure mistake and is
|
|
37
|
+
// gone. If a file in this directory declares SQL=false, the runner fails
|
|
38
|
+
// loudly so the structural error is caught at startup, not at the next
|
|
39
|
+
// schema-bricking copy-paste.
|
|
40
|
+
//
|
|
41
|
+
// Exported (v1.5.1 W3.B) so search.js (and any other consumer that needs
|
|
42
|
+
// the sync migration pipeline) can reuse the SAME discovery path instead
|
|
43
|
+
// of maintaining a parallel hardcoded list. Single source of truth: drop
|
|
44
|
+
// a new NNN-name.js into ./migrations/ and every consumer sees it.
|
|
45
|
+
export async function loadMigrations() {
|
|
33
46
|
let files;
|
|
34
47
|
try {
|
|
35
48
|
files = readdirSync(MIGRATIONS_DIR);
|
|
@@ -43,9 +56,30 @@ async function loadMigrations() {
|
|
|
43
56
|
for (const f of matches) {
|
|
44
57
|
const url = pathToFileURL(join(MIGRATIONS_DIR, f)).href;
|
|
45
58
|
const mod = await import(url);
|
|
59
|
+
// v1.5.2.1 F3: SQL=false is no longer an in-directory escape hatch. The
|
|
60
|
+
// fs-layout migrations live in ../layout-migrations/ and are statically
|
|
61
|
+
// registered there. If anything in this directory still declares
|
|
62
|
+
// SQL=false it's a structural error (likely a misplaced fs-layout
|
|
63
|
+
// migration). Fail loudly rather than silently skip.
|
|
64
|
+
if (mod.SQL === false) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
`Memory migration ${f} declares SQL=false — fs-layout migrations belong in ` +
|
|
67
|
+
`layout-migrations/, not migrations/. Move the file or remove SQL=false.`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
46
70
|
if (typeof mod.VERSION !== 'number' || typeof mod.up !== 'function') {
|
|
47
71
|
throw new Error(`Memory migration ${f} is missing VERSION or up().`);
|
|
48
72
|
}
|
|
73
|
+
// v1.5.2.1 F3.2: filename numeric prefix MUST equal exported VERSION.
|
|
74
|
+
// Catches reordering / rename mistakes immediately rather than at the
|
|
75
|
+
// next user_version comparison (where the failure mode is silent).
|
|
76
|
+
const filenamePrefix = parseInt(f.match(/^(\d+)/)[1], 10);
|
|
77
|
+
if (filenamePrefix !== mod.VERSION) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
`Memory migration ${f}: filename prefix ${filenamePrefix} does not match ` +
|
|
80
|
+
`exported VERSION ${mod.VERSION}. Rename the file or fix the VERSION.`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
49
83
|
out.push({
|
|
50
84
|
file: f,
|
|
51
85
|
version: mod.VERSION,
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// IJFW v1.5.1 -- memory migration 009: M1 obsidian-index backfill.
|
|
2
|
+
//
|
|
3
|
+
// Source authority: Trident r5 finding 1.2 (HIGH).
|
|
4
|
+
//
|
|
5
|
+
// Round-4 Fix-1 (commit 3218812) wired M1 (Obsidian wikilink/tag/meta
|
|
6
|
+
// indexing -- indexObsidianRelations) and M2 (A-Mem auto-linking -- autoLink)
|
|
7
|
+
// into the production memory-write path (handleStore, search.js#autoIndex).
|
|
8
|
+
// But the fix is forward-only: every memory_entries row written during
|
|
9
|
+
// v1.5.0 -- when M1/M2 were bypassed -- has empty memory_links / memory_tags
|
|
10
|
+
// / memory_meta. An existing user upgrading to v1.5.1 got auto-linking +
|
|
11
|
+
// wikilink indexing only on NEW entries; their accumulated memory stayed
|
|
12
|
+
// un-indexed.
|
|
13
|
+
//
|
|
14
|
+
// This migration runs a ONE-TIME M1 backfill: it walks every existing
|
|
15
|
+
// memory_entries row and runs indexObsidianRelations over its body. M1 is:
|
|
16
|
+
// - free -- pure markdown parse, zero LLM / network / cost
|
|
17
|
+
// - idempotent -- indexObsidianRelations does DELETE-then-INSERT per id,
|
|
18
|
+
// so re-applying produces identical aux rows
|
|
19
|
+
// which is exactly what makes it safe to run inside a schema migration: it
|
|
20
|
+
// runs once (user_version gates re-application), deterministically, and a
|
|
21
|
+
// crash rolls the whole txn back to user_version 8.
|
|
22
|
+
//
|
|
23
|
+
// M2 (autoLink) is NOT backfilled here -- it makes one LLM call per row and
|
|
24
|
+
// can cost real money over a large memory. M2 backfill is opt-in via the
|
|
25
|
+
// `ijfw memory reindex --m2` CLI verb, which is budget-gated (respects
|
|
26
|
+
// IJFW_AUTOLINK_BUDGET_USD / IJFW_AUTOLINK_OFF / IJFW_AUTOLINK_BACKFILL).
|
|
27
|
+
//
|
|
28
|
+
// Ordering: migration 001 creates memory_entries; migration 006 creates
|
|
29
|
+
// memory_links / memory_tags / memory_meta. Both run before 009, so by the
|
|
30
|
+
// time up() executes the source table and the three aux tables all exist.
|
|
31
|
+
//
|
|
32
|
+
// Crash safety: the migration runner wraps up() in BEGIN IMMEDIATE.
|
|
33
|
+
// backfillObsidianIndex's per-row indexObsidianRelations opens a nested
|
|
34
|
+
// SQLite transaction (savepoint) -- valid inside the outer txn -- and the
|
|
35
|
+
// whole thing rolls back to user_version 8 on any failure.
|
|
36
|
+
|
|
37
|
+
import { backfillObsidianIndex } from '../obsidian-parser.js';
|
|
38
|
+
|
|
39
|
+
export const VERSION = 9;
|
|
40
|
+
export const DESCRIPTION =
|
|
41
|
+
'memory v1.5.1 -- one-time M1 obsidian-index backfill for pre-fix rows (Trident r5 1.2)';
|
|
42
|
+
|
|
43
|
+
export function up(db) {
|
|
44
|
+
// backfillObsidianIndex tolerates a missing memory_entries table (returns
|
|
45
|
+
// zero counts) so a brand-new db that jumps straight to v9 is a clean
|
|
46
|
+
// no-op -- there are no pre-fix rows to backfill on a fresh install.
|
|
47
|
+
backfillObsidianIndex(db);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export default { version: VERSION, description: DESCRIPTION, up };
|
|
@@ -84,8 +84,71 @@ export function indexObsidianRelations(db, memoryId, text) {
|
|
|
84
84
|
for (const t of parsed.tags) insTag.run(memoryId, t.path, t.depth);
|
|
85
85
|
for (const m of parsed.meta) insMeta.run(memoryId, m.key, m.value);
|
|
86
86
|
});
|
|
87
|
-
|
|
87
|
+
// F-LENS2-01: use IMMEDIATE so all facts-table sister-writers acquire the
|
|
88
|
+
// RESERVED lock in the same order — no DEFERRED→IMMEDIATE upgrade collision.
|
|
89
|
+
tx.immediate();
|
|
88
90
|
return parsed;
|
|
89
91
|
}
|
|
90
92
|
|
|
91
|
-
|
|
93
|
+
// v1.5.1 R5-1.2 -- one-time M1 backfill for memory written during v1.5.0,
|
|
94
|
+
// when indexObsidianRelations was NOT wired into the production write path.
|
|
95
|
+
// Round-4 Fix-1 (commit 3218812) wired M1+M2 into handleStore/autoIndex but
|
|
96
|
+
// forward-only: rows already in memory_entries have empty memory_links /
|
|
97
|
+
// memory_tags / memory_meta. This walks EVERY row and re-runs M1 over it.
|
|
98
|
+
//
|
|
99
|
+
// Safe to run over everything:
|
|
100
|
+
// - free -- pure markdown parse, zero LLM / network
|
|
101
|
+
// - idempotent-- indexObsidianRelations clears prior aux rows per id before
|
|
102
|
+
// re-inserting, so a re-run produces identical state
|
|
103
|
+
//
|
|
104
|
+
// The walk reads ids in batches so a very large memory_entries doesn't pin
|
|
105
|
+
// the whole table in memory; each row's indexObsidianRelations call carries
|
|
106
|
+
// its own transaction (DELETE-then-INSERT) so a single bad row never aborts
|
|
107
|
+
// the rest of the backfill.
|
|
108
|
+
//
|
|
109
|
+
// Returns { rows, links, tags, meta } -- counts re-indexed across the run.
|
|
110
|
+
export function backfillObsidianIndex(db, opts = {}) {
|
|
111
|
+
if (!db || typeof db.prepare !== 'function') {
|
|
112
|
+
throw new Error('backfillObsidianIndex: db handle is invalid.');
|
|
113
|
+
}
|
|
114
|
+
const batchSize = Math.max(1, opts.batchSize || 500);
|
|
115
|
+
const result = { rows: 0, links: 0, tags: 0, meta: 0, errors: 0 };
|
|
116
|
+
let lastId = 0;
|
|
117
|
+
// eslint-disable-next-line no-constant-condition
|
|
118
|
+
while (true) {
|
|
119
|
+
let batch;
|
|
120
|
+
try {
|
|
121
|
+
batch = db
|
|
122
|
+
.prepare(
|
|
123
|
+
'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
|
|
124
|
+
)
|
|
125
|
+
.all(lastId, batchSize);
|
|
126
|
+
} catch {
|
|
127
|
+
// memory_entries missing (fresh db before migration 001) -- nothing to do.
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
if (!batch || batch.length === 0) break;
|
|
131
|
+
for (const row of batch) {
|
|
132
|
+
lastId = row.id;
|
|
133
|
+
if (typeof row.body !== 'string' || row.body.length === 0) continue;
|
|
134
|
+
try {
|
|
135
|
+
const parsed = indexObsidianRelations(db, String(row.id), row.body);
|
|
136
|
+
result.rows += 1;
|
|
137
|
+
result.links += parsed.links.length;
|
|
138
|
+
result.tags += parsed.tags.length;
|
|
139
|
+
result.meta += parsed.meta.length;
|
|
140
|
+
} catch (e) {
|
|
141
|
+
result.errors += 1;
|
|
142
|
+
try {
|
|
143
|
+
console.error(
|
|
144
|
+
'[obsidian] backfill failed for id', row.id, ':', e?.message || e,
|
|
145
|
+
);
|
|
146
|
+
} catch { /* never throw out of the backfill */ }
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (batch.length < batchSize) break;
|
|
150
|
+
}
|
|
151
|
+
return result;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export default { parseObsidian, indexObsidianRelations, backfillObsidianIndex };
|
package/src/memory/reader.js
CHANGED
|
@@ -22,7 +22,8 @@ const PREVIEW_CHARS = 300;
|
|
|
22
22
|
/** Parse YAML-style frontmatter (key: value lines between --- fences). */
|
|
23
23
|
function parseFrontmatter(raw) {
|
|
24
24
|
const fm = { title: null, description: null, type: null };
|
|
25
|
-
const
|
|
25
|
+
const stripped = String(raw).replace(/^/, '').replace(/^\s+/, '');
|
|
26
|
+
const m = stripped.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
|
26
27
|
if (!m) return fm;
|
|
27
28
|
for (const line of m[1].split('\n')) {
|
|
28
29
|
const kv = line.match(/^(\w+):\s*(.+)/);
|