@ijfw/memory-server 1.5.0 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/bin/ijfw-memorize +14 -7
  2. package/fixtures/team/book.json +6 -6
  3. package/fixtures/team/business.json +146 -20
  4. package/fixtures/team/content.json +6 -6
  5. package/fixtures/team/design.json +148 -20
  6. package/fixtures/team/mixed.json +206 -27
  7. package/fixtures/team/research.json +146 -20
  8. package/fixtures/team/software.json +148 -20
  9. package/package.json +8 -4
  10. package/src/brain/budget-guard.js +86 -0
  11. package/src/brain/citation-resolver.js +41 -0
  12. package/src/brain/context-injection.js +69 -0
  13. package/src/brain/discovery.js +83 -0
  14. package/src/brain/dream-pipeline.js +324 -0
  15. package/src/brain/dump-ingest.js +88 -0
  16. package/src/brain/entity-collapse.js +28 -0
  17. package/src/brain/export.js +112 -0
  18. package/src/brain/extractors/index.js +24 -0
  19. package/src/brain/extractors/markdown.js +27 -0
  20. package/src/brain/extractors/pdf.js +31 -0
  21. package/src/brain/extractors/transcript.js +38 -0
  22. package/src/brain/first-run-scan.js +61 -0
  23. package/src/brain/index.js +1 -0
  24. package/src/brain/layout-sentinel.js +29 -0
  25. package/src/brain/migrate-facts-internal-once.js +87 -0
  26. package/src/brain/path-guard.js +103 -0
  27. package/src/brain/paths.js +26 -0
  28. package/src/brain/promotion-suggester.js +41 -0
  29. package/src/brain/stub-detector.js +33 -0
  30. package/src/brain/tiered-llm.js +83 -0
  31. package/src/brain/wiki-compiler.js +144 -0
  32. package/src/brain/wiki-sentinels.js +45 -0
  33. package/src/brain/wiki-templates.js +94 -0
  34. package/src/cross-orchestrator-cli.js +336 -150
  35. package/src/cross-orchestrator.js +52 -3
  36. package/src/dashboard-server.js +1 -1
  37. package/src/dispatch/extension.js +1 -1
  38. package/src/dream/runner.mjs +21 -0
  39. package/src/extension-registry.js +2 -2
  40. package/src/handlers/brain-handler.js +319 -0
  41. package/src/hardware-signer.js +4 -2
  42. package/src/lib/ui-review-runner.js +48 -7
  43. package/src/memory/auto-linker.js +121 -2
  44. package/src/memory/benchmark.js +4 -3
  45. package/src/memory/layout-migrations/001-visible-layer.js +131 -0
  46. package/src/memory/layout-migrations/index.js +50 -0
  47. package/src/memory/migration-runner.js +37 -3
  48. package/src/memory/migrations/009-obsidian-backfill.js +50 -0
  49. package/src/memory/obsidian-parser.js +65 -2
  50. package/src/memory/reader.js +2 -1
  51. package/src/memory/search.js +190 -41
  52. package/src/memory/temporal.js +40 -1
  53. package/src/orchestrator/agents-md-blackboard.js +114 -1
  54. package/src/orchestrator/debug-trident-trigger.js +374 -0
  55. package/src/orchestrator/discipline-selector.js +276 -0
  56. package/src/orchestrator/merge-block-aware.js +15 -5
  57. package/src/orchestrator/post-done-runner.js +36 -8
  58. package/src/orchestrator/state-sdk.js +216 -10
  59. package/src/orchestrator/subagent-telemetry.js +19 -0
  60. package/src/orchestrator/wave-state.js +38 -0
  61. package/src/override-resolver.js +5 -3
  62. package/src/recovery/code-fixer.js +311 -6
  63. package/src/runtime-mediator.js +0 -1
  64. package/src/server.js +486 -132
  65. package/src/swarm-config.js +30 -22
  66. package/src/team/domain-templates/business.json +4 -1
  67. package/src/team/domain-templates/research.json +4 -1
  68. package/src/team/generator.js +162 -0
  69. package/src/update-apply.js +1 -1
  70. package/src/dashboard-charts.js +0 -239
  71. package/src/orchestrator/runtime-loop.js +0 -430
@@ -101,7 +101,11 @@ function applyProposal(db, entry, proposal) {
101
101
  }
102
102
  }
103
103
  });
104
- tx();
104
+ // F-LENS2-01: use IMMEDIATE so all facts-table sister-writers acquire the
105
+ // RESERVED lock in the same order — no DEFERRED→IMMEDIATE upgrade collision
106
+ // when another writer (storeFactBitemporal, dream-pipeline, obsidian-parser)
107
+ // holds the lock.
108
+ tx.immediate();
105
109
  return { links_added: linksAdded, neighbor_tags_added: neighborTagsAdded };
106
110
  }
107
111
 
@@ -149,4 +153,119 @@ export async function autoLink(db, entry, opts = {}) {
149
153
  return { skipped: false, neighbors, proposal, applied };
150
154
  }
151
155
 
152
- export default { autoLink };
156
+ // v1.5.1 R5-1.2 -- one-time M2 (A-Mem auto-link) backfill for memory written
157
+ // during v1.5.0, when autoLink was NOT wired into the production write path.
158
+ //
159
+ // UNLIKE the M1 backfill (free, always-on), M2 backfill makes one LLM call
160
+ // per row -- backfilling over a large memory can cost real money. So M2
161
+ // backfill is OPT-IN and budget-gated:
162
+ //
163
+ // - IJFW_AUTOLINK_OFF=1 -> backfill is a no-op (kill switch)
164
+ // - IJFW_AUTOLINK_BACKFILL!=1 -> backfill is a no-op by default
165
+ // (M1-always, M2-opt-in is the safe
166
+ // default per R5-1.2)
167
+ // - IJFW_AUTOLINK_BUDGET_USD unset -> backfill is a no-op. A budget MUST be
168
+ // OR <= 0 explicitly configured. The per-call
169
+ // llm-call.js path treats an unset
170
+ // budget as "uncapped"; for a bulk
171
+ // backfill that is unsafe -- a large
172
+ // memory could spend without bound. So
173
+ // the backfill REQUIRES a positive cap.
174
+ // - no API key -> backfill is a no-op (autoLink skips)
175
+ //
176
+ // The per-row autoLink call independently re-checks the SAME env gates (off /
177
+ // budget / key), so even mid-run the backfill respects a budget that drops to
178
+ // zero or a kill switch that flips. Returns aggregate counts.
179
+ export async function backfillAutoLink(db, opts = {}) {
180
+ if (!db || typeof db.prepare !== 'function') {
181
+ throw new Error('backfillAutoLink: db handle is invalid.');
182
+ }
183
+ const force = opts.force === true;
184
+ // Opt-in gate: M2 backfill only runs when explicitly enabled. M1 backfill
185
+ // (obsidian-parser.js) is the always-on default; M2 costs money so it is
186
+ // off unless the operator opts in via IJFW_AUTOLINK_BACKFILL=1.
187
+ if (!force && process.env.IJFW_AUTOLINK_BACKFILL !== '1') {
188
+ return { skipped: true, reason: 'backfill_not_enabled', rows: 0 };
189
+ }
190
+ if (process.env.IJFW_AUTOLINK_OFF === '1') {
191
+ return { skipped: true, reason: 'autolink_off', rows: 0 };
192
+ }
193
+ // Budget cap is MANDATORY for the backfill. An unset budget means
194
+ // llm-call.js runs uncapped -- fine for one-off write-time autoLink, but a
195
+ // bulk backfill over thousands of rows would spend without bound. Refuse
196
+ // unless the operator has set a positive IJFW_AUTOLINK_BUDGET_USD.
197
+ const budget = process.env.IJFW_AUTOLINK_BUDGET_USD;
198
+ if (budget === undefined || !(Number(budget) > 0)) {
199
+ return {
200
+ skipped: true,
201
+ reason: budget === undefined ? 'budget_not_set' : 'budget_exhausted',
202
+ rows: 0,
203
+ };
204
+ }
205
+ const hasKey = !!(process.env.IJFW_AUTOLINK_API_KEY || process.env.ANTHROPIC_API_KEY);
206
+ if (!hasKey) {
207
+ return { skipped: true, reason: 'no_key', rows: 0 };
208
+ }
209
+
210
+ const batchSize = Math.max(1, opts.batchSize || 200);
211
+ const result = {
212
+ skipped: false, rows: 0, linked: 0, links_added: 0,
213
+ neighbor_tags_added: 0, stopped_early: false,
214
+ };
215
+ let lastId = 0;
216
+ // eslint-disable-next-line no-constant-condition
217
+ while (true) {
218
+ let batch;
219
+ try {
220
+ batch = db
221
+ .prepare(
222
+ 'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
223
+ )
224
+ .all(lastId, batchSize);
225
+ } catch {
226
+ break;
227
+ }
228
+ if (!batch || batch.length === 0) break;
229
+ for (const row of batch) {
230
+ lastId = row.id;
231
+ if (typeof row.body !== 'string' || row.body.length === 0) continue;
232
+ // Per-row re-check: a budget that drops to zero or a kill switch that
233
+ // flips mid-run stops the backfill before the next paid call. autoLink
234
+ // itself ALSO re-checks, but stopping here avoids the wasted SELECT.
235
+ if (process.env.IJFW_AUTOLINK_OFF === '1') {
236
+ result.stopped_early = true;
237
+ return result;
238
+ }
239
+ const b = process.env.IJFW_AUTOLINK_BUDGET_USD;
240
+ if (b === undefined || !(Number(b) > 0)) {
241
+ result.stopped_early = true;
242
+ return result;
243
+ }
244
+ result.rows += 1;
245
+ let res;
246
+ try {
247
+ res = await autoLink(db, { id: row.id, body: row.body });
248
+ } catch {
249
+ continue;
250
+ }
251
+ if (res && res.skipped) {
252
+ // autoLink skipped (budget exhausted / off / no key / parse fail).
253
+ // budget_exhausted + autolink_off mean stop the whole run.
254
+ if (res.reason === 'budget_exhausted' || res.reason === 'autolink_off') {
255
+ result.stopped_early = true;
256
+ return result;
257
+ }
258
+ continue;
259
+ }
260
+ result.linked += 1;
261
+ if (res && res.applied) {
262
+ result.links_added += res.applied.links_added || 0;
263
+ result.neighbor_tags_added += res.applied.neighbor_tags_added || 0;
264
+ }
265
+ }
266
+ if (batch.length < batchSize) break;
267
+ }
268
+ return result;
269
+ }
270
+
271
+ export default { autoLink, backfillAutoLink };
@@ -37,10 +37,11 @@
37
37
  // stale_visible_with_flag: bool }
38
38
  // sanity proof the warm filter still gates.
39
39
  //
40
- // What this harness does NOT do (yet -- on the v1.5.0 backlog):
40
+ // What this harness does NOT do (yet -- folded into the v1.6.0 IJFW Brain
41
+ // workstream alongside the cold-tier vector index):
41
42
  // - cross-tier promotion timing (hot->warm happens at first search; warm
42
- // never promotes to cold without a model). Future T23+ work owns the
43
- // bi-temporal + decay-on-retrieval axes.
43
+ // never promotes to cold without a model). The v1.6.0 Brain work owns
44
+ // the bi-temporal + decay-on-retrieval axes.
44
45
  // - multi-writer throughput. Single-writer is the published norm because
45
46
  // SQLite's BEGIN IMMEDIATE queue dominates; that's already covered by
46
47
  // test-memory-fts5.js's concurrent-writers test.
@@ -0,0 +1,131 @@
1
+ // IJFW v1.5.2.1 -- fs-layout migration 001: visible ijfw/ layer.
2
+ //
3
+ // Lives in src/memory/layout-migrations/ (NOT src/memory/migrations/). This
4
+ // directory is reserved for filesystem-layout migrations — they reshape on-disk
5
+ // directory layout and track version via sentinel files (see
6
+ // brain/layout-sentinel.js), NOT via SQLite user_version. The SQL
7
+ // migration-runner deliberately rejects files declaring SQL=false (F3 root
8
+ // cause: when SQL and fs-layout migrations coexist, an accidental copy-paste
9
+ // can brick schema migrations). These files are statically registered in
10
+ // layout-migrations/index.js and invoked by server.js at startup.
11
+ //
12
+ // Trident F-B3 safety:
13
+ // - acquires withLayoutLock (serializes concurrent migrations)
14
+ // - freshness gate refuses if any .md mtime < 30s old (concurrent writer)
15
+ // - sentinel flipped LAST so a crash mid-copy leaves v1 + a recoverable retry
16
+ // - copy-not-move keeps legacy .ijfw/ paths intact for one-version fallback
17
+
18
+ import {
19
+ existsSync, mkdirSync, statSync, readdirSync, cpSync,
20
+ } from 'node:fs';
21
+ import { join } from 'node:path';
22
+ import {
23
+ readLayoutVersion, writeLayoutVersion, withLayoutLock,
24
+ } from '../../brain/layout-sentinel.js';
25
+
26
+ const FRESHNESS_MS = 30_000;
27
+
28
+ const SCAFFOLD_DIRS = [
29
+ ['ijfw', 'dump', 'inbox'],
30
+ ['ijfw', 'dump', 'processed'],
31
+ ['ijfw', 'wiki', 'concepts'],
32
+ ['ijfw', 'wiki', 'entities'],
33
+ ['ijfw', 'wiki', 'decisions'],
34
+ ['ijfw', 'wiki', 'milestones'],
35
+ ];
36
+
37
+ function walkMd(dir) {
38
+ if (!existsSync(dir)) return [];
39
+ const out = [];
40
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
41
+ const p = join(dir, entry.name);
42
+ if (entry.isDirectory()) out.push(...walkMd(p));
43
+ else if (entry.isFile() && entry.name.endsWith('.md')) out.push(p);
44
+ }
45
+ return out;
46
+ }
47
+
48
+ function findFreshFiles(repoRoot) {
49
+ const cutoff = Date.now() - FRESHNESS_MS;
50
+ const candidates = [
51
+ ...walkMd(join(repoRoot, '.ijfw', 'memory')),
52
+ ...walkMd(join(repoRoot, '.ijfw', 'sessions')),
53
+ ];
54
+ return candidates.filter((p) => statSync(p).mtimeMs >= cutoff);
55
+ }
56
+
57
+ export const DESCRIPTION =
58
+ 'fs-layout v2 -- visible ijfw/ + scaffolded dump/wiki dirs (NOT a SQL migration)';
59
+
60
+ export async function up(repoRoot) {
61
+ if (readLayoutVersion(repoRoot) >= 2) {
62
+ return { skipped: true, reason: 'already-migrated' };
63
+ }
64
+ return await withLayoutLock(repoRoot, async () => {
65
+ if (readLayoutVersion(repoRoot) >= 2) {
66
+ return { skipped: true, reason: 'already-migrated' };
67
+ }
68
+ // F4: freshness gate runs INSIDE the lock so a writer cannot sneak in
69
+ // between gate-pass and lock-acquire. The lock holds the freshness
70
+ // contract for the entire copy phase.
71
+ const freshFiles = findFreshFiles(repoRoot);
72
+ if (freshFiles.length > 0) {
73
+ // v1.5.2.1 F2: observability — surface the deferral to stderr so an
74
+ // operator running `ijfw doctor` or watching server logs can see why
75
+ // the visible layer hasn't materialised yet. Silent skip leaves the
76
+ // operator wondering what happened.
77
+ try {
78
+ process.stderr.write(
79
+ `[ijfw layout-migrate] deferred: ${freshFiles.length} file(s) written ` +
80
+ `< ${FRESHNESS_MS}ms ago in .ijfw/memory or .ijfw/sessions; ` +
81
+ `will retry on next server start\n`
82
+ );
83
+ } catch { /* stderr may be detached */ }
84
+ return { skipped: true, reason: 'fresh-writes-detected', freshFiles };
85
+ }
86
+ // v1.5.2.1 F4: detect operator downgrade. If sentinel is 1 but the visible
87
+ // layer destination already has .md content, the operator probably flipped
88
+ // .ijfw/.layout-version back to 1 manually (e.g. attempting downgrade to
89
+ // v1.5.1). cpSync({force:false}) would silently skip the existing files
90
+ // and leave drift between the two layers. Refuse, log, keep sentinel at 1
91
+ // so the operator resolves the conflict manually.
92
+ const memoryDst = join(repoRoot, 'ijfw', 'memory');
93
+ const sessionsDst = join(repoRoot, 'ijfw', 'sessions');
94
+ if (walkMd(memoryDst).length > 0 || walkMd(sessionsDst).length > 0) {
95
+ try {
96
+ process.stderr.write(
97
+ `[ijfw layout-migrate] aborted: visible layer (ijfw/memory or ijfw/sessions) ` +
98
+ `already populated but sentinel=1 (downgrade detected). Resolve manually, ` +
99
+ `then set .ijfw/.layout-version to 2.\n`
100
+ );
101
+ } catch { /* stderr may be detached */ }
102
+ return { skipped: true, reason: 'downgrade-conflict' };
103
+ }
104
+ let copiedFiles = 0;
105
+ // FLAG-4: force:false preserves any user-authored content already at the
106
+ // visible-layer destination (e.g. operator following the README's
107
+ // "commit ijfw/ to git" advice before migration runs). errorOnExist:false
108
+ // means existing destination files cause the COPY of THAT file to skip
109
+ // silently, but the rest of the tree still copies. Behaviour: union of
110
+ // existing visible files (winner) + legacy hidden files (filler).
111
+ const memorySrc = join(repoRoot, '.ijfw', 'memory');
112
+ if (existsSync(memorySrc)) {
113
+ cpSync(memorySrc, memoryDst,
114
+ { recursive: true, force: false, errorOnExist: false });
115
+ copiedFiles += walkMd(memorySrc).length;
116
+ }
117
+ const sessionsSrc = join(repoRoot, '.ijfw', 'sessions');
118
+ if (existsSync(sessionsSrc)) {
119
+ cpSync(sessionsSrc, sessionsDst,
120
+ { recursive: true, force: false, errorOnExist: false });
121
+ copiedFiles += walkMd(sessionsSrc).length;
122
+ }
123
+ for (const parts of SCAFFOLD_DIRS) {
124
+ mkdirSync(join(repoRoot, ...parts), { recursive: true });
125
+ }
126
+ writeLayoutVersion(repoRoot, 2);
127
+ return { skipped: false, version: 2, copiedFiles, scaffoldedDirs: SCAFFOLD_DIRS.length };
128
+ });
129
+ }
130
+
131
+ export default { description: DESCRIPTION, up };
@@ -0,0 +1,50 @@
1
+ // mcp-server/src/memory/layout-migrations/index.js
2
+ //
3
+ // Filesystem-layout migrations. These are NOT SQL migrations — they reshape
4
+ // on-disk directory layout (e.g., copying internal .ijfw/memory/ to visible
5
+ // ijfw/memory/) and track their version via sentinel files, not SQLite
6
+ // user_version. They live in a sibling directory to migrations/ so the SQL
7
+ // migration runner cannot accidentally load them (root cause of F1 in v1.5.2.1).
8
+ //
9
+ // To add a new fs-layout migration:
10
+ // 1. Create NNN-foo.js in this directory exporting DESCRIPTION + up(repoRoot).
11
+ // 2. Import it below and add it to LAYOUT_MIGRATIONS in version order.
12
+ // 3. The server entry-point invokes runLayoutMigrations(repoRoot) at startup.
13
+
14
+ import * as visibleLayer from './001-visible-layer.js';
15
+
16
+ // L-1 (Lens 1): defense-in-depth deep freeze. ESM namespace objects ARE
17
+ // already frozen by spec (Node throws TypeError if you Object.freeze them),
18
+ // so we rewrap each namespace import in a plain object literal exposing the
19
+ // minimal contract, then freeze that. Future refactors that swap a
20
+ // namespace import for a class instance or factory keep the immutability
21
+ // invariant rather than silently going mutable.
22
+ function wrap(ns) {
23
+ return Object.freeze({
24
+ DESCRIPTION: ns.DESCRIPTION,
25
+ up: ns.up,
26
+ });
27
+ }
28
+ export const LAYOUT_MIGRATIONS = Object.freeze([wrap(visibleLayer)]);
29
+
30
+ // L-2 (Lens 1): per-migration try/catch so one failing migration cannot
31
+ // abort subsequent independent ones. Caller decides whether to bail on
32
+ // failures (server.js __mainEntryPoint logs each failure to stderr).
33
+ // Returns an array of { description, ok, result?, error? } in invocation
34
+ // order — preserved even on partial failure for debugging.
35
+ export async function runLayoutMigrations(repoRoot) {
36
+ const results = [];
37
+ for (const m of LAYOUT_MIGRATIONS) {
38
+ try {
39
+ const result = await m.up(repoRoot);
40
+ results.push({ description: m.DESCRIPTION, ok: true, result });
41
+ } catch (err) {
42
+ results.push({
43
+ description: m.DESCRIPTION,
44
+ ok: false,
45
+ error: err && err.message ? err.message : String(err),
46
+ });
47
+ }
48
+ }
49
+ return results;
50
+ }
@@ -27,9 +27,22 @@ export class SchemaVersionError extends Error {
27
27
  }
28
28
 
29
29
  // Discover and load every migration module under ./migrations/, sorted by
30
- // numeric prefix ascending. Each module must export VERSION (integer),
31
- // DESCRIPTION (string), and up(db) (function).
32
- async function loadMigrations() {
30
+ // numeric prefix ascending. Each module MUST export VERSION (number),
31
+ // DESCRIPTION (string), and up(db) (function). The filename's numeric prefix
32
+ // MUST equal the exported VERSION (enforced below).
33
+ //
34
+ // SQL=false is REJECTED — fs-layout migrations live in ../layout-migrations/
35
+ // (see v1.5.2.1 F3). The runner used to silently skip SQL=false; that escape
36
+ // hatch was a runtime workaround for a directory-structure mistake and is
37
+ // gone. If a file in this directory declares SQL=false, the runner fails
38
+ // loudly so the structural error is caught at startup, not at the next
39
+ // schema-bricking copy-paste.
40
+ //
41
+ // Exported (v1.5.1 W3.B) so search.js (and any other consumer that needs
42
+ // the sync migration pipeline) can reuse the SAME discovery path instead
43
+ // of maintaining a parallel hardcoded list. Single source of truth: drop
44
+ // a new NNN-name.js into ./migrations/ and every consumer sees it.
45
+ export async function loadMigrations() {
33
46
  let files;
34
47
  try {
35
48
  files = readdirSync(MIGRATIONS_DIR);
@@ -43,9 +56,30 @@ async function loadMigrations() {
43
56
  for (const f of matches) {
44
57
  const url = pathToFileURL(join(MIGRATIONS_DIR, f)).href;
45
58
  const mod = await import(url);
59
+ // v1.5.2.1 F3: SQL=false is no longer an in-directory escape hatch. The
60
+ // fs-layout migrations live in ../layout-migrations/ and are statically
61
+ // registered there. If anything in this directory still declares
62
+ // SQL=false it's a structural error (likely a misplaced fs-layout
63
+ // migration). Fail loudly rather than silently skip.
64
+ if (mod.SQL === false) {
65
+ throw new Error(
66
+ `Memory migration ${f} declares SQL=false — fs-layout migrations belong in ` +
67
+ `layout-migrations/, not migrations/. Move the file or remove SQL=false.`
68
+ );
69
+ }
46
70
  if (typeof mod.VERSION !== 'number' || typeof mod.up !== 'function') {
47
71
  throw new Error(`Memory migration ${f} is missing VERSION or up().`);
48
72
  }
73
+ // v1.5.2.1 F3.2: filename numeric prefix MUST equal exported VERSION.
74
+ // Catches reordering / rename mistakes immediately rather than at the
75
+ // next user_version comparison (where the failure mode is silent).
76
+ const filenamePrefix = parseInt(f.match(/^(\d+)/)[1], 10);
77
+ if (filenamePrefix !== mod.VERSION) {
78
+ throw new Error(
79
+ `Memory migration ${f}: filename prefix ${filenamePrefix} does not match ` +
80
+ `exported VERSION ${mod.VERSION}. Rename the file or fix the VERSION.`
81
+ );
82
+ }
49
83
  out.push({
50
84
  file: f,
51
85
  version: mod.VERSION,
@@ -0,0 +1,50 @@
1
+ // IJFW v1.5.1 -- memory migration 009: M1 obsidian-index backfill.
2
+ //
3
+ // Source authority: Trident r5 finding 1.2 (HIGH).
4
+ //
5
+ // Round-4 Fix-1 (commit 3218812) wired M1 (Obsidian wikilink/tag/meta
6
+ // indexing -- indexObsidianRelations) and M2 (A-Mem auto-linking -- autoLink)
7
+ // into the production memory-write path (handleStore, search.js#autoIndex).
8
+ // But the fix is forward-only: every memory_entries row written during
9
+ // v1.5.0 -- when M1/M2 were bypassed -- has empty memory_links / memory_tags
10
+ // / memory_meta. An existing user upgrading to v1.5.1 got auto-linking +
11
+ // wikilink indexing only on NEW entries; their accumulated memory stayed
12
+ // un-indexed.
13
+ //
14
+ // This migration runs a ONE-TIME M1 backfill: it walks every existing
15
+ // memory_entries row and runs indexObsidianRelations over its body. M1 is:
16
+ // - free -- pure markdown parse, zero LLM / network / cost
17
+ // - idempotent -- indexObsidianRelations does DELETE-then-INSERT per id,
18
+ // so re-applying produces identical aux rows
19
+ // which is exactly what makes it safe to run inside a schema migration: it
20
+ // runs once (user_version gates re-application), deterministically, and a
21
+ // crash rolls the whole txn back to user_version 8.
22
+ //
23
+ // M2 (autoLink) is NOT backfilled here -- it makes one LLM call per row and
24
+ // can cost real money over a large memory. M2 backfill is opt-in via the
25
+ // `ijfw memory reindex --m2` CLI verb, which is budget-gated (respects
26
+ // IJFW_AUTOLINK_BUDGET_USD / IJFW_AUTOLINK_OFF / IJFW_AUTOLINK_BACKFILL).
27
+ //
28
+ // Ordering: migration 001 creates memory_entries; migration 006 creates
29
+ // memory_links / memory_tags / memory_meta. Both run before 009, so by the
30
+ // time up() executes the source table and the three aux tables all exist.
31
+ //
32
+ // Crash safety: the migration runner wraps up() in BEGIN IMMEDIATE.
33
+ // backfillObsidianIndex's per-row indexObsidianRelations opens a nested
34
+ // SQLite transaction (savepoint) -- valid inside the outer txn -- and the
35
+ // whole thing rolls back to user_version 8 on any failure.
36
+
37
+ import { backfillObsidianIndex } from '../obsidian-parser.js';
38
+
39
+ export const VERSION = 9;
40
+ export const DESCRIPTION =
41
+ 'memory v1.5.1 -- one-time M1 obsidian-index backfill for pre-fix rows (Trident r5 1.2)';
42
+
43
+ export function up(db) {
44
+ // backfillObsidianIndex tolerates a missing memory_entries table (returns
45
+ // zero counts) so a brand-new db that jumps straight to v9 is a clean
46
+ // no-op -- there are no pre-fix rows to backfill on a fresh install.
47
+ backfillObsidianIndex(db);
48
+ }
49
+
50
+ export default { version: VERSION, description: DESCRIPTION, up };
@@ -84,8 +84,71 @@ export function indexObsidianRelations(db, memoryId, text) {
84
84
  for (const t of parsed.tags) insTag.run(memoryId, t.path, t.depth);
85
85
  for (const m of parsed.meta) insMeta.run(memoryId, m.key, m.value);
86
86
  });
87
- tx();
87
+ // F-LENS2-01: use IMMEDIATE so all facts-table sister-writers acquire the
88
+ // RESERVED lock in the same order — no DEFERRED→IMMEDIATE upgrade collision.
89
+ tx.immediate();
88
90
  return parsed;
89
91
  }
90
92
 
91
- export default { parseObsidian, indexObsidianRelations };
93
+ // v1.5.1 R5-1.2 -- one-time M1 backfill for memory written during v1.5.0,
94
+ // when indexObsidianRelations was NOT wired into the production write path.
95
+ // Round-4 Fix-1 (commit 3218812) wired M1+M2 into handleStore/autoIndex but
96
+ // forward-only: rows already in memory_entries have empty memory_links /
97
+ // memory_tags / memory_meta. This walks EVERY row and re-runs M1 over it.
98
+ //
99
+ // Safe to run over everything:
100
+ // - free -- pure markdown parse, zero LLM / network
101
+ // - idempotent-- indexObsidianRelations clears prior aux rows per id before
102
+ // re-inserting, so a re-run produces identical state
103
+ //
104
+ // The walk reads ids in batches so a very large memory_entries doesn't pin
105
+ // the whole table in memory; each row's indexObsidianRelations call carries
106
+ // its own transaction (DELETE-then-INSERT) so a single bad row never aborts
107
+ // the rest of the backfill.
108
+ //
109
+ // Returns { rows, links, tags, meta } -- counts re-indexed across the run.
110
+ export function backfillObsidianIndex(db, opts = {}) {
111
+ if (!db || typeof db.prepare !== 'function') {
112
+ throw new Error('backfillObsidianIndex: db handle is invalid.');
113
+ }
114
+ const batchSize = Math.max(1, opts.batchSize || 500);
115
+ const result = { rows: 0, links: 0, tags: 0, meta: 0, errors: 0 };
116
+ let lastId = 0;
117
+ // eslint-disable-next-line no-constant-condition
118
+ while (true) {
119
+ let batch;
120
+ try {
121
+ batch = db
122
+ .prepare(
123
+ 'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
124
+ )
125
+ .all(lastId, batchSize);
126
+ } catch {
127
+ // memory_entries missing (fresh db before migration 001) -- nothing to do.
128
+ break;
129
+ }
130
+ if (!batch || batch.length === 0) break;
131
+ for (const row of batch) {
132
+ lastId = row.id;
133
+ if (typeof row.body !== 'string' || row.body.length === 0) continue;
134
+ try {
135
+ const parsed = indexObsidianRelations(db, String(row.id), row.body);
136
+ result.rows += 1;
137
+ result.links += parsed.links.length;
138
+ result.tags += parsed.tags.length;
139
+ result.meta += parsed.meta.length;
140
+ } catch (e) {
141
+ result.errors += 1;
142
+ try {
143
+ console.error(
144
+ '[obsidian] backfill failed for id', row.id, ':', e?.message || e,
145
+ );
146
+ } catch { /* never throw out of the backfill */ }
147
+ }
148
+ }
149
+ if (batch.length < batchSize) break;
150
+ }
151
+ return result;
152
+ }
153
+
154
+ export default { parseObsidian, indexObsidianRelations, backfillObsidianIndex };
@@ -22,7 +22,8 @@ const PREVIEW_CHARS = 300;
22
22
  /** Parse YAML-style frontmatter (key: value lines between --- fences). */
23
23
  function parseFrontmatter(raw) {
24
24
  const fm = { title: null, description: null, type: null };
25
- const m = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
25
+ const stripped = String(raw).replace(/^/, '').replace(/^\s+/, '');
26
+ const m = stripped.match(/^---\r?\n([\s\S]*?)\r?\n---/);
26
27
  if (!m) return fm;
27
28
  for (const line of m[1].split('\n')) {
28
29
  const kv = line.match(/^(\w+):\s*(.+)/);