@ghilteras/opencode-agent-memory 1.2.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -6,7 +6,7 @@
6
6
  >
7
7
  > Originally created by Joshua David Thomas and licensed under the MIT license. Attribution preserved below; see License section.
8
8
 
9
- A journal-only memory plugin for [OpenCode](https://opencode.ai): an append-only, tagged journal with local semantic search.
9
+ A journal-only memory plugin for [OpenCode](https://opencode.ai): an append-only, tagged journal with local lexical search.
10
10
 
11
11
  ## Experimental
12
12
 
@@ -17,7 +17,7 @@ Think of it as a searchable sidecar to `AGENTS.md`. OpenCode supports [rules](ht
17
17
  ## Features
18
18
 
19
19
  - **Append-only journal** - Entries survive across sessions and context compaction
20
- - **Local semantic search** - Find entries by meaning, not just keywords
20
+ - **Local lexical search** - Find candidate entries by matching query words in titles and bodies
21
21
  - **Metadata on every entry** - Project, model, provider, agent, session, timestamp, tags
22
22
  - **Guidance in tool descriptions** - Journal usage guidance lives in the descriptions of the journal tools, so an agent sees it only when it is allowed to call them
23
23
  - **No data leaves the machine** - Embeddings run locally
@@ -32,7 +32,7 @@ Add to your OpenCode config (`~/.config/opencode/opencode.json`):
32
32
 
33
33
  ```json
34
34
  {
35
- "plugins": ["@ghilteras/opencode-agent-memory@1.1.0"]
35
+ "plugins": ["@ghilteras/opencode-agent-memory@2.0.1"]
36
36
  }
37
37
  ```
38
38
 
@@ -63,7 +63,7 @@ The agent gets 3 tools:
63
63
  | Tool | Description |
64
64
  |------|-------------|
65
65
  | `journal_write` | Write a new append-only entry (title, body, optional tags) |
66
- | `journal_search` | Search entries semantically, filter by project or tags, with pagination |
66
+ | `journal_search` | Find lexical candidates by matching query words in titles and bodies; filter by project or tags, with pagination |
67
67
  | `journal_read` | Read a specific journal entry by ID |
68
68
 
69
69
  ### Entry format
@@ -74,7 +74,7 @@ Entries are markdown files with YAML frontmatter in `~/.config/opencode/journal/
74
74
  ---
75
75
  title: "Reasoning-pruner wiring"
76
76
  created: 2026-09-21T18:16:11.300Z
77
- project: /home/angelo/homelab-config
77
+ project: /path/to/project
78
78
  model: deepseek-v4.1-flash
79
79
  provider: opencode-go
80
80
  agent: executor
@@ -85,9 +85,9 @@ tags: [plugin, wiring]
85
85
  Body text...
86
86
  ```
87
87
 
88
- ### Semantic search
88
+ ### Text search
89
89
 
90
- Search uses local embeddings ([paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2), 384d, multilingual EN/IT) - no data leaves your machine. An `.embedding` sidecar is written next to each entry in a versioned format (`{ v: 2, model, dimension, vector }`). The search gate is a **dimension** check: an entry whose recorded `dimension` differs from the plugin's expected dimension is not cosine-compared - it falls back to text matching instead of failing the search. Legacy bare-array embeddings from v0.3.x remain readable.
90
+ Text search runs one MiniSearch BM25+ pass over title and body. Query terms are OR-matched, with native score order and a deterministic tie-break by creation time and ID. This is keyword matching, not a claim of semantic relevance: results are candidates for the agent to inspect. Matching works across languages by matching literal words, not by meaning. When a query has no tokenizable terms (for example, `!!!`), search falls back to case-insensitive exact-substring matching over title and body. The retained index is reused while the ordered candidate-entry ID list is unchanged, avoiding repeated index construction. Embedding sidecars are still written in the versioned `{ v: 2, model, dimension, vector }` format for compatibility, but search does not read them.
91
91
 
92
92
  ### Suggested tags
93
93
 
@@ -113,7 +113,7 @@ By default the transformers.js model cache lives in the default location (`~/.ca
113
113
 
114
114
  ```json
115
115
  {
116
- "cacheDir": "/home/angelo/.cache/opencode/memory-model",
116
+ "cacheDir": "~/.cache/opencode/memory-model",
117
117
  "journal": {
118
118
  "enabled": true
119
119
  }
@@ -129,6 +129,12 @@ Two behaviour changes:
129
129
  - **The journal tools are registered by default.** Previously you had to opt in through `agent-memory.json`; now an absent config means the tools are on, and you turn them off with `"enabled": false`.
130
130
  - **Storage is unchanged.** Existing entries under `~/.config/opencode/journal/` and their `.embedding` sidecars stay readable; nothing is migrated or rewritten.
131
131
 
132
+ ## Upgrading to lexical candidate search
133
+
134
+ - Text search now ranks entries with lexical OR matching: a candidate may match any query term in its title or body.
135
+ - Results are candidates, not relevance claims. `Found N` reports the candidate count before pagination and will usually be large.
136
+ - A punctuation-only query falls back to case-insensitive substring matching over title and body.
137
+
132
138
  ## Upgrading from 0.4.x
133
139
 
134
140
  Memory blocks and their tools (`memory_list`, `memory_set`, `memory_replace`) are **removed**. Blocks are no longer read, written, or injected into the system prompt; the plugin ships only the journal.
@@ -140,7 +146,7 @@ Existing `~/.config/opencode/memory/` and `.opencode/memory/` directories are **
140
146
  - **Requires OpenCode v2.** This release uses the OpenCode v2 plugin API and does not run on v1.
141
147
  - **Restart after config changes.** Plugin config changes require a full OpenCode restart to take effect; editing the config file alone is not sufficient.
142
148
  - **Journal is on by default.** Turn it off with `"enabled": false`; an explicit disable is silent, while a config that exists but is unreadable, malformed, or schema-invalid logs a warning and registers no tools (fail-closed).
143
- - **Local embeddings, no data leaves the machine.** Semantic search uses a locally cached transformers.js model; no external API calls are made for search or embedding.
149
+ - **Embedding sidecars remain compatible.** They are written locally for the existing format contract, diagnostics, and future consumers; text search is lexical and does not read embeddings.
144
150
  - **Known limitation (since 0.5.1): guidance-in-descriptions depends on an unverified premise.** 0.5.1 moved journal guidance from the injected system-prompt note into the three journal tool descriptions, so only agents allowed to call those tools should see it. That relies on opencode omitting a permission-denied tool from the definitions sent to the model. This is **unverified**: a `tool.definition` probe on the isolated instance fires before permission filtering, so it cannot confirm what the model actually receives. If the premise does not hold, an agent denied `journal_*` may still read the description text; `deny` still blocks execution, so there is no capability escalation and no data disclosure — the blast radius is the three tools' description strings.
145
151
 
146
152
  ## Inspiration
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ghilteras/opencode-agent-memory",
3
- "version": "1.2.0",
3
+ "version": "2.0.1",
4
4
  "description": "Journal-only memory plugin for OpenCode: tagged, searchable, append-only journal with semantic search.",
5
5
  "author": "Angelo Pantano <ghilteras@protonmail.com>",
6
6
  "license": "MIT",
@@ -13,9 +13,7 @@
13
13
  },
14
14
  "main": "src/plugin.ts",
15
15
  "type": "module",
16
- "files": [
17
- "src"
18
- ],
16
+ "files": ["src", "!src/**/*.test.ts", "!src/test-suite.test.ts"],
19
17
  "scripts": {
20
18
  "test": "bun run test:core && bun run test:plugin",
21
19
  "test:core": "bun test src/test-suite.test.ts src/doctor.test.ts src/embeddings.test.ts src/tools.test.ts src/journal.test.ts",
@@ -27,6 +25,7 @@
27
25
  "@huggingface/transformers": "^3.8.1",
28
26
  "@opencode/plugin": "^2.0.18",
29
27
  "js-yaml": "^4.1.0",
28
+ "minisearch": "^7.2.0",
30
29
  "zod": "^4.1.13"
31
30
  },
32
31
  "overrides": {
package/src/journal.ts CHANGED
@@ -3,9 +3,10 @@ import * as os from "node:os";
3
3
  import * as path from "node:path";
4
4
 
5
5
  import * as yaml from "js-yaml";
6
+ import MiniSearch from "minisearch";
6
7
  import { z } from "zod";
7
8
 
8
- import { cosineSimilarity, EMBEDDING_DIMENSION, embeddingModelName, generateEmbedding } from "./embeddings";
9
+ import { EMBEDDING_DIMENSION, embeddingModelName, generateEmbedding } from "./embeddings";
9
10
  import { atomicWriteFile, buildFrontmatterDocument, splitFrontmatter } from "./frontmatter";
10
11
 
11
12
  const TagSchema = z.looseObject({
@@ -139,11 +140,20 @@ function embeddingPath(entryPath: string): string {
139
140
  return entryPath.replace(/\.md$/, ".embedding");
140
141
  }
141
142
 
142
- function titleMatches(queryText: string, title: string): boolean {
143
- const q = queryText.trim().toLowerCase();
144
- if (!q) return false;
145
- const t = title.trim().toLowerCase();
146
- return t.includes(q) || q.includes(t);
143
+ function hasLexicalTerms(text: string): boolean {
144
+ return /[\p{L}\p{N}]+/u.test(text);
145
+ }
146
+
147
+ function substringMatches(
148
+ entries: CachedEntry[],
149
+ query: string,
150
+ ): { entry: JournalEntry; score: number }[] {
151
+ const needle = query.toLowerCase();
152
+ return entries.flatMap(({ entry }) =>
153
+ `${entry.title}\n${entry.body}`.toLowerCase().includes(needle)
154
+ ? [{ entry, score: 0 }]
155
+ : [],
156
+ );
147
157
  }
148
158
 
149
159
  async function readEntryFile(filePath: string): Promise<JournalEntry> {
@@ -178,55 +188,12 @@ async function readEntryFile(filePath: string): Promise<JournalEntry> {
178
188
  };
179
189
  }
180
190
 
181
- type ParsedEmbedding =
182
- | { vector: number[]; model: string; dimension: number }
183
- | undefined;
184
-
185
191
  type CachedEntry = {
186
192
  entryMtimeMs: number;
187
193
  entrySize: number;
188
- embMtimeMs: number;
189
- embSize: number;
190
194
  entry: JournalEntry;
191
- embedding: ParsedEmbedding;
192
195
  };
193
196
 
194
- async function loadEmbedding(entryPath: string): Promise<ParsedEmbedding> {
195
- const ePath = embeddingPath(entryPath);
196
- try {
197
- const raw = await fs.readFile(ePath, "utf-8");
198
- const parsed = JSON.parse(raw);
199
- if (Array.isArray(parsed)) {
200
- // Legacy v1 (bare array from all-MiniLM-L6-v2)
201
- return {
202
- vector: parsed as number[],
203
- model: "unknown-legacy",
204
- dimension: parsed.length,
205
- };
206
- }
207
- if (parsed && typeof parsed === "object" && parsed.v === 2) {
208
- const vector = Array.isArray(parsed.vector)
209
- ? parsed.vector
210
- : undefined;
211
- if (
212
- vector &&
213
- vector.length === (parsed.dimension as number) &&
214
- vector.length > 0
215
- ) {
216
- return {
217
- vector,
218
- model: parsed.model as string,
219
- dimension: parsed.dimension as number,
220
- };
221
- }
222
- return undefined;
223
- }
224
- return undefined;
225
- } catch {
226
- return undefined;
227
- }
228
- }
229
-
230
197
  const SAFE_ID = /^[a-zA-Z0-9_-]+$/;
231
198
 
232
199
  function validateId(id: string): string {
@@ -270,6 +237,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
270
237
  // different (including nested or ephemeral) journal directories never
271
238
  // interfere and no foreign keys can accumulate.
272
239
  const entryCache = new Map<string, CachedEntry>();
240
+ let lexicalIndexSnapshot: { ids: string[]; index: MiniSearch } | undefined;
273
241
 
274
242
  return {
275
243
  async write(entry) {
@@ -294,7 +262,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
294
262
  const content = buildFrontmatterDocument(frontmatter, entry.body);
295
263
  await atomicWriteFile(filePath, content);
296
264
 
297
- // Generate and save embedding for semantic search
265
+ // Keep the sidecar for compatibility with doctor, the format contract, and future consumers; search does not read it.
298
266
  const searchableText = `${entry.title}\n${entry.body}`;
299
267
  try {
300
268
  const embedding = await generateEmbedding(searchableText, cacheDir);
@@ -345,8 +313,6 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
345
313
  const limit = Math.min(Math.max(query.limit ?? 20, 1), 50);
346
314
  const offset = Math.max(query.offset ?? 0, 0);
347
315
 
348
- let entries: { entry: JournalEntry; score: number }[] = [];
349
-
350
316
  // Read all entry files
351
317
  let files: string[];
352
318
  try {
@@ -368,48 +334,28 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
368
334
  if (!alive.has(key)) entryCache.delete(key);
369
335
  }
370
336
 
371
- // If a text query is provided, try semantic search first
372
- let queryEmbedding: number[] | undefined;
373
- if (query.text) {
374
- try {
375
- queryEmbedding = await generateEmbedding(query.text, cacheDir);
376
- } catch {
377
- // Fall back to text search if embedding fails
378
- }
379
- }
380
-
381
337
  // Collect all tags across every entry (before filtering)
382
338
  const tagSet = new Set<string>();
339
+ let entryCacheChanged = false;
383
340
 
384
341
  const indexed = await Promise.all(files.map(async (file) => {
385
342
  const filePath = path.join(journalDir, file);
386
343
  try {
387
- const ePath = embeddingPath(filePath);
388
- const [entryStat, embStat] = await Promise.all([
389
- fs.stat(filePath),
390
- fs.stat(ePath).catch(() => ({ mtimeMs: -1, size: -1 } as const)),
391
- ]);
344
+ const entryStat = await fs.stat(filePath);
392
345
  const cached = entryCache.get(filePath);
393
346
  if (
394
347
  cached &&
395
348
  cached.entryMtimeMs === entryStat.mtimeMs &&
396
- cached.entrySize === entryStat.size &&
397
- cached.embMtimeMs === embStat.mtimeMs &&
398
- cached.embSize === embStat.size
349
+ cached.entrySize === entryStat.size
399
350
  ) {
400
351
  return cached;
401
352
  }
402
- const [entry, embedding] = await Promise.all([
403
- readEntryFile(filePath),
404
- loadEmbedding(filePath),
405
- ]);
353
+ const entry = await readEntryFile(filePath);
354
+ entryCacheChanged = true;
406
355
  const value: CachedEntry = {
407
356
  entryMtimeMs: entryStat.mtimeMs,
408
357
  entrySize: entryStat.size,
409
- embMtimeMs: embStat.mtimeMs,
410
- embSize: embStat.size,
411
358
  entry,
412
- embedding,
413
359
  };
414
360
  entryCache.set(filePath, value);
415
361
  return value;
@@ -418,6 +364,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
418
364
  }
419
365
  }));
420
366
 
367
+ const filtered: CachedEntry[] = [];
421
368
  for (const cached of indexed) {
422
369
  if (!cached) continue;
423
370
  const { entry } = cached;
@@ -439,65 +386,62 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
439
386
  if (!allTagsMatch) continue;
440
387
  }
441
388
 
442
- // Score the entry
443
- let score = 0;
444
-
445
- if (query.text) {
446
- if (queryEmbedding) {
447
- // Semantic search
448
- const storedEmbedding = cached.embedding;
449
- if (storedEmbedding) {
450
- // Hard gate: dimension must match, or the embedding is stale
451
- // (different model) and cannot be compared with cosine.
452
- // A mismatched/stale embedding degrades gracefully to text match.
453
- if (storedEmbedding.dimension !== EMBEDDING_DIMENSION) {
454
- const haystack =
455
- `${entry.title}\n${entry.body}`.toLowerCase();
456
- score = haystack.includes(query.text.toLowerCase()) ? 0.5 : 0;
457
- } else {
458
- score = cosineSimilarity(
459
- queryEmbedding,
460
- storedEmbedding.vector,
461
- );
462
- }
463
- } else {
464
- // No embedding stored; fall back to text match
465
- const haystack =
466
- `${entry.title}\n${entry.body}`.toLowerCase();
467
- score = haystack.includes(query.text.toLowerCase()) ? 0.5 : 0;
468
- }
469
- } else {
470
- // Text search fallback
471
- const haystack = `${entry.title}\n${entry.body}`.toLowerCase();
472
- score = haystack.includes(query.text.toLowerCase()) ? 1 : 0;
473
- }
474
-
475
- // Title-anchor floor: a query that matches the entry title (either
476
- // direction, case-insensitive) must not be diluted below the top
477
- // ranks by long-body mean pooling (see 20260905-171239-533).
478
- if (titleMatches(query.text, entry.title)) {
479
- score = Math.max(score, 0.75);
480
- }
389
+ filtered.push(cached);
390
+ }
481
391
 
482
- if (score <= 0) continue;
392
+ let ranked: { entry: JournalEntry; score: number }[];
393
+ if (!query.text) {
394
+ ranked = filtered.map(({ entry }) => ({
395
+ entry,
396
+ score: entry.created.getTime(),
397
+ }));
398
+ ranked.sort((a, b) => b.score - a.score || (a.entry.id < b.entry.id ? 1 : a.entry.id > b.entry.id ? -1 : 0));
399
+ } else {
400
+ const ids = filtered.map(({ entry }) => entry.id);
401
+ const sameEntries = !entryCacheChanged && lexicalIndexSnapshot !== undefined &&
402
+ lexicalIndexSnapshot.ids.length === ids.length &&
403
+ ids.every((id, index) => lexicalIndexSnapshot!.ids[index] === id);
404
+ let lexicalIndex: MiniSearch;
405
+ if (sameEntries) {
406
+ lexicalIndex = lexicalIndexSnapshot!.index;
483
407
  } else {
484
- // No text query - chronological order (score by recency)
485
- score = entry.created.getTime();
408
+ const builtIndex = new MiniSearch({
409
+ fields: ["title", "body"],
410
+ storeFields: ["id"],
411
+ });
412
+ builtIndex.addAll(filtered.map(({ entry }) => ({
413
+ id: entry.id,
414
+ title: entry.title,
415
+ body: entry.body,
416
+ })));
417
+ lexicalIndexSnapshot = { ids, index: builtIndex };
418
+ lexicalIndex = builtIndex;
486
419
  }
487
-
488
- entries.push({ entry, score });
420
+ const byId = new Map(filtered.map(({ entry }) => [entry.id, entry]));
421
+ const lexical = lexicalIndex.search(query.text, { combineWith: "OR" });
422
+ if (hasLexicalTerms(query.text)) {
423
+ ranked = lexical.flatMap((match) => {
424
+ const entry = byId.get(String(match.id));
425
+ return entry ? [{ entry, score: match.score }] : [];
426
+ });
427
+ } else {
428
+ ranked = substringMatches(filtered, query.text);
429
+ }
430
+ ranked.sort((a, b) =>
431
+ b.score - a.score ||
432
+ b.entry.created.getTime() - a.entry.created.getTime() ||
433
+ (a.entry.id < b.entry.id ? 1 : a.entry.id > b.entry.id ? -1 : 0),
434
+ );
489
435
  }
490
436
 
491
- const total = entries.length;
492
-
493
- // Sort by score descending
494
- entries.sort((a, b) => b.score - a.score);
437
+ // Count lexical candidates after metadata filtering, before paging.
438
+ const total = ranked.length;
495
439
 
496
440
  // Apply offset + limit
497
- entries = entries.slice(offset, offset + limit);
441
+ ranked = ranked.slice(offset, offset + limit);
498
442
 
499
443
  return {
500
- entries: entries.map((e) => e.entry),
444
+ entries: ranked.map((e) => e.entry),
501
445
  total,
502
446
  allTags: Array.from(tagSet).sort(),
503
447
  };
package/src/tools.ts CHANGED
@@ -76,11 +76,11 @@ export function JournalRead(store: JournalStore): ToolDefinition {
76
76
  export function JournalSearch(store: JournalStore): ToolDefinition {
77
77
  return {
78
78
  name: "journal_search",
79
- description: "Search journal entries using semantic similarity. Returns matching entries sorted by relevance. All filters are optional and combined with AND logic. Use with no arguments to list recent entries. Use offset to paginate. Before starting complex tasks, search the journal for relevant past context. The journal is global across all projects but each entry records which project it was written from.",
79
+ description: "Find candidate journal entries by lexical OR matching query words in titles and bodies. Candidates are not relevance claims. All filters are optional and combined with AND logic. Use with no arguments to list recent entries. Use offset to paginate. Before starting complex tasks, search the journal for relevant past context. The journal is global across all projects but each entry records which project it was written from.",
80
80
  input: {
81
81
  type: "object",
82
82
  properties: {
83
- text: { type: "string", description: "Semantic search text" },
83
+ text: { type: "string", description: "Query words for lexical candidate matching in titles and bodies" },
84
84
  project: { type: "string", description: "Filter by project directory" },
85
85
  tags: { type: "string", description: "Comma-separated tags; all must match" },
86
86
  limit: { type: "integer", minimum: 1, description: "Maximum entries to return" },
@@ -99,7 +99,7 @@ export function JournalSearch(store: JournalStore): ToolDefinition {
99
99
  const tagsLine = formatTagsInUse(result.allTags);
100
100
  if (!result.entries.length) return { content: `No journal entries found.${tagsLine}` };
101
101
  const offset = (input.offset as number | undefined) ?? 0;
102
- const header = `Found ${result.total} entries (showing ${offset + 1}–${offset + result.entries.length}):`;
102
+ const header = `Found ${result.total} candidate entries (showing ${offset + 1}–${offset + result.entries.length}):`;
103
103
  const lines = result.entries.map((entry) => `${entry.id}\n ${entry.title}${entry.tags.length ? ` [${entry.tags.join(", ")}]` : ""}\n ${entry.created.toISOString()}`);
104
104
  return { content: `${header}${tagsLine}\n\n${lines.join("\n\n")}` };
105
105
  },
@@ -1,183 +0,0 @@
1
- import { afterEach, expect, test } from "bun:test";
2
- import * as fs from "node:fs/promises";
3
- import * as os from "node:os";
4
- import * as path from "node:path";
5
- import { diagnoseJournal } from "./doctor";
6
-
7
- const roots: string[] = [];
8
- async function fixture() {
9
- const root = await fs.mkdtemp(path.join(os.tmpdir(), "journal-doctor-"));
10
- roots.push(root);
11
- await fs.mkdir(path.join(root, "journal"));
12
- return path.join(root, "journal");
13
- }
14
- async function entry(dir: string, name: string, yamlText?: string, sidecar?: string) {
15
- await fs.writeFile(path.join(dir, `${name}.md`), yamlText === undefined ? "plain text" : `---\n${yamlText}\n---\nbody\n`);
16
- if (sidecar !== undefined) await fs.writeFile(path.join(dir, `${name}.embedding`), sidecar);
17
- }
18
- afterEach(async () => { await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); });
19
-
20
- test("classifies frontmatter, created values, sidecars and multiple findings deterministically", async () => {
21
- const dir = await fixture();
22
- const valid = 'title: Good\ncreated: "2025-01-01T00:00:00Z"';
23
- await entry(dir, "00-valid", valid, JSON.stringify({ v: 2, model: "wrong-model", dimension: 2, vector: [0.1, 0.2] }));
24
- await entry(dir, "01-missing-title", "created: \"2025-01-01\"");
25
- await entry(dir, "02-empty-title", 'title: ""');
26
- await entry(dir, "03-wrong-title", "title: 42");
27
- await entry(dir, "04-missing-created", "title: Present", "[1,2]");
28
- await entry(dir, "05-date-only", "title: Present\ndate: \"2025-01-01\"");
29
- await entry(dir, "06-invalid-created", 'title: Present\ncreated: "not a date"');
30
- await entry(dir, "07-wrong-created", "title: Present\ncreated: 42");
31
- await entry(dir, "08-yaml-timestamp", "title: Present\ncreated: 2025-01-01T00:00:00Z");
32
- await entry(dir, "09-bad-yaml", "title: [broken");
33
- await entry(dir, "10-no-frontmatter");
34
- await entry(dir, "11-malformed-json", valid, "{");
35
- await entry(dir, "12-vector-type", valid, '{"v":2,"model":"m","dimension":1,"vector":["x"]}');
36
- await entry(dir, "13-empty-vector", valid, '{"v":2,"model":"m","dimension":0,"vector":[]}');
37
- await entry(dir, "14-vector-dimension", valid, '{"v":2,"model":"m","dimension":3,"vector":[1,2]}');
38
- await entry(dir, "15-legacy", valid, "[1,2]");
39
- await entry(dir, "16-legacy-invalid", valid, '[1,"x"]');
40
- await entry(dir, "17-missing-sidecar", valid);
41
-
42
- const inventory = async () => Promise.all((await fs.readdir(dir)).sort().map(async (f) => [f, await fs.readFile(path.join(dir, f)), (await fs.stat(path.join(dir, f))).mtimeMs] as const));
43
- const before = await inventory();
44
- const report = await diagnoseJournal(path.dirname(dir));
45
- expect(await diagnoseJournal(path.dirname(dir))).toEqual(report);
46
- expect(report.counts.files).toBe(18);
47
- const codes = (name: string) => report.findings.filter((f) => f.path.startsWith(name)).map((f) => f.code);
48
- expect(codes("01-missing-title")).toContain("frontmatter_title_invalid");
49
- expect(codes("02-empty-title")).toContain("frontmatter_title_invalid");
50
- expect(codes("03-wrong-title")).toContain("frontmatter_title_invalid");
51
- expect(codes("04-missing-created")).toContain("created_missing");
52
- expect(codes("05-date-only")).toContain("created_missing");
53
- expect(codes("06-invalid-created")).toContain("created_invalid");
54
- expect(codes("07-wrong-created")).toContain("frontmatter_created_invalid");
55
- expect(codes("08-yaml-timestamp")).toContain("frontmatter_created_invalid");
56
- expect(codes("09-bad-yaml")).toContain("yaml_malformed");
57
- expect(codes("10-no-frontmatter")).toContain("frontmatter_missing_or_malformed");
58
- expect(codes("11-malformed-json")).toContain("embedding_json_malformed");
59
- for (const name of ["12-vector-type", "13-empty-vector", "14-vector-dimension", "16-legacy-invalid"]) expect(codes(name)).toContain("embedding_invalid");
60
- expect(codes("00-valid")).toContain("embedding_incompatible");
61
- expect(codes("15-legacy")).toContain("embedding_model_unknown");
62
- expect(codes("17-missing-sidecar")).toContain("embedding_missing");
63
- expect(codes("04-missing-created")).toEqual(expect.arrayContaining(["created_missing", "embedding_model_unknown"]));
64
- expect(report.findings).toEqual([...report.findings].sort((a, b) => a.path.localeCompare(b.path) || a.code.localeCompare(b.code) || a.severity.localeCompare(b.severity)));
65
- expect(report.findings.some((f) => /broken|Good|Present|not a date/.test(JSON.stringify(f)))).toBe(false);
66
- expect(await inventory()).toEqual(before);
67
- });
68
-
69
- test("empty, missing, and injected unreadable roots/files have explicit incomplete semantics", async () => {
70
- const missing = await diagnoseJournal("/path/that/does/not/exist");
71
- expect(missing.complete).toBe(false);
72
- expect(missing.findings[0]?.code).toBe("root_unreadable");
73
- const dir = await fixture();
74
- expect((await diagnoseJournal(path.dirname(dir))).complete).toBe(true);
75
- await entry(dir, "a", "title: A"); await entry(dir, "b", "title: B");
76
- const failingRead = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
77
- if (String(p).endsWith("a.md")) throw new Error("injected");
78
- return fs.readFile(p, ...(args as [any]));
79
- } } as unknown as typeof fs;
80
- const report = await diagnoseJournal(path.dirname(dir), failingRead);
81
- expect(report.complete).toBe(false);
82
- expect(report.findings.some((f) => f.path === "a.md" && f.code === "file_unreadable")).toBe(true);
83
- expect(report.findings.some((f) => f.path === "b.md" && f.code === "created_missing")).toBe(true);
84
- });
85
-
86
- test("CLI emits exact bytes and exit codes for clean, warning, error, incomplete and bad invocation", async () => {
87
- const dir = await fixture();
88
- const cli = path.resolve(import.meta.dir, "journal-doctor.ts");
89
- const invoke = (...args: string[]) => Bun.spawnSync([process.execPath, "run", cli, ...args], { stdout: "pipe", stderr: "pipe" });
90
- const assertCli = (result: ReturnType<typeof invoke>, stdout: string, stderr: string, exitCode: number) =>
91
- expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout, stderr, exitCode });
92
- assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":0,"errors":0,"warnings":0},"findings":[]}\n', "", 0);
93
-
94
- await entry(dir, "warning", 'title: Warn\ncreated: "2025-01-01"');
95
- assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":1,"errors":0,"warnings":1},"findings":[{"path":"warning.embedding","code":"embedding_missing","severity":"warning"}]}\n', "", 0);
96
-
97
- await entry(dir, "invalid", "title: 12");
98
- assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":2,"errors":1,"warnings":2},"findings":[{"path":"invalid.embedding","code":"embedding_missing","severity":"warning"},{"path":"invalid.md","code":"frontmatter_title_invalid","severity":"error"},{"path":"warning.embedding","code":"embedding_missing","severity":"warning"}]}\n', "", 1);
99
-
100
- assertCli(invoke("/path/that/does/not/exist"), '{"complete":false,"counts":{"files":0,"errors":1,"warnings":0},"findings":[{"path":".","code":"root_unreadable","severity":"error"}]}\n', "", 2);
101
- assertCli(invoke(path.dirname(dir), "extra"), "", "usage: bun run journal:doctor [config-dir]\n", 2);
102
- });
103
-
104
- test("inventories orphan and associated sidecars independently and emits stable full report", async () => {
105
- const dir = await fixture();
106
- const valid = 'title: Good\ncreated: "2025-01-01T00:00:00Z"';
107
- await entry(dir, "A", valid);
108
- await entry(dir, "a", valid, "{");
109
- await fs.writeFile(path.join(dir, "_.embedding"), "{");
110
- const expected = { complete: true, counts: { files: 2, errors: 2, warnings: 2 }, findings: [
111
- { path: "A.embedding", code: "embedding_missing", severity: "warning" },
112
- { path: "_.embedding", code: "embedding_json_malformed", severity: "error" },
113
- { path: "_.embedding", code: "embedding_orphan", severity: "warning" },
114
- { path: "a.embedding", code: "embedding_json_malformed", severity: "error" },
115
- ] };
116
- const report = await diagnoseJournal(path.dirname(dir));
117
- expect(JSON.stringify(report)).toBe(JSON.stringify(expected));
118
- const cli = path.resolve(import.meta.dir, "journal-doctor.ts");
119
- const result = Bun.spawnSync([process.execPath, "run", cli, path.dirname(dir)], { stdout: "pipe", stderr: "pipe" });
120
- expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout: `${JSON.stringify(expected)}\n`, stderr: "", exitCode: 1 });
121
- });
122
-
123
- test("sidecar validation survives markdown read failure; listed sidecar disappearance is incomplete", async () => {
124
- const dir = await fixture();
125
- await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"', "{");
126
- const failingRead = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
127
- if (String(p).endsWith("a.md")) throw Object.assign(new Error("denied"), { code: "EACCES" });
128
- return fs.readFile(p, ...(args as [any]));
129
- } } as unknown as typeof fs;
130
- const failed = await diagnoseJournal(path.dirname(dir), failingRead);
131
- expect(failed.complete).toBe(false);
132
- expect(failed.findings.map((f) => f.code)).toEqual(expect.arrayContaining(["file_unreadable", "embedding_json_malformed"]));
133
-
134
- let removed = false;
135
- const disappearing = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
136
- if (String(p).endsWith("a.embedding") && !removed) { removed = true; throw Object.assign(new Error("gone"), { code: "ENOENT" }); }
137
- return fs.readFile(p, ...(args as [any]));
138
- } } as unknown as typeof fs;
139
- const gone = await diagnoseJournal(path.dirname(dir), disappearing);
140
- expect(gone.complete).toBe(false);
141
- expect(gone.findings.some((f) => f.path === "a.embedding" && f.code === "embedding_unreadable")).toBe(true);
142
- });
143
-
144
- test("diagnosis never initializes model pipeline or fetches, while warmup activates trap in child", async () => {
145
- const dir = await fixture();
146
- await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"');
147
- const childCode = `
148
- import { mock } from "bun:test";
149
- let pipelines = 0, fetches = 0;
150
- const originalFetch = globalThis.fetch;
151
- globalThis.fetch = async (...args) => { fetches++; throw new Error("FETCH_SENTINEL"); };
152
- mock.module("@huggingface/transformers", () => ({ pipeline: async () => { pipelines++; throw new Error("PIPELINE_SENTINEL"); } }));
153
- const { diagnoseJournal } = await import(${JSON.stringify(path.resolve(import.meta.dir, "doctor.ts"))});
154
- const { warmupEmbedder } = await import(${JSON.stringify(path.resolve(import.meta.dir, "embeddings.ts"))});
155
- await diagnoseJournal(${JSON.stringify(path.dirname(dir))});
156
- if (pipelines !== 0 || fetches !== 0) throw new Error("diagnosis activated model/fetch trap");
157
- try { await warmupEmbedder(); throw new Error("pipeline sentinel was not caught"); } catch (error) { if (!String(error).includes("PIPELINE_SENTINEL")) throw error; }
158
- if (pipelines !== 1 || fetches !== 0) throw new Error("pipeline trap count mismatch");
159
- globalThis.fetch = originalFetch;
160
- console.log(JSON.stringify({ pipelines, fetches }));
161
- `;
162
- const result = Bun.spawnSync([process.execPath, "-e", childCode], { stdout: "pipe", stderr: "pipe" });
163
- expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout: '{"pipelines":1,"fetches":0}\n', stderr: "", exitCode: 0 });
164
- });
165
-
166
- test("only flat regular inventory files are read and each sidecar is read once", async () => {
167
- const dir = await fixture();
168
- await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"', "[1]");
169
- await fs.mkdir(path.join(dir, "nested"));
170
- await fs.writeFile(path.join(dir, "nested", "hidden.md"), "bad");
171
- await fs.writeFile(path.join(dir, "nested", "hidden.embedding"), "{");
172
- await fs.writeFile(path.join(dir, "ignored.txt"), "bad");
173
- await fs.symlink(path.join(dir, "a.md"), path.join(dir, "linked.md"));
174
- const reads = new Map<string, number>();
175
- const counting = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
176
- const name = path.basename(String(p)); reads.set(name, (reads.get(name) ?? 0) + 1);
177
- return fs.readFile(p, ...(args as [any]));
178
- } } as unknown as typeof fs;
179
- const report = await diagnoseJournal(path.dirname(dir), counting);
180
- expect(report.counts.files).toBe(1);
181
- expect([...reads.entries()].sort()).toEqual([["a.embedding", 1], ["a.md", 1]]);
182
- expect(report.findings.map((f) => f.path)).toEqual(["a.embedding", "a.embedding"]);
183
- });