@ghilteras/opencode-agent-memory 1.2.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -9
- package/package.json +3 -4
- package/src/journal.ts +71 -127
- package/src/tools.ts +3 -3
- package/src/doctor.test.ts +0 -183
- package/src/embeddings.test.ts +0 -47
- package/src/journal.test.ts +0 -634
- package/src/plugin-disabled.test.ts +0 -39
- package/src/plugin.test.ts +0 -220
- package/src/test-suite.test.ts +0 -15
- package/src/tools.test.ts +0 -115
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
>
|
|
7
7
|
> Originally created by Joshua David Thomas and licensed under the MIT license. Attribution preserved below; see License section.
|
|
8
8
|
|
|
9
|
-
A journal-only memory plugin for [OpenCode](https://opencode.ai): an append-only, tagged journal with local
|
|
9
|
+
A journal-only memory plugin for [OpenCode](https://opencode.ai): an append-only, tagged journal with local lexical search.
|
|
10
10
|
|
|
11
11
|
## Experimental
|
|
12
12
|
|
|
@@ -17,7 +17,7 @@ Think of it as a searchable sidecar to `AGENTS.md`. OpenCode supports [rules](ht
|
|
|
17
17
|
## Features
|
|
18
18
|
|
|
19
19
|
- **Append-only journal** - Entries survive across sessions and context compaction
|
|
20
|
-
- **Local
|
|
20
|
+
- **Local lexical search** - Find candidate entries by matching query words in titles and bodies
|
|
21
21
|
- **Metadata on every entry** - Project, model, provider, agent, session, timestamp, tags
|
|
22
22
|
- **Guidance in tool descriptions** - Journal usage guidance lives in the descriptions of the journal tools, so an agent sees it only when it is allowed to call them
|
|
23
23
|
- **No data leaves the machine** - Embeddings run locally
|
|
@@ -32,7 +32,7 @@ Add to your OpenCode config (`~/.config/opencode/opencode.json`):
|
|
|
32
32
|
|
|
33
33
|
```json
|
|
34
34
|
{
|
|
35
|
-
"plugins": ["@ghilteras/opencode-agent-memory@
|
|
35
|
+
"plugins": ["@ghilteras/opencode-agent-memory@2.0.1"]
|
|
36
36
|
}
|
|
37
37
|
```
|
|
38
38
|
|
|
@@ -63,7 +63,7 @@ The agent gets 3 tools:
|
|
|
63
63
|
| Tool | Description |
|
|
64
64
|
|------|-------------|
|
|
65
65
|
| `journal_write` | Write a new append-only entry (title, body, optional tags) |
|
|
66
|
-
| `journal_search` |
|
|
66
|
+
| `journal_search` | Find lexical candidates by matching query words in titles and bodies; filter by project or tags, with pagination |
|
|
67
67
|
| `journal_read` | Read a specific journal entry by ID |
|
|
68
68
|
|
|
69
69
|
### Entry format
|
|
@@ -74,7 +74,7 @@ Entries are markdown files with YAML frontmatter in `~/.config/opencode/journal/
|
|
|
74
74
|
---
|
|
75
75
|
title: "Reasoning-pruner wiring"
|
|
76
76
|
created: 2026-09-21T18:16:11.300Z
|
|
77
|
-
project: /
|
|
77
|
+
project: /path/to/project
|
|
78
78
|
model: deepseek-v4.1-flash
|
|
79
79
|
provider: opencode-go
|
|
80
80
|
agent: executor
|
|
@@ -85,9 +85,9 @@ tags: [plugin, wiring]
|
|
|
85
85
|
Body text...
|
|
86
86
|
```
|
|
87
87
|
|
|
88
|
-
###
|
|
88
|
+
### Text search
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
Text search runs one MiniSearch BM25+ pass over title and body. Query terms are OR-matched, with native score order and a deterministic tie-break by creation time and ID. This is keyword matching, not a claim of semantic relevance: results are candidates for the agent to inspect. Matching works across languages by matching literal words, not by meaning. When a query has no tokenizable terms (for example, `!!!`), search falls back to case-insensitive exact-substring matching over title and body. The retained index is reused while the ordered candidate-entry ID list is unchanged, avoiding repeated index construction. Embedding sidecars are still written in the versioned `{ v: 2, model, dimension, vector }` format for compatibility, but search does not read them.
|
|
91
91
|
|
|
92
92
|
### Suggested tags
|
|
93
93
|
|
|
@@ -113,7 +113,7 @@ By default the transformers.js model cache lives in the default location (`~/.ca
|
|
|
113
113
|
|
|
114
114
|
```json
|
|
115
115
|
{
|
|
116
|
-
"cacheDir": "
|
|
116
|
+
"cacheDir": "~/.cache/opencode/memory-model",
|
|
117
117
|
"journal": {
|
|
118
118
|
"enabled": true
|
|
119
119
|
}
|
|
@@ -129,6 +129,12 @@ Two behaviour changes:
|
|
|
129
129
|
- **The journal tools are registered by default.** Previously you had to opt in through `agent-memory.json`; now an absent config means the tools are on, and you turn them off with `"enabled": false`.
|
|
130
130
|
- **Storage is unchanged.** Existing entries under `~/.config/opencode/journal/` and their `.embedding` sidecars stay readable; nothing is migrated or rewritten.
|
|
131
131
|
|
|
132
|
+
## Upgrading to lexical candidate search
|
|
133
|
+
|
|
134
|
+
- Text search now ranks entries with lexical OR matching: a candidate may match any query term in its title or body.
|
|
135
|
+
- Results are candidates, not relevance claims. `Found N` reports the candidate count before pagination and will usually be large.
|
|
136
|
+
- A punctuation-only query falls back to case-insensitive substring matching over title and body.
|
|
137
|
+
|
|
132
138
|
## Upgrading from 0.4.x
|
|
133
139
|
|
|
134
140
|
Memory blocks and their tools (`memory_list`, `memory_set`, `memory_replace`) are **removed**. Blocks are no longer read, written, or injected into the system prompt; the plugin ships only the journal.
|
|
@@ -140,7 +146,7 @@ Existing `~/.config/opencode/memory/` and `.opencode/memory/` directories are **
|
|
|
140
146
|
- **Requires OpenCode v2.** This release uses the OpenCode v2 plugin API and does not run on v1.
|
|
141
147
|
- **Restart after config changes.** Plugin config changes require a full OpenCode restart to take effect; editing the config file alone is not sufficient.
|
|
142
148
|
- **Journal is on by default.** Turn it off with `"enabled": false`; an explicit disable is silent, while a config that exists but is unreadable, malformed, or schema-invalid logs a warning and registers no tools (fail-closed).
|
|
143
|
-
- **
|
|
149
|
+
- **Embedding sidecars remain compatible.** They are written locally for the existing format contract, diagnostics, and future consumers; text search is lexical and does not read embeddings.
|
|
144
150
|
- **Known limitation (since 0.5.1): guidance-in-descriptions depends on an unverified premise.** 0.5.1 moved journal guidance from the injected system-prompt note into the three journal tool descriptions, so only agents allowed to call those tools should see it. That relies on opencode omitting a permission-denied tool from the definitions sent to the model. This is **unverified**: a `tool.definition` probe on the isolated instance fires before permission filtering, so it cannot confirm what the model actually receives. If the premise does not hold, an agent denied `journal_*` may still read the description text; `deny` still blocks execution, so there is no capability escalation and no data disclosure — the blast radius is the three tools' description strings.
|
|
145
151
|
|
|
146
152
|
## Inspiration
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ghilteras/opencode-agent-memory",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "Journal-only memory plugin for OpenCode: tagged, searchable, append-only journal with semantic search.",
|
|
5
5
|
"author": "Angelo Pantano <ghilteras@protonmail.com>",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,9 +13,7 @@
|
|
|
13
13
|
},
|
|
14
14
|
"main": "src/plugin.ts",
|
|
15
15
|
"type": "module",
|
|
16
|
-
"files": [
|
|
17
|
-
"src"
|
|
18
|
-
],
|
|
16
|
+
"files": ["src", "!src/**/*.test.ts", "!src/test-suite.test.ts"],
|
|
19
17
|
"scripts": {
|
|
20
18
|
"test": "bun run test:core && bun run test:plugin",
|
|
21
19
|
"test:core": "bun test src/test-suite.test.ts src/doctor.test.ts src/embeddings.test.ts src/tools.test.ts src/journal.test.ts",
|
|
@@ -27,6 +25,7 @@
|
|
|
27
25
|
"@huggingface/transformers": "^3.8.1",
|
|
28
26
|
"@opencode/plugin": "^2.0.18",
|
|
29
27
|
"js-yaml": "^4.1.0",
|
|
28
|
+
"minisearch": "^7.2.0",
|
|
30
29
|
"zod": "^4.1.13"
|
|
31
30
|
},
|
|
32
31
|
"overrides": {
|
package/src/journal.ts
CHANGED
|
@@ -3,9 +3,10 @@ import * as os from "node:os";
|
|
|
3
3
|
import * as path from "node:path";
|
|
4
4
|
|
|
5
5
|
import * as yaml from "js-yaml";
|
|
6
|
+
import MiniSearch from "minisearch";
|
|
6
7
|
import { z } from "zod";
|
|
7
8
|
|
|
8
|
-
import {
|
|
9
|
+
import { EMBEDDING_DIMENSION, embeddingModelName, generateEmbedding } from "./embeddings";
|
|
9
10
|
import { atomicWriteFile, buildFrontmatterDocument, splitFrontmatter } from "./frontmatter";
|
|
10
11
|
|
|
11
12
|
const TagSchema = z.looseObject({
|
|
@@ -139,11 +140,20 @@ function embeddingPath(entryPath: string): string {
|
|
|
139
140
|
return entryPath.replace(/\.md$/, ".embedding");
|
|
140
141
|
}
|
|
141
142
|
|
|
142
|
-
function
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
143
|
+
function hasLexicalTerms(text: string): boolean {
|
|
144
|
+
return /[\p{L}\p{N}]+/u.test(text);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function substringMatches(
|
|
148
|
+
entries: CachedEntry[],
|
|
149
|
+
query: string,
|
|
150
|
+
): { entry: JournalEntry; score: number }[] {
|
|
151
|
+
const needle = query.toLowerCase();
|
|
152
|
+
return entries.flatMap(({ entry }) =>
|
|
153
|
+
`${entry.title}\n${entry.body}`.toLowerCase().includes(needle)
|
|
154
|
+
? [{ entry, score: 0 }]
|
|
155
|
+
: [],
|
|
156
|
+
);
|
|
147
157
|
}
|
|
148
158
|
|
|
149
159
|
async function readEntryFile(filePath: string): Promise<JournalEntry> {
|
|
@@ -178,55 +188,12 @@ async function readEntryFile(filePath: string): Promise<JournalEntry> {
|
|
|
178
188
|
};
|
|
179
189
|
}
|
|
180
190
|
|
|
181
|
-
type ParsedEmbedding =
|
|
182
|
-
| { vector: number[]; model: string; dimension: number }
|
|
183
|
-
| undefined;
|
|
184
|
-
|
|
185
191
|
type CachedEntry = {
|
|
186
192
|
entryMtimeMs: number;
|
|
187
193
|
entrySize: number;
|
|
188
|
-
embMtimeMs: number;
|
|
189
|
-
embSize: number;
|
|
190
194
|
entry: JournalEntry;
|
|
191
|
-
embedding: ParsedEmbedding;
|
|
192
195
|
};
|
|
193
196
|
|
|
194
|
-
async function loadEmbedding(entryPath: string): Promise<ParsedEmbedding> {
|
|
195
|
-
const ePath = embeddingPath(entryPath);
|
|
196
|
-
try {
|
|
197
|
-
const raw = await fs.readFile(ePath, "utf-8");
|
|
198
|
-
const parsed = JSON.parse(raw);
|
|
199
|
-
if (Array.isArray(parsed)) {
|
|
200
|
-
// Legacy v1 (bare array from all-MiniLM-L6-v2)
|
|
201
|
-
return {
|
|
202
|
-
vector: parsed as number[],
|
|
203
|
-
model: "unknown-legacy",
|
|
204
|
-
dimension: parsed.length,
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
if (parsed && typeof parsed === "object" && parsed.v === 2) {
|
|
208
|
-
const vector = Array.isArray(parsed.vector)
|
|
209
|
-
? parsed.vector
|
|
210
|
-
: undefined;
|
|
211
|
-
if (
|
|
212
|
-
vector &&
|
|
213
|
-
vector.length === (parsed.dimension as number) &&
|
|
214
|
-
vector.length > 0
|
|
215
|
-
) {
|
|
216
|
-
return {
|
|
217
|
-
vector,
|
|
218
|
-
model: parsed.model as string,
|
|
219
|
-
dimension: parsed.dimension as number,
|
|
220
|
-
};
|
|
221
|
-
}
|
|
222
|
-
return undefined;
|
|
223
|
-
}
|
|
224
|
-
return undefined;
|
|
225
|
-
} catch {
|
|
226
|
-
return undefined;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
197
|
const SAFE_ID = /^[a-zA-Z0-9_-]+$/;
|
|
231
198
|
|
|
232
199
|
function validateId(id: string): string {
|
|
@@ -270,6 +237,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
270
237
|
// different (including nested or ephemeral) journal directories never
|
|
271
238
|
// interfere and no foreign keys can accumulate.
|
|
272
239
|
const entryCache = new Map<string, CachedEntry>();
|
|
240
|
+
let lexicalIndexSnapshot: { ids: string[]; index: MiniSearch } | undefined;
|
|
273
241
|
|
|
274
242
|
return {
|
|
275
243
|
async write(entry) {
|
|
@@ -294,7 +262,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
294
262
|
const content = buildFrontmatterDocument(frontmatter, entry.body);
|
|
295
263
|
await atomicWriteFile(filePath, content);
|
|
296
264
|
|
|
297
|
-
//
|
|
265
|
+
// Keep the sidecar for compatibility with doctor, the format contract, and future consumers; search does not read it.
|
|
298
266
|
const searchableText = `${entry.title}\n${entry.body}`;
|
|
299
267
|
try {
|
|
300
268
|
const embedding = await generateEmbedding(searchableText, cacheDir);
|
|
@@ -345,8 +313,6 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
345
313
|
const limit = Math.min(Math.max(query.limit ?? 20, 1), 50);
|
|
346
314
|
const offset = Math.max(query.offset ?? 0, 0);
|
|
347
315
|
|
|
348
|
-
let entries: { entry: JournalEntry; score: number }[] = [];
|
|
349
|
-
|
|
350
316
|
// Read all entry files
|
|
351
317
|
let files: string[];
|
|
352
318
|
try {
|
|
@@ -368,48 +334,28 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
368
334
|
if (!alive.has(key)) entryCache.delete(key);
|
|
369
335
|
}
|
|
370
336
|
|
|
371
|
-
// If a text query is provided, try semantic search first
|
|
372
|
-
let queryEmbedding: number[] | undefined;
|
|
373
|
-
if (query.text) {
|
|
374
|
-
try {
|
|
375
|
-
queryEmbedding = await generateEmbedding(query.text, cacheDir);
|
|
376
|
-
} catch {
|
|
377
|
-
// Fall back to text search if embedding fails
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
|
|
381
337
|
// Collect all tags across every entry (before filtering)
|
|
382
338
|
const tagSet = new Set<string>();
|
|
339
|
+
let entryCacheChanged = false;
|
|
383
340
|
|
|
384
341
|
const indexed = await Promise.all(files.map(async (file) => {
|
|
385
342
|
const filePath = path.join(journalDir, file);
|
|
386
343
|
try {
|
|
387
|
-
const
|
|
388
|
-
const [entryStat, embStat] = await Promise.all([
|
|
389
|
-
fs.stat(filePath),
|
|
390
|
-
fs.stat(ePath).catch(() => ({ mtimeMs: -1, size: -1 } as const)),
|
|
391
|
-
]);
|
|
344
|
+
const entryStat = await fs.stat(filePath);
|
|
392
345
|
const cached = entryCache.get(filePath);
|
|
393
346
|
if (
|
|
394
347
|
cached &&
|
|
395
348
|
cached.entryMtimeMs === entryStat.mtimeMs &&
|
|
396
|
-
cached.entrySize === entryStat.size
|
|
397
|
-
cached.embMtimeMs === embStat.mtimeMs &&
|
|
398
|
-
cached.embSize === embStat.size
|
|
349
|
+
cached.entrySize === entryStat.size
|
|
399
350
|
) {
|
|
400
351
|
return cached;
|
|
401
352
|
}
|
|
402
|
-
const
|
|
403
|
-
|
|
404
|
-
loadEmbedding(filePath),
|
|
405
|
-
]);
|
|
353
|
+
const entry = await readEntryFile(filePath);
|
|
354
|
+
entryCacheChanged = true;
|
|
406
355
|
const value: CachedEntry = {
|
|
407
356
|
entryMtimeMs: entryStat.mtimeMs,
|
|
408
357
|
entrySize: entryStat.size,
|
|
409
|
-
embMtimeMs: embStat.mtimeMs,
|
|
410
|
-
embSize: embStat.size,
|
|
411
358
|
entry,
|
|
412
|
-
embedding,
|
|
413
359
|
};
|
|
414
360
|
entryCache.set(filePath, value);
|
|
415
361
|
return value;
|
|
@@ -418,6 +364,7 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
418
364
|
}
|
|
419
365
|
}));
|
|
420
366
|
|
|
367
|
+
const filtered: CachedEntry[] = [];
|
|
421
368
|
for (const cached of indexed) {
|
|
422
369
|
if (!cached) continue;
|
|
423
370
|
const { entry } = cached;
|
|
@@ -439,65 +386,62 @@ export function createJournalStore(configDir?: string, cacheDir?: string): Journ
|
|
|
439
386
|
if (!allTagsMatch) continue;
|
|
440
387
|
}
|
|
441
388
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
if (query.text) {
|
|
446
|
-
if (queryEmbedding) {
|
|
447
|
-
// Semantic search
|
|
448
|
-
const storedEmbedding = cached.embedding;
|
|
449
|
-
if (storedEmbedding) {
|
|
450
|
-
// Hard gate: dimension must match, or the embedding is stale
|
|
451
|
-
// (different model) and cannot be compared with cosine.
|
|
452
|
-
// A mismatched/stale embedding degrades gracefully to text match.
|
|
453
|
-
if (storedEmbedding.dimension !== EMBEDDING_DIMENSION) {
|
|
454
|
-
const haystack =
|
|
455
|
-
`${entry.title}\n${entry.body}`.toLowerCase();
|
|
456
|
-
score = haystack.includes(query.text.toLowerCase()) ? 0.5 : 0;
|
|
457
|
-
} else {
|
|
458
|
-
score = cosineSimilarity(
|
|
459
|
-
queryEmbedding,
|
|
460
|
-
storedEmbedding.vector,
|
|
461
|
-
);
|
|
462
|
-
}
|
|
463
|
-
} else {
|
|
464
|
-
// No embedding stored; fall back to text match
|
|
465
|
-
const haystack =
|
|
466
|
-
`${entry.title}\n${entry.body}`.toLowerCase();
|
|
467
|
-
score = haystack.includes(query.text.toLowerCase()) ? 0.5 : 0;
|
|
468
|
-
}
|
|
469
|
-
} else {
|
|
470
|
-
// Text search fallback
|
|
471
|
-
const haystack = `${entry.title}\n${entry.body}`.toLowerCase();
|
|
472
|
-
score = haystack.includes(query.text.toLowerCase()) ? 1 : 0;
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
// Title-anchor floor: a query that matches the entry title (either
|
|
476
|
-
// direction, case-insensitive) must not be diluted below the top
|
|
477
|
-
// ranks by long-body mean pooling (see 20260905-171239-533).
|
|
478
|
-
if (titleMatches(query.text, entry.title)) {
|
|
479
|
-
score = Math.max(score, 0.75);
|
|
480
|
-
}
|
|
389
|
+
filtered.push(cached);
|
|
390
|
+
}
|
|
481
391
|
|
|
482
|
-
|
|
392
|
+
let ranked: { entry: JournalEntry; score: number }[];
|
|
393
|
+
if (!query.text) {
|
|
394
|
+
ranked = filtered.map(({ entry }) => ({
|
|
395
|
+
entry,
|
|
396
|
+
score: entry.created.getTime(),
|
|
397
|
+
}));
|
|
398
|
+
ranked.sort((a, b) => b.score - a.score || (a.entry.id < b.entry.id ? 1 : a.entry.id > b.entry.id ? -1 : 0));
|
|
399
|
+
} else {
|
|
400
|
+
const ids = filtered.map(({ entry }) => entry.id);
|
|
401
|
+
const sameEntries = !entryCacheChanged && lexicalIndexSnapshot !== undefined &&
|
|
402
|
+
lexicalIndexSnapshot.ids.length === ids.length &&
|
|
403
|
+
ids.every((id, index) => lexicalIndexSnapshot!.ids[index] === id);
|
|
404
|
+
let lexicalIndex: MiniSearch;
|
|
405
|
+
if (sameEntries) {
|
|
406
|
+
lexicalIndex = lexicalIndexSnapshot!.index;
|
|
483
407
|
} else {
|
|
484
|
-
|
|
485
|
-
|
|
408
|
+
const builtIndex = new MiniSearch({
|
|
409
|
+
fields: ["title", "body"],
|
|
410
|
+
storeFields: ["id"],
|
|
411
|
+
});
|
|
412
|
+
builtIndex.addAll(filtered.map(({ entry }) => ({
|
|
413
|
+
id: entry.id,
|
|
414
|
+
title: entry.title,
|
|
415
|
+
body: entry.body,
|
|
416
|
+
})));
|
|
417
|
+
lexicalIndexSnapshot = { ids, index: builtIndex };
|
|
418
|
+
lexicalIndex = builtIndex;
|
|
486
419
|
}
|
|
487
|
-
|
|
488
|
-
|
|
420
|
+
const byId = new Map(filtered.map(({ entry }) => [entry.id, entry]));
|
|
421
|
+
const lexical = lexicalIndex.search(query.text, { combineWith: "OR" });
|
|
422
|
+
if (hasLexicalTerms(query.text)) {
|
|
423
|
+
ranked = lexical.flatMap((match) => {
|
|
424
|
+
const entry = byId.get(String(match.id));
|
|
425
|
+
return entry ? [{ entry, score: match.score }] : [];
|
|
426
|
+
});
|
|
427
|
+
} else {
|
|
428
|
+
ranked = substringMatches(filtered, query.text);
|
|
429
|
+
}
|
|
430
|
+
ranked.sort((a, b) =>
|
|
431
|
+
b.score - a.score ||
|
|
432
|
+
b.entry.created.getTime() - a.entry.created.getTime() ||
|
|
433
|
+
(a.entry.id < b.entry.id ? 1 : a.entry.id > b.entry.id ? -1 : 0),
|
|
434
|
+
);
|
|
489
435
|
}
|
|
490
436
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
// Sort by score descending
|
|
494
|
-
entries.sort((a, b) => b.score - a.score);
|
|
437
|
+
// Count lexical candidates after metadata filtering, before paging.
|
|
438
|
+
const total = ranked.length;
|
|
495
439
|
|
|
496
440
|
// Apply offset + limit
|
|
497
|
-
|
|
441
|
+
ranked = ranked.slice(offset, offset + limit);
|
|
498
442
|
|
|
499
443
|
return {
|
|
500
|
-
entries:
|
|
444
|
+
entries: ranked.map((e) => e.entry),
|
|
501
445
|
total,
|
|
502
446
|
allTags: Array.from(tagSet).sort(),
|
|
503
447
|
};
|
package/src/tools.ts
CHANGED
|
@@ -76,11 +76,11 @@ export function JournalRead(store: JournalStore): ToolDefinition {
|
|
|
76
76
|
export function JournalSearch(store: JournalStore): ToolDefinition {
|
|
77
77
|
return {
|
|
78
78
|
name: "journal_search",
|
|
79
|
-
description: "
|
|
79
|
+
description: "Find candidate journal entries by lexical OR matching query words in titles and bodies. Candidates are not relevance claims. All filters are optional and combined with AND logic. Use with no arguments to list recent entries. Use offset to paginate. Before starting complex tasks, search the journal for relevant past context. The journal is global across all projects but each entry records which project it was written from.",
|
|
80
80
|
input: {
|
|
81
81
|
type: "object",
|
|
82
82
|
properties: {
|
|
83
|
-
text: { type: "string", description: "
|
|
83
|
+
text: { type: "string", description: "Query words for lexical candidate matching in titles and bodies" },
|
|
84
84
|
project: { type: "string", description: "Filter by project directory" },
|
|
85
85
|
tags: { type: "string", description: "Comma-separated tags; all must match" },
|
|
86
86
|
limit: { type: "integer", minimum: 1, description: "Maximum entries to return" },
|
|
@@ -99,7 +99,7 @@ export function JournalSearch(store: JournalStore): ToolDefinition {
|
|
|
99
99
|
const tagsLine = formatTagsInUse(result.allTags);
|
|
100
100
|
if (!result.entries.length) return { content: `No journal entries found.${tagsLine}` };
|
|
101
101
|
const offset = (input.offset as number | undefined) ?? 0;
|
|
102
|
-
const header = `Found ${result.total} entries (showing ${offset + 1}–${offset + result.entries.length}):`;
|
|
102
|
+
const header = `Found ${result.total} candidate entries (showing ${offset + 1}–${offset + result.entries.length}):`;
|
|
103
103
|
const lines = result.entries.map((entry) => `${entry.id}\n ${entry.title}${entry.tags.length ? ` [${entry.tags.join(", ")}]` : ""}\n ${entry.created.toISOString()}`);
|
|
104
104
|
return { content: `${header}${tagsLine}\n\n${lines.join("\n\n")}` };
|
|
105
105
|
},
|
package/src/doctor.test.ts
DELETED
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
import { afterEach, expect, test } from "bun:test";
|
|
2
|
-
import * as fs from "node:fs/promises";
|
|
3
|
-
import * as os from "node:os";
|
|
4
|
-
import * as path from "node:path";
|
|
5
|
-
import { diagnoseJournal } from "./doctor";
|
|
6
|
-
|
|
7
|
-
const roots: string[] = [];
|
|
8
|
-
async function fixture() {
|
|
9
|
-
const root = await fs.mkdtemp(path.join(os.tmpdir(), "journal-doctor-"));
|
|
10
|
-
roots.push(root);
|
|
11
|
-
await fs.mkdir(path.join(root, "journal"));
|
|
12
|
-
return path.join(root, "journal");
|
|
13
|
-
}
|
|
14
|
-
async function entry(dir: string, name: string, yamlText?: string, sidecar?: string) {
|
|
15
|
-
await fs.writeFile(path.join(dir, `${name}.md`), yamlText === undefined ? "plain text" : `---\n${yamlText}\n---\nbody\n`);
|
|
16
|
-
if (sidecar !== undefined) await fs.writeFile(path.join(dir, `${name}.embedding`), sidecar);
|
|
17
|
-
}
|
|
18
|
-
afterEach(async () => { await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); });
|
|
19
|
-
|
|
20
|
-
test("classifies frontmatter, created values, sidecars and multiple findings deterministically", async () => {
|
|
21
|
-
const dir = await fixture();
|
|
22
|
-
const valid = 'title: Good\ncreated: "2025-01-01T00:00:00Z"';
|
|
23
|
-
await entry(dir, "00-valid", valid, JSON.stringify({ v: 2, model: "wrong-model", dimension: 2, vector: [0.1, 0.2] }));
|
|
24
|
-
await entry(dir, "01-missing-title", "created: \"2025-01-01\"");
|
|
25
|
-
await entry(dir, "02-empty-title", 'title: ""');
|
|
26
|
-
await entry(dir, "03-wrong-title", "title: 42");
|
|
27
|
-
await entry(dir, "04-missing-created", "title: Present", "[1,2]");
|
|
28
|
-
await entry(dir, "05-date-only", "title: Present\ndate: \"2025-01-01\"");
|
|
29
|
-
await entry(dir, "06-invalid-created", 'title: Present\ncreated: "not a date"');
|
|
30
|
-
await entry(dir, "07-wrong-created", "title: Present\ncreated: 42");
|
|
31
|
-
await entry(dir, "08-yaml-timestamp", "title: Present\ncreated: 2025-01-01T00:00:00Z");
|
|
32
|
-
await entry(dir, "09-bad-yaml", "title: [broken");
|
|
33
|
-
await entry(dir, "10-no-frontmatter");
|
|
34
|
-
await entry(dir, "11-malformed-json", valid, "{");
|
|
35
|
-
await entry(dir, "12-vector-type", valid, '{"v":2,"model":"m","dimension":1,"vector":["x"]}');
|
|
36
|
-
await entry(dir, "13-empty-vector", valid, '{"v":2,"model":"m","dimension":0,"vector":[]}');
|
|
37
|
-
await entry(dir, "14-vector-dimension", valid, '{"v":2,"model":"m","dimension":3,"vector":[1,2]}');
|
|
38
|
-
await entry(dir, "15-legacy", valid, "[1,2]");
|
|
39
|
-
await entry(dir, "16-legacy-invalid", valid, '[1,"x"]');
|
|
40
|
-
await entry(dir, "17-missing-sidecar", valid);
|
|
41
|
-
|
|
42
|
-
const inventory = async () => Promise.all((await fs.readdir(dir)).sort().map(async (f) => [f, await fs.readFile(path.join(dir, f)), (await fs.stat(path.join(dir, f))).mtimeMs] as const));
|
|
43
|
-
const before = await inventory();
|
|
44
|
-
const report = await diagnoseJournal(path.dirname(dir));
|
|
45
|
-
expect(await diagnoseJournal(path.dirname(dir))).toEqual(report);
|
|
46
|
-
expect(report.counts.files).toBe(18);
|
|
47
|
-
const codes = (name: string) => report.findings.filter((f) => f.path.startsWith(name)).map((f) => f.code);
|
|
48
|
-
expect(codes("01-missing-title")).toContain("frontmatter_title_invalid");
|
|
49
|
-
expect(codes("02-empty-title")).toContain("frontmatter_title_invalid");
|
|
50
|
-
expect(codes("03-wrong-title")).toContain("frontmatter_title_invalid");
|
|
51
|
-
expect(codes("04-missing-created")).toContain("created_missing");
|
|
52
|
-
expect(codes("05-date-only")).toContain("created_missing");
|
|
53
|
-
expect(codes("06-invalid-created")).toContain("created_invalid");
|
|
54
|
-
expect(codes("07-wrong-created")).toContain("frontmatter_created_invalid");
|
|
55
|
-
expect(codes("08-yaml-timestamp")).toContain("frontmatter_created_invalid");
|
|
56
|
-
expect(codes("09-bad-yaml")).toContain("yaml_malformed");
|
|
57
|
-
expect(codes("10-no-frontmatter")).toContain("frontmatter_missing_or_malformed");
|
|
58
|
-
expect(codes("11-malformed-json")).toContain("embedding_json_malformed");
|
|
59
|
-
for (const name of ["12-vector-type", "13-empty-vector", "14-vector-dimension", "16-legacy-invalid"]) expect(codes(name)).toContain("embedding_invalid");
|
|
60
|
-
expect(codes("00-valid")).toContain("embedding_incompatible");
|
|
61
|
-
expect(codes("15-legacy")).toContain("embedding_model_unknown");
|
|
62
|
-
expect(codes("17-missing-sidecar")).toContain("embedding_missing");
|
|
63
|
-
expect(codes("04-missing-created")).toEqual(expect.arrayContaining(["created_missing", "embedding_model_unknown"]));
|
|
64
|
-
expect(report.findings).toEqual([...report.findings].sort((a, b) => a.path.localeCompare(b.path) || a.code.localeCompare(b.code) || a.severity.localeCompare(b.severity)));
|
|
65
|
-
expect(report.findings.some((f) => /broken|Good|Present|not a date/.test(JSON.stringify(f)))).toBe(false);
|
|
66
|
-
expect(await inventory()).toEqual(before);
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
test("empty, missing, and injected unreadable roots/files have explicit incomplete semantics", async () => {
|
|
70
|
-
const missing = await diagnoseJournal("/path/that/does/not/exist");
|
|
71
|
-
expect(missing.complete).toBe(false);
|
|
72
|
-
expect(missing.findings[0]?.code).toBe("root_unreadable");
|
|
73
|
-
const dir = await fixture();
|
|
74
|
-
expect((await diagnoseJournal(path.dirname(dir))).complete).toBe(true);
|
|
75
|
-
await entry(dir, "a", "title: A"); await entry(dir, "b", "title: B");
|
|
76
|
-
const failingRead = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
|
|
77
|
-
if (String(p).endsWith("a.md")) throw new Error("injected");
|
|
78
|
-
return fs.readFile(p, ...(args as [any]));
|
|
79
|
-
} } as unknown as typeof fs;
|
|
80
|
-
const report = await diagnoseJournal(path.dirname(dir), failingRead);
|
|
81
|
-
expect(report.complete).toBe(false);
|
|
82
|
-
expect(report.findings.some((f) => f.path === "a.md" && f.code === "file_unreadable")).toBe(true);
|
|
83
|
-
expect(report.findings.some((f) => f.path === "b.md" && f.code === "created_missing")).toBe(true);
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
test("CLI emits exact bytes and exit codes for clean, warning, error, incomplete and bad invocation", async () => {
|
|
87
|
-
const dir = await fixture();
|
|
88
|
-
const cli = path.resolve(import.meta.dir, "journal-doctor.ts");
|
|
89
|
-
const invoke = (...args: string[]) => Bun.spawnSync([process.execPath, "run", cli, ...args], { stdout: "pipe", stderr: "pipe" });
|
|
90
|
-
const assertCli = (result: ReturnType<typeof invoke>, stdout: string, stderr: string, exitCode: number) =>
|
|
91
|
-
expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout, stderr, exitCode });
|
|
92
|
-
assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":0,"errors":0,"warnings":0},"findings":[]}\n', "", 0);
|
|
93
|
-
|
|
94
|
-
await entry(dir, "warning", 'title: Warn\ncreated: "2025-01-01"');
|
|
95
|
-
assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":1,"errors":0,"warnings":1},"findings":[{"path":"warning.embedding","code":"embedding_missing","severity":"warning"}]}\n', "", 0);
|
|
96
|
-
|
|
97
|
-
await entry(dir, "invalid", "title: 12");
|
|
98
|
-
assertCli(invoke(path.dirname(dir)), '{"complete":true,"counts":{"files":2,"errors":1,"warnings":2},"findings":[{"path":"invalid.embedding","code":"embedding_missing","severity":"warning"},{"path":"invalid.md","code":"frontmatter_title_invalid","severity":"error"},{"path":"warning.embedding","code":"embedding_missing","severity":"warning"}]}\n', "", 1);
|
|
99
|
-
|
|
100
|
-
assertCli(invoke("/path/that/does/not/exist"), '{"complete":false,"counts":{"files":0,"errors":1,"warnings":0},"findings":[{"path":".","code":"root_unreadable","severity":"error"}]}\n', "", 2);
|
|
101
|
-
assertCli(invoke(path.dirname(dir), "extra"), "", "usage: bun run journal:doctor [config-dir]\n", 2);
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
test("inventories orphan and associated sidecars independently and emits stable full report", async () => {
|
|
105
|
-
const dir = await fixture();
|
|
106
|
-
const valid = 'title: Good\ncreated: "2025-01-01T00:00:00Z"';
|
|
107
|
-
await entry(dir, "A", valid);
|
|
108
|
-
await entry(dir, "a", valid, "{");
|
|
109
|
-
await fs.writeFile(path.join(dir, "_.embedding"), "{");
|
|
110
|
-
const expected = { complete: true, counts: { files: 2, errors: 2, warnings: 2 }, findings: [
|
|
111
|
-
{ path: "A.embedding", code: "embedding_missing", severity: "warning" },
|
|
112
|
-
{ path: "_.embedding", code: "embedding_json_malformed", severity: "error" },
|
|
113
|
-
{ path: "_.embedding", code: "embedding_orphan", severity: "warning" },
|
|
114
|
-
{ path: "a.embedding", code: "embedding_json_malformed", severity: "error" },
|
|
115
|
-
] };
|
|
116
|
-
const report = await diagnoseJournal(path.dirname(dir));
|
|
117
|
-
expect(JSON.stringify(report)).toBe(JSON.stringify(expected));
|
|
118
|
-
const cli = path.resolve(import.meta.dir, "journal-doctor.ts");
|
|
119
|
-
const result = Bun.spawnSync([process.execPath, "run", cli, path.dirname(dir)], { stdout: "pipe", stderr: "pipe" });
|
|
120
|
-
expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout: `${JSON.stringify(expected)}\n`, stderr: "", exitCode: 1 });
|
|
121
|
-
});
|
|
122
|
-
|
|
123
|
-
test("sidecar validation survives markdown read failure; listed sidecar disappearance is incomplete", async () => {
|
|
124
|
-
const dir = await fixture();
|
|
125
|
-
await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"', "{");
|
|
126
|
-
const failingRead = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
|
|
127
|
-
if (String(p).endsWith("a.md")) throw Object.assign(new Error("denied"), { code: "EACCES" });
|
|
128
|
-
return fs.readFile(p, ...(args as [any]));
|
|
129
|
-
} } as unknown as typeof fs;
|
|
130
|
-
const failed = await diagnoseJournal(path.dirname(dir), failingRead);
|
|
131
|
-
expect(failed.complete).toBe(false);
|
|
132
|
-
expect(failed.findings.map((f) => f.code)).toEqual(expect.arrayContaining(["file_unreadable", "embedding_json_malformed"]));
|
|
133
|
-
|
|
134
|
-
let removed = false;
|
|
135
|
-
const disappearing = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
|
|
136
|
-
if (String(p).endsWith("a.embedding") && !removed) { removed = true; throw Object.assign(new Error("gone"), { code: "ENOENT" }); }
|
|
137
|
-
return fs.readFile(p, ...(args as [any]));
|
|
138
|
-
} } as unknown as typeof fs;
|
|
139
|
-
const gone = await diagnoseJournal(path.dirname(dir), disappearing);
|
|
140
|
-
expect(gone.complete).toBe(false);
|
|
141
|
-
expect(gone.findings.some((f) => f.path === "a.embedding" && f.code === "embedding_unreadable")).toBe(true);
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
test("diagnosis never initializes model pipeline or fetches, while warmup activates trap in child", async () => {
|
|
145
|
-
const dir = await fixture();
|
|
146
|
-
await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"');
|
|
147
|
-
const childCode = `
|
|
148
|
-
import { mock } from "bun:test";
|
|
149
|
-
let pipelines = 0, fetches = 0;
|
|
150
|
-
const originalFetch = globalThis.fetch;
|
|
151
|
-
globalThis.fetch = async (...args) => { fetches++; throw new Error("FETCH_SENTINEL"); };
|
|
152
|
-
mock.module("@huggingface/transformers", () => ({ pipeline: async () => { pipelines++; throw new Error("PIPELINE_SENTINEL"); } }));
|
|
153
|
-
const { diagnoseJournal } = await import(${JSON.stringify(path.resolve(import.meta.dir, "doctor.ts"))});
|
|
154
|
-
const { warmupEmbedder } = await import(${JSON.stringify(path.resolve(import.meta.dir, "embeddings.ts"))});
|
|
155
|
-
await diagnoseJournal(${JSON.stringify(path.dirname(dir))});
|
|
156
|
-
if (pipelines !== 0 || fetches !== 0) throw new Error("diagnosis activated model/fetch trap");
|
|
157
|
-
try { await warmupEmbedder(); throw new Error("pipeline sentinel was not caught"); } catch (error) { if (!String(error).includes("PIPELINE_SENTINEL")) throw error; }
|
|
158
|
-
if (pipelines !== 1 || fetches !== 0) throw new Error("pipeline trap count mismatch");
|
|
159
|
-
globalThis.fetch = originalFetch;
|
|
160
|
-
console.log(JSON.stringify({ pipelines, fetches }));
|
|
161
|
-
`;
|
|
162
|
-
const result = Bun.spawnSync([process.execPath, "-e", childCode], { stdout: "pipe", stderr: "pipe" });
|
|
163
|
-
expect({ stdout: result.stdout.toString(), stderr: result.stderr.toString(), exitCode: result.exitCode }).toEqual({ stdout: '{"pipelines":1,"fetches":0}\n', stderr: "", exitCode: 0 });
|
|
164
|
-
});
|
|
165
|
-
|
|
166
|
-
test("only flat regular inventory files are read and each sidecar is read once", async () => {
|
|
167
|
-
const dir = await fixture();
|
|
168
|
-
await entry(dir, "a", 'title: Good\ncreated: "2025-01-01T00:00:00Z"', "[1]");
|
|
169
|
-
await fs.mkdir(path.join(dir, "nested"));
|
|
170
|
-
await fs.writeFile(path.join(dir, "nested", "hidden.md"), "bad");
|
|
171
|
-
await fs.writeFile(path.join(dir, "nested", "hidden.embedding"), "{");
|
|
172
|
-
await fs.writeFile(path.join(dir, "ignored.txt"), "bad");
|
|
173
|
-
await fs.symlink(path.join(dir, "a.md"), path.join(dir, "linked.md"));
|
|
174
|
-
const reads = new Map<string, number>();
|
|
175
|
-
const counting = { ...fs, readFile: async (p: Parameters<typeof fs.readFile>[0], ...args: any[]) => {
|
|
176
|
-
const name = path.basename(String(p)); reads.set(name, (reads.get(name) ?? 0) + 1);
|
|
177
|
-
return fs.readFile(p, ...(args as [any]));
|
|
178
|
-
} } as unknown as typeof fs;
|
|
179
|
-
const report = await diagnoseJournal(path.dirname(dir), counting);
|
|
180
|
-
expect(report.counts.files).toBe(1);
|
|
181
|
-
expect([...reads.entries()].sort()).toEqual([["a.embedding", 1], ["a.md", 1]]);
|
|
182
|
-
expect(report.findings.map((f) => f.path)).toEqual(["a.embedding", "a.embedding"]);
|
|
183
|
-
});
|