open-memex 0.6.0 → 0.6.1-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -0
- package/README.md +546 -387
- package/README.zh-CN.md +381 -447
- package/dist/cli.js +71 -7
- package/dist/doctor.js +23 -1
- package/dist/init.js +3 -0
- package/dist/retrieve/cjk.js +14 -2
- package/dist/retrieve/query.js +54 -0
- package/dist/retrieve/search.js +20 -19
- package/dist/store/audit.js +101 -0
- package/dist/store/db.js +5 -0
- package/dist/store/markdown.js +6 -1
- package/dist/tools/memory.js +9 -1
- package/dist/tools/ops.js +6 -6
- package/docs/CONCEPTS.md +181 -0
- package/docs/CONCEPTS.zh-CN.md +163 -0
- package/docs/V2-DESIGN.md +80 -0
- package/package.json +1 -1
- package/scripts/smoke-pure.ts +36 -0
- package/src/cli.ts +78 -7
- package/src/doctor.ts +23 -1
- package/src/init.ts +3 -0
- package/src/retrieve/cjk.ts +18 -2
- package/src/retrieve/query.ts +58 -0
- package/src/retrieve/search.ts +27 -19
- package/src/store/audit.ts +136 -0
- package/src/store/db.ts +5 -0
- package/src/store/markdown.ts +6 -1
- package/src/tools/memory.ts +16 -1
- package/src/tools/ops.ts +6 -6
- package/docs/USER-GUIDE.md +0 -196
- package/docs/USER-GUIDE.zh-CN.md +0 -160
package/dist/cli.js
CHANGED
|
@@ -41,12 +41,14 @@ Examples:
|
|
|
41
41
|
open-memex list --scope personal --limit 20`,
|
|
42
42
|
search: `Search memories by keyword (BM25 full-text), best matches first.
|
|
43
43
|
|
|
44
|
-
Usage: open-memex search "query" [--scope project|personal|both] [--type T] [--limit N]
|
|
44
|
+
Usage: open-memex search "query" [--scope project|personal|both] [--type T] [--limit N] [--explain]
|
|
45
45
|
|
|
46
46
|
Flags:
|
|
47
|
-
--scope
|
|
48
|
-
--type
|
|
49
|
-
--limit
|
|
47
|
+
--scope project (default), personal, or both
|
|
48
|
+
--type filter by memory type
|
|
49
|
+
--limit max results
|
|
50
|
+
--explain show the FTS expression, per-hit score, and how many matches
|
|
51
|
+
lifecycle hid (superseded / retracted / archived)
|
|
50
52
|
|
|
51
53
|
Example:
|
|
52
54
|
open-memex search "deploy checklist" --scope both`,
|
|
@@ -304,6 +306,18 @@ Usage: open-memex doctor
|
|
|
304
306
|
|
|
305
307
|
Example:
|
|
306
308
|
open-memex doctor`,
|
|
309
|
+
audit: `Memory health audit (read-only, reports never mutates): near-duplicate
|
|
310
|
+
pairs among active memories, stale actives untouched for 90+ days, broken
|
|
311
|
+
supersede chains, personal files found inside the repo memory dir, and
|
|
312
|
+
index/file drift.
|
|
313
|
+
|
|
314
|
+
Usage: open-memex audit [--scope project|personal|both]
|
|
315
|
+
|
|
316
|
+
Flags:
|
|
317
|
+
--scope project (default), personal, or both
|
|
318
|
+
|
|
319
|
+
Example:
|
|
320
|
+
open-memex audit --scope both`,
|
|
307
321
|
};
|
|
308
322
|
function usage(exitCode = 1) {
|
|
309
323
|
console.log(`open-memex CLI
|
|
@@ -311,7 +325,7 @@ function usage(exitCode = 1) {
|
|
|
311
325
|
Usage:
|
|
312
326
|
open-memex where
|
|
313
327
|
open-memex list [--scope project|personal] [--type T] [--limit N]
|
|
314
|
-
open-memex search "query" [--scope project|personal|both] [--type T] [--limit N]
|
|
328
|
+
open-memex search "query" [--scope project|personal|both] [--type T] [--limit N] [--explain]
|
|
315
329
|
open-memex add "content" [--scope project|personal] [--type T] [--tag t1,t2]
|
|
316
330
|
open-memex supersede <id> "new content" [--type T] [--tag t1,t2]
|
|
317
331
|
open-memex status <id> active|deprecated|retracted|archived
|
|
@@ -328,6 +342,7 @@ Usage:
|
|
|
328
342
|
open-memex submit <id...> [--branch <name>] [--base <branch>]
|
|
329
343
|
open-memex pr-status [--apply]
|
|
330
344
|
open-memex reindex
|
|
345
|
+
open-memex audit [--scope project|personal|both]
|
|
331
346
|
open-memex scopes
|
|
332
347
|
open-memex migrate [--from <key>] [--to <key>]
|
|
333
348
|
[--dry-run] [--on-conflict newer|overwrite|skip]
|
|
@@ -621,6 +636,36 @@ async function main() {
|
|
|
621
636
|
process.exitCode = 1;
|
|
622
637
|
return;
|
|
623
638
|
}
|
|
639
|
+
if (cmd === "audit") {
|
|
640
|
+
const flags = parseFlags(rest);
|
|
641
|
+
syncScope(project.key, "cli");
|
|
642
|
+
syncScope(PERSONAL_SCOPE.key, "cli");
|
|
643
|
+
const keys = flags.scope === "personal"
|
|
644
|
+
? [PERSONAL_SCOPE.key]
|
|
645
|
+
: flags.scope === "both"
|
|
646
|
+
? [project.key, PERSONAL_SCOPE.key]
|
|
647
|
+
: [project.key];
|
|
648
|
+
const { runAudit, STALE_DAYS } = await import("./store/audit.js");
|
|
649
|
+
const r = runAudit(keys);
|
|
650
|
+
console.log(`audit: ${r.totalMemories} memories (${keys.length === 2 ? "project+personal" : keys[0] === PERSONAL_SCOPE.key ? "personal" : "project"})`);
|
|
651
|
+
console.log(` near-duplicate active pairs: ${r.duplicatePairs.length}`);
|
|
652
|
+
for (const p of r.duplicatePairs.slice(0, 10)) {
|
|
653
|
+
console.log(` ${p.a} ~ ${p.b} (similarity ${p.score.toFixed(2)}) — consider supersede`);
|
|
654
|
+
}
|
|
655
|
+
console.log(` stale actives (>= ${STALE_DAYS} days untouched): ${r.stale.length}`);
|
|
656
|
+
for (const s of r.stale.slice(0, 10)) {
|
|
657
|
+
console.log(` ${s.id} — ${s.ageDays} days`);
|
|
658
|
+
}
|
|
659
|
+
console.log(` broken supersede chains: ${r.brokenChains.length}`);
|
|
660
|
+
for (const b of r.brokenChains) {
|
|
661
|
+
console.log(` ${b.id} -> missing ${b.missingTarget}`);
|
|
662
|
+
}
|
|
663
|
+
console.log(` personal files inside repo memory dir: ${r.personalInRepo.length}`);
|
|
664
|
+
for (const f of r.personalInRepo)
|
|
665
|
+
console.log(` ${f}`);
|
|
666
|
+
console.log(` index drift: ${r.drift.filesNotIndexed} files not indexed, ${r.drift.rowsMissingFiles} rows missing files`);
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
624
669
|
// `config` prints the effective configuration (defaults + file). No DB needed.
|
|
625
670
|
// `config set <key> <value>` persists a setting to the config file.
|
|
626
671
|
if (cmd === "config") {
|
|
@@ -743,11 +788,20 @@ async function main() {
|
|
|
743
788
|
: flags.scope === "project"
|
|
744
789
|
? [project.key]
|
|
745
790
|
: [project.key, PERSONAL_SCOPE.key];
|
|
791
|
+
const explain = flags.explain === "true";
|
|
792
|
+
const stats = explain
|
|
793
|
+
? { ftsQuery: "", candidates: 0, hiddenSuperseded: 0, hiddenExcluded: 0 }
|
|
794
|
+
: undefined;
|
|
746
795
|
const hits = search(query, {
|
|
747
796
|
scopeKeys: keys,
|
|
748
797
|
limit: flags.limit ? Number(flags.limit) : undefined,
|
|
749
798
|
type: flags.type,
|
|
799
|
+
stats,
|
|
750
800
|
});
|
|
801
|
+
if (explain && stats) {
|
|
802
|
+
console.log(`fts: ${stats.ftsQuery || "(empty query)"}`);
|
|
803
|
+
console.log(`candidates: ${stats.candidates}, hidden by lifecycle: ${stats.hiddenSuperseded} superseded, ${stats.hiddenExcluded} retracted/archived`);
|
|
804
|
+
}
|
|
751
805
|
if (hits.length === 0) {
|
|
752
806
|
console.log("(no matches)");
|
|
753
807
|
return;
|
|
@@ -756,7 +810,8 @@ async function main() {
|
|
|
756
810
|
const tag = h.scope_key === PERSONAL_SCOPE.key ? "personal" : "project";
|
|
757
811
|
const dep = h.status === "deprecated" ? " [deprecated]" : "";
|
|
758
812
|
const rev = hitStateLabel(h);
|
|
759
|
-
|
|
813
|
+
const sc = explain ? ` score=${h.score.toFixed(3)}` : "";
|
|
814
|
+
console.log(`[${tag}/${h.type}]${dep}${rev}${sc} ${h.id} ${h.snippet.replace(/\s+/g, " ").trim()}`);
|
|
760
815
|
}
|
|
761
816
|
return;
|
|
762
817
|
}
|
|
@@ -1261,7 +1316,16 @@ async function main() {
|
|
|
1261
1316
|
}
|
|
1262
1317
|
main()
|
|
1263
1318
|
.catch((err) => {
|
|
1264
|
-
|
|
1319
|
+
// D57: lock contention past the busy timeout should read as a transient
|
|
1320
|
+
// condition, not a stack trace. The memory file is written before the
|
|
1321
|
+
// index update, so a locked-out write still lands on the next sync.
|
|
1322
|
+
const code = err?.code;
|
|
1323
|
+
if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED") {
|
|
1324
|
+
console.error("open-memex: the memory store is busy (another open-memex process is writing). Wait a moment and retry — nothing was lost.");
|
|
1325
|
+
}
|
|
1326
|
+
else {
|
|
1327
|
+
console.error(err);
|
|
1328
|
+
}
|
|
1265
1329
|
process.exit(1);
|
|
1266
1330
|
})
|
|
1267
1331
|
.finally(() => {
|
package/dist/doctor.js
CHANGED
|
@@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url";
|
|
|
10
10
|
import { loadConfig, configSource, DEFAULT_CONFIG } from "./config.js";
|
|
11
11
|
import { paths } from "./paths.js";
|
|
12
12
|
import { resolveCwdScope } from "./scope.js";
|
|
13
|
+
import { db, backendName } from "./store/db.js";
|
|
13
14
|
import { LEGACY_PACKAGE_NAME, opencodeGlobalConfigPath, userMcpConfigPath } from "./init.js";
|
|
14
15
|
const EXPECTED_TOOLS = [
|
|
15
16
|
"memory_add",
|
|
@@ -59,6 +60,27 @@ function storageCheck() {
|
|
|
59
60
|
return { name: "storage", ok: false, detail: String(err) };
|
|
60
61
|
}
|
|
61
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* D57: several processes share one index (CLI, MCP per editor, opencode
|
|
65
|
+
* plugin). WAL + a 5s busy timeout is what keeps an overlapping write a
|
|
66
|
+
* brief wait instead of an instant failure — report the live settings.
|
|
67
|
+
*/
|
|
68
|
+
function lockingCheck() {
|
|
69
|
+
try {
|
|
70
|
+
const d = db();
|
|
71
|
+
const jm = d.prepare("PRAGMA journal_mode").get().journal_mode;
|
|
72
|
+
const bt = d.prepare("PRAGMA busy_timeout").get().timeout;
|
|
73
|
+
const ok = jm === "wal" && bt >= 1000;
|
|
74
|
+
return {
|
|
75
|
+
name: "locking",
|
|
76
|
+
ok,
|
|
77
|
+
detail: `journal=${jm}, busy_timeout=${bt}ms (${backendName()})`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
return { name: "locking", ok: false, detail: String(err) };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
62
84
|
/** Spawn the real MCP server, handshake, and verify the five tools list. */
|
|
63
85
|
function mcpCheck() {
|
|
64
86
|
const name = "mcp";
|
|
@@ -277,7 +299,7 @@ function vscodeMcpCheck() {
|
|
|
277
299
|
}
|
|
278
300
|
export async function runDoctor() {
|
|
279
301
|
console.log("open-memex doctor");
|
|
280
|
-
const checks = [nodeCheck(), configCheck(), scopeCheck(), storageCheck(), vscodeMcpCheck()];
|
|
302
|
+
const checks = [nodeCheck(), configCheck(), scopeCheck(), storageCheck(), lockingCheck(), vscodeMcpCheck()];
|
|
281
303
|
checks.push(await mcpCheck());
|
|
282
304
|
checks.push(legacyCheck());
|
|
283
305
|
let allOk = true;
|
package/dist/init.js
CHANGED
|
@@ -792,6 +792,9 @@ export async function initProject(opts) {
|
|
|
792
792
|
// D51: close the init→first-use gap — one concrete next step so a new user
|
|
793
793
|
// sees what "it works" looks like instead of stopping at "it's wired".
|
|
794
794
|
console.log(` Next step: \`open-memex add "standup is at 9:30"\` — then ask your agent what it remembers.`);
|
|
795
|
+
// Privacy in one line, at the moment of maximum doubt (right after a tool
|
|
796
|
+
// just wrote into the user's editors): local-only, nothing uploaded.
|
|
797
|
+
console.log(` Privacy: memories live only on this machine — personal ones never leave it, and nothing is uploaded anywhere.`);
|
|
795
798
|
// D50: init completed — the bare-`open-memex` first-run offer won't ask again.
|
|
796
799
|
markFirstRunDone("initialized");
|
|
797
800
|
}
|
package/dist/retrieve/cjk.js
CHANGED
|
@@ -40,8 +40,11 @@ function esc(t) {
|
|
|
40
40
|
* column. Multi-char runs become an OR of bigrams (recall-oriented; bm25
|
|
41
41
|
* ranks docs matching more bigrams higher). Single chars stay unigrams.
|
|
42
42
|
* Returns "" when the query has no CJK.
|
|
43
|
+
*
|
|
44
|
+
* Options: `dropBigrams` filters question bigrams (什么/怎么/…) when other
|
|
45
|
+
* terms remain; `maxTerms` caps the term count (applied after dedupe).
|
|
43
46
|
*/
|
|
44
|
-
export function cjkQueryExpr(query) {
|
|
47
|
+
export function cjkQueryExpr(query, opts = {}) {
|
|
45
48
|
const parts = [];
|
|
46
49
|
for (const m of query.matchAll(CJK_RUN_RE)) {
|
|
47
50
|
const chars = [...m[0]];
|
|
@@ -54,5 +57,14 @@ export function cjkQueryExpr(query) {
|
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
}
|
|
57
|
-
|
|
60
|
+
let terms = [...new Set(parts)];
|
|
61
|
+
if (opts.dropBigrams && terms.length > 1) {
|
|
62
|
+
const kept = terms.filter((t) => !opts.dropBigrams.has(t.slice(1, -1).replace(/""/g, '"')));
|
|
63
|
+
if (kept.length > 0)
|
|
64
|
+
terms = kept;
|
|
65
|
+
}
|
|
66
|
+
if (opts.maxTerms && terms.length > opts.maxTerms) {
|
|
67
|
+
terms = terms.slice(0, opts.maxTerms);
|
|
68
|
+
}
|
|
69
|
+
return terms.join(" OR ");
|
|
58
70
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Search query construction (pure logic, no sqlite).
|
|
2
|
+
//
|
|
3
|
+
// Free-text questions — "how do we configure the SSO login", "中文记忆怎么
|
|
4
|
+
// 检索" — are the natural way users and agents ask, but fed raw into FTS5
|
|
5
|
+
// they dilute ranking: question words ("how", "什么", "怎么") match
|
|
6
|
+
// everything and drag the score. So the query builder keeps the existing
|
|
7
|
+
// shape (Latin prefix terms OR-ed, CJK bigrams OR-ed against the `cjk`
|
|
8
|
+
// column, bm25 ranks) and additionally drops a small set of function words
|
|
9
|
+
// on both sides, dedupes, and caps term counts so a rambling sentence
|
|
10
|
+
// cannot crowd out its own content words. If filtering would empty a
|
|
11
|
+
// side, the unfiltered terms are used — a query must never be made worse
|
|
12
|
+
// than the raw one.
|
|
13
|
+
import { cjkQueryExpr, hasCjk } from "./cjk.js";
|
|
14
|
+
/** English function words that carry no retrieval signal in questions. */
|
|
15
|
+
const LATIN_STOPWORDS = new Set([
|
|
16
|
+
"the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",
|
|
17
|
+
"without", "is", "are", "was", "were", "be", "been", "do", "does", "did",
|
|
18
|
+
"how", "what", "why", "when", "where", "which", "who", "whom", "we", "i",
|
|
19
|
+
"you", "he", "she", "it", "they", "them", "us", "me", "my", "our", "your",
|
|
20
|
+
"his", "her", "its", "their", "this", "that", "these", "those", "there",
|
|
21
|
+
"here", "can", "could", "should", "would", "will", "shall", "may", "might",
|
|
22
|
+
"please", "if", "then", "than", "so", "as", "at", "by", "from", "into",
|
|
23
|
+
"about", "over", "under", "again", "once", "just",
|
|
24
|
+
]);
|
|
25
|
+
/** Chinese question bigrams — the CJK counterpart of the list above. */
|
|
26
|
+
const CJK_QUESTION_BIGRAMS = new Set([
|
|
27
|
+
"什么", "怎么", "如何", "为何", "为什", "哪些", "哪个", "请问",
|
|
28
|
+
]);
|
|
29
|
+
const MAX_TERMS = 24;
|
|
30
|
+
function latinTerms(q) {
|
|
31
|
+
const raw = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
|
|
32
|
+
const kept = raw.filter((t) => t.length > 1 && !LATIN_STOPWORDS.has(t));
|
|
33
|
+
const chosen = kept.length > 0 ? kept : raw;
|
|
34
|
+
return [...new Set(chosen)].slice(0, MAX_TERMS);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Convert free-text query into a safe FTS5 MATCH expression.
|
|
38
|
+
* Latin tokens keep the old behavior (prefix match on content/tags/type).
|
|
39
|
+
* CJK runs become an OR of bigrams against the `cjk` column (see cjk.ts).
|
|
40
|
+
* Mixed queries OR the two parts together.
|
|
41
|
+
*/
|
|
42
|
+
export function toFtsQuery(q) {
|
|
43
|
+
const latin = latinTerms(q)
|
|
44
|
+
.map((t) => `"${t.replace(/"/g, '""')}"*`)
|
|
45
|
+
.join(" OR ");
|
|
46
|
+
const cjk = hasCjk(q)
|
|
47
|
+
? cjkQueryExpr(q, { dropBigrams: CJK_QUESTION_BIGRAMS, maxTerms: MAX_TERMS })
|
|
48
|
+
: "";
|
|
49
|
+
if (latin && cjk)
|
|
50
|
+
return `(${latin}) OR {cjk}:(${cjk})`;
|
|
51
|
+
if (cjk)
|
|
52
|
+
return `{cjk}:(${cjk})`;
|
|
53
|
+
return latin;
|
|
54
|
+
}
|
package/dist/retrieve/search.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { db } from "../store/db.js";
|
|
2
|
-
import {
|
|
2
|
+
import { toFtsQuery } from "./query.js";
|
|
3
3
|
function toHit(r) {
|
|
4
4
|
return {
|
|
5
5
|
id: r.id,
|
|
@@ -44,7 +44,7 @@ export function hitStateLabel(h) {
|
|
|
44
44
|
* drafts/rejected rank after unreviewed content;
|
|
45
45
|
* - deprecated stays visible as a warning but ranks after active.
|
|
46
46
|
*/
|
|
47
|
-
function resolveVisible(rows, limit) {
|
|
47
|
+
function resolveVisible(rows, limit, stats) {
|
|
48
48
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
49
49
|
const fullRow = (id) => {
|
|
50
50
|
const cached = byId.get(id);
|
|
@@ -65,10 +65,15 @@ function resolveVisible(rows, limit) {
|
|
|
65
65
|
const tiers = [[], [], [], []];
|
|
66
66
|
const deprecated = [];
|
|
67
67
|
for (const r of rows) {
|
|
68
|
-
if (r.status === "retracted" || r.status === "archived")
|
|
68
|
+
if (r.status === "retracted" || r.status === "archived") {
|
|
69
|
+
if (stats)
|
|
70
|
+
stats.hiddenExcluded++;
|
|
69
71
|
continue;
|
|
72
|
+
}
|
|
70
73
|
let target = r;
|
|
71
74
|
if (r.status === "superseded") {
|
|
75
|
+
if (stats)
|
|
76
|
+
stats.hiddenSuperseded++;
|
|
72
77
|
let cur = r;
|
|
73
78
|
const chain = new Set([r.id]);
|
|
74
79
|
while (cur.status === "superseded" && cur.superseded_by) {
|
|
@@ -96,24 +101,18 @@ function resolveVisible(rows, limit) {
|
|
|
96
101
|
return [...tiers[0], ...tiers[1], ...tiers[2], ...deprecated].slice(0, limit);
|
|
97
102
|
}
|
|
98
103
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
* Mixed queries OR the two parts together.
|
|
104
|
+
* FTS5 query construction lives in query.ts (pure, unit-tested): free-text
|
|
105
|
+
* questions are split into content terms (OR-ed; bm25 ranks), with
|
|
106
|
+
* function/question words filtered so they can't dilute the ranking.
|
|
103
107
|
*/
|
|
104
|
-
function toFtsQuery(q) {
|
|
105
|
-
const latin = (q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [])
|
|
106
|
-
.map((t) => `"${t.replace(/"/g, '""')}"*`)
|
|
107
|
-
.join(" OR ");
|
|
108
|
-
const cjk = hasCjk(q) ? cjkQueryExpr(q) : "";
|
|
109
|
-
if (latin && cjk)
|
|
110
|
-
return `(${latin}) OR {cjk}:(${cjk})`;
|
|
111
|
-
if (cjk)
|
|
112
|
-
return `{cjk}:(${cjk})`;
|
|
113
|
-
return latin;
|
|
114
|
-
}
|
|
115
108
|
export function search(query, opts = {}) {
|
|
116
109
|
const q = toFtsQuery(query);
|
|
110
|
+
if (opts.stats) {
|
|
111
|
+
opts.stats.ftsQuery = q;
|
|
112
|
+
opts.stats.candidates = 0;
|
|
113
|
+
opts.stats.hiddenSuperseded = 0;
|
|
114
|
+
opts.stats.hiddenExcluded = 0;
|
|
115
|
+
}
|
|
117
116
|
if (!q)
|
|
118
117
|
return [];
|
|
119
118
|
const limit = Math.max(1, Math.min(opts.limit ?? 8, 50));
|
|
@@ -146,7 +145,9 @@ export function search(query, opts = {}) {
|
|
|
146
145
|
.all(...params);
|
|
147
146
|
// FTS5 bm25: lower = better; invert for intuition.
|
|
148
147
|
const raw = rows.map((r) => ({ ...r, score: -r.score }));
|
|
149
|
-
|
|
148
|
+
if (opts.stats)
|
|
149
|
+
opts.stats.candidates = raw.length;
|
|
150
|
+
return resolveVisible(raw, limit, opts.stats);
|
|
150
151
|
}
|
|
151
152
|
export function list(scopeKey, opts = {}) {
|
|
152
153
|
const limit = Math.max(1, Math.min(opts.limit ?? 20, 100));
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { db } from "./db.js";
|
|
3
|
+
import { readMemoryFile, iterInRepoMemoryFiles, iterMemoryFiles, } from "./markdown.js";
|
|
4
|
+
import { similarity, NEAR_DUP_THRESHOLD } from "./lifecycle.js";
|
|
5
|
+
import { projectRoot } from "../paths.js";
|
|
6
|
+
import { loadConfig } from "../config.js";
|
|
7
|
+
// Memory health audit (D57). Read-only: reports, never mutates.
|
|
8
|
+
// Checks, per open question "retrieval explainability & memory health":
|
|
9
|
+
// - near-duplicate pairs among active memories (same dedup yardstick
|
|
10
|
+
// as write-time: Jaccard >= 0.8) that were never superseded;
|
|
11
|
+
// - stale actives: untouched for STALE_DAYS, still in the retrieval /
|
|
12
|
+
// injection pool;
|
|
13
|
+
// - broken supersede chains: superseded_by pointing at a missing id;
|
|
14
|
+
// - personal files inside the in-repo memory dir (iron-rule breach:
|
|
15
|
+
// personal memory must never land in a shared repo);
|
|
16
|
+
// - index/file drift after sync (should be zero; reported if not).
|
|
17
|
+
export const STALE_DAYS = 90;
|
|
18
|
+
export function runAudit(scopeKeys) {
|
|
19
|
+
const d = db();
|
|
20
|
+
const rows = d
|
|
21
|
+
.prepare(`SELECT id, scope_key, status, updated_at, superseded_by, content, file_path
|
|
22
|
+
FROM memories`)
|
|
23
|
+
.all();
|
|
24
|
+
const inScope = rows.filter((r) => scopeKeys.includes(r.scope_key));
|
|
25
|
+
const ids = new Set(rows.map((r) => r.id));
|
|
26
|
+
// Near-duplicate pairs: active only, same scope. O(n^2) per scope with a
|
|
27
|
+
// length pre-filter; local stores are small enough for this to be fine.
|
|
28
|
+
const pairs = [];
|
|
29
|
+
const byScope = new Map();
|
|
30
|
+
for (const r of inScope) {
|
|
31
|
+
if (r.status !== "active")
|
|
32
|
+
continue;
|
|
33
|
+
const arr = byScope.get(r.scope_key) ?? [];
|
|
34
|
+
arr.push(r);
|
|
35
|
+
byScope.set(r.scope_key, arr);
|
|
36
|
+
}
|
|
37
|
+
for (const [scopeKey, arr] of byScope) {
|
|
38
|
+
for (let i = 0; i < arr.length; i++) {
|
|
39
|
+
for (let j = i + 1; j < arr.length; j++) {
|
|
40
|
+
const a = arr[i];
|
|
41
|
+
const b = arr[j];
|
|
42
|
+
const la = a.content.length;
|
|
43
|
+
const lb = b.content.length;
|
|
44
|
+
if (Math.min(la, lb) / Math.max(la, lb) < 0.5)
|
|
45
|
+
continue;
|
|
46
|
+
const score = similarity(a.content, b.content);
|
|
47
|
+
if (score >= NEAR_DUP_THRESHOLD) {
|
|
48
|
+
pairs.push({ a: a.id, b: b.id, scopeKey, score });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
pairs.sort((x, y) => y.score - x.score);
|
|
54
|
+
const now = Date.now();
|
|
55
|
+
const stale = inScope
|
|
56
|
+
.filter((r) => r.status === "active")
|
|
57
|
+
.map((r) => ({ id: r.id, scopeKey: r.scope_key, ageDays: Math.floor((now - r.updated_at) / 86_400_000) }))
|
|
58
|
+
.filter((s) => s.ageDays >= STALE_DAYS)
|
|
59
|
+
.sort((x, y) => y.ageDays - x.ageDays);
|
|
60
|
+
const brokenChains = inScope
|
|
61
|
+
.filter((r) => r.superseded_by && !ids.has(r.superseded_by))
|
|
62
|
+
.map((r) => ({ id: r.id, scopeKey: r.scope_key, missingTarget: r.superseded_by }));
|
|
63
|
+
// Personal files must never sit in the in-repo dir. Read-only: this only
|
|
64
|
+
// parses frontmatter of files already in the repo tree.
|
|
65
|
+
const personalInRepo = [];
|
|
66
|
+
try {
|
|
67
|
+
const root = projectRoot();
|
|
68
|
+
const memoryDir = loadConfig().memoryDir;
|
|
69
|
+
for (const fp of iterInRepoMemoryFiles(root, memoryDir)) {
|
|
70
|
+
const mf = readMemoryFile(fp);
|
|
71
|
+
if (mf && mf.fm.scope === "personal")
|
|
72
|
+
personalInRepo.push(fp);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// No project root (e.g. personal-only usage): nothing to scan.
|
|
77
|
+
}
|
|
78
|
+
// Drift: index rows whose file vanished, and files on disk the index
|
|
79
|
+
// never picked up. syncScope normally heals both before audit runs.
|
|
80
|
+
const indexedPaths = new Set(rows.map((r) => r.file_path));
|
|
81
|
+
let rowsMissingFiles = 0;
|
|
82
|
+
for (const r of inScope) {
|
|
83
|
+
if (!r.file_path || !fs.existsSync(r.file_path))
|
|
84
|
+
rowsMissingFiles++;
|
|
85
|
+
}
|
|
86
|
+
let filesNotIndexed = 0;
|
|
87
|
+
for (const key of scopeKeys) {
|
|
88
|
+
for (const fp of iterMemoryFiles(key)) {
|
|
89
|
+
if (!indexedPaths.has(fp))
|
|
90
|
+
filesNotIndexed++;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
totalMemories: inScope.length,
|
|
95
|
+
duplicatePairs: pairs,
|
|
96
|
+
stale,
|
|
97
|
+
brokenChains,
|
|
98
|
+
personalInRepo,
|
|
99
|
+
drift: { filesNotIndexed, rowsMissingFiles },
|
|
100
|
+
};
|
|
101
|
+
}
|
package/dist/store/db.js
CHANGED
|
@@ -111,6 +111,11 @@ export function db() {
|
|
|
111
111
|
const Database = loadDatabase();
|
|
112
112
|
const d = new Database(indexDb);
|
|
113
113
|
d.exec("PRAGMA journal_mode = WAL;");
|
|
114
|
+
// Multiple processes share this index (CLI, one MCP server per editor,
|
|
115
|
+
// the opencode plugin). Without a busy timeout, bun:sqlite defaults to
|
|
116
|
+
// 0 — any overlapping write fails instantly. 5s matches the tolerance
|
|
117
|
+
// better-sqlite3 was already giving the CLI/MCP side (D57).
|
|
118
|
+
d.exec("PRAGMA busy_timeout = 5000;");
|
|
114
119
|
d.exec("PRAGMA synchronous = NORMAL;");
|
|
115
120
|
d.exec("PRAGMA foreign_keys = ON;");
|
|
116
121
|
d.exec(TABLE_SCHEMA);
|
package/dist/store/markdown.js
CHANGED
|
@@ -205,7 +205,12 @@ export function writeMemoryFile(fm, body) {
|
|
|
205
205
|
// Personal stays in appdata and never leaves the machine.
|
|
206
206
|
const dir = memoriesDirFor(fm.scope_key);
|
|
207
207
|
const filePath = path.join(dir, `${fm.id}.md`);
|
|
208
|
-
|
|
208
|
+
// Atomic publish (D57): write a sibling tmp file and rename over the
|
|
209
|
+
// target, so a concurrent reader (another process mid-sync) never sees a
|
|
210
|
+
// torn half-written memory. Same-directory rename is atomic on one fs.
|
|
211
|
+
const tmpPath = `${filePath}.tmp-${process.pid}`;
|
|
212
|
+
fs.writeFileSync(tmpPath, serialize(fm, body), "utf8");
|
|
213
|
+
fs.renameSync(tmpPath, filePath);
|
|
209
214
|
const st = fs.statSync(filePath);
|
|
210
215
|
return { filePath, mtimeMs: st.mtimeMs };
|
|
211
216
|
}
|
package/dist/tools/memory.js
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
import { tool } from "@opencode-ai/plugin/tool";
|
|
2
1
|
import { addMemory, searchMemories, listMemories, supersedeMemory, forgetMemory, memoryAddArgs, memorySearchArgs, memoryListArgs, memorySupersedeArgs, memoryForgetArgs, TOOL_DESCRIPTIONS, withOutboxNote, } from "./ops.js";
|
|
2
|
+
// Local stand-in for the SDK's tool() helper — same signature, and the SDK
|
|
3
|
+
// function is a runtime identity (`return input`). Do NOT import it from
|
|
4
|
+
// @opencode-ai/plugin at runtime: that package is a devDependency, and when
|
|
5
|
+
// opencode loads this plugin by file:// URL from a global npm install
|
|
6
|
+
// nothing resolves the module — the import fails and opencode silently
|
|
7
|
+
// skips the whole plugin: no error, no tools (D58).
|
|
8
|
+
function tool(input) {
|
|
9
|
+
return input;
|
|
10
|
+
}
|
|
3
11
|
export function makeTools(getScope, cfg) {
|
|
4
12
|
const memory_add = tool({
|
|
5
13
|
description: TOOL_DESCRIPTIONS.memory_add,
|
package/dist/tools/ops.js
CHANGED
|
@@ -40,12 +40,12 @@ export const TOOL_DESCRIPTIONS = {
|
|
|
40
40
|
memory_list: "List memories in a scope, newest first. Useful for browsing what is remembered, or verifying that a save landed.",
|
|
41
41
|
memory_supersede: "Replace an existing memory with a newer version. The old memory is kept as history (status: superseded) and retrieval returns the new one. Use when a saved fact becomes outdated and should be replaced rather than duplicated.",
|
|
42
42
|
memory_forget: "Delete a memory by id. Use when the user asks to forget something.",
|
|
43
|
-
memory_status: "Show the project memory sync pipeline: drafts waiting in the outbox (appdata), memories in the repo awaiting review or published, and any repo files not yet committed. Call this at session start, when the server reports drafts waiting for review, or when the user says 'sync memory' (or '同步记忆'); then ask the user which drafts to sync.",
|
|
44
|
-
memory_submit: "Move outbox drafts into the repo memory dir for review: copies the drafts in as proposed (or keeps a local approval), commits locally on the current branch, and moves the outbox originals out. Never creates a branch on its own — pass branch= only with the user's explicit approval for the full chain. Prints the push and PR commands — those need the user's explicit approval and are never run automatically.",
|
|
45
|
-
memory_propose: "Copy personal memories into the project outbox as review drafts. The personal originals stay put.",
|
|
46
|
-
memory_promote: "Advance a project memory one step up the review ladder (proposed → approved → published), or reject it with a note. Rejected memories are never deleted — they can be revised and resubmitted.",
|
|
47
|
-
memory_resolve: "List git-conflicted memory files, or attempt a field-level 3-way merge of one. Semantic conflicts are reported, never auto-resolved.",
|
|
48
|
-
memory_pr_status: "Read the current branch's GitHub PR and map its review state onto each in-repo memory: merged PR → published, PR approval → approved (approved_by = reviewer), changes-requested → suggestion only. Report by default; apply=true performs the mapped transitions locally (no push).",
|
|
43
|
+
memory_status: "Show the project memory sync pipeline: drafts waiting in the outbox (appdata), memories in the repo awaiting review or published, and any repo files not yet committed. Call this at session start, when the server reports drafts waiting for review, or when the user says 'sync memory' (or '同步记忆'); then ask the user which drafts to sync. (MCP server and the `open-memex sync-status` CLI; the opencode native plugin does not expose this tool.)",
|
|
44
|
+
memory_submit: "Move outbox drafts into the repo memory dir for review: copies the drafts in as proposed (or keeps a local approval), commits locally on the current branch, and moves the outbox originals out. Never creates a branch on its own — pass branch= only with the user's explicit approval for the full chain. Prints the push and PR commands — those need the user's explicit approval and are never run automatically. (MCP server and the `open-memex submit` CLI; the opencode native plugin does not expose this tool.)",
|
|
45
|
+
memory_propose: "Copy personal memories into the project outbox as review drafts. The personal originals stay put. (MCP server and the `open-memex propose` CLI; the opencode native plugin does not expose this tool.)",
|
|
46
|
+
memory_promote: "Advance a project memory one step up the review ladder (proposed → approved → published), or reject it with a note. Rejected memories are never deleted — they can be revised and resubmitted. (MCP server and the `open-memex promote` CLI; the opencode native plugin does not expose this tool.)",
|
|
47
|
+
memory_resolve: "List git-conflicted memory files, or attempt a field-level 3-way merge of one. Semantic conflicts are reported, never auto-resolved. (MCP server and the `open-memex resolve` CLI; the opencode native plugin does not expose this tool.)",
|
|
48
|
+
memory_pr_status: "Read the current branch's GitHub PR and map its review state onto each in-repo memory: merged PR → published, PR approval → approved (approved_by = reviewer), changes-requested → suggestion only. Report by default; apply=true performs the mapped transitions locally (no push). (MCP server and the `open-memex pr-status` CLI; the opencode native plugin does not expose this tool.)",
|
|
49
49
|
};
|
|
50
50
|
/** Shared zod input shapes (raw shape, not z.object — hosts wrap as needed). */
|
|
51
51
|
export const scopeArg = z
|