open-memex 0.7.3 → 0.8.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -3
- package/CHANGELOG.md +14 -0
- package/README.md +61 -5
- package/README.zh-CN.md +14 -1
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +18 -2
- package/dist/init.js +5 -3
- package/dist/retrieve/alias-memory.js +130 -0
- package/dist/retrieve/query.js +20 -9
- package/dist/retrieve/search.js +51 -14
- package/dist/retrieve/synonyms.js +183 -0
- package/dist/store/db.js +3 -1
- package/dist/store/lifecycle.js +3 -0
- package/dist/store/markdown.js +7 -0
- package/dist/store/sync.js +15 -4
- package/dist/tools/ops.js +37 -0
- package/docs/V2-DESIGN.md +9 -2
- package/docs/release-runbook.md +8 -2
- package/docs/retrieval-ablation-study.md +191 -0
- package/package.json +5 -3
- package/scripts/bench-longmemeval.ts +342 -0
- package/scripts/bench-semantic.ts +198 -0
- package/scripts/retrieval-eval/fixture.ts +182 -0
- package/scripts/retrieval-eval.ts +83 -0
- package/scripts/smoke-pure.ts +82 -0
- package/src/cli.ts +18 -2
- package/src/init.ts +5 -3
- package/src/retrieve/alias-memory.ts +154 -0
- package/src/retrieve/query.ts +23 -9
- package/src/retrieve/search.ts +67 -15
- package/src/retrieve/synonyms.ts +184 -0
- package/src/store/db.ts +3 -1
- package/src/store/lifecycle.ts +6 -0
- package/src/store/markdown.ts +18 -0
- package/src/store/sync.ts +18 -3
- package/src/tools/ops.ts +49 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index-time synonym/translation expansion (D80).
|
|
3
|
+
*
|
|
4
|
+
* Stone's rule: solve vocabulary mismatch at write time, so any related
|
|
5
|
+
* word finds the answer with a plain round-1 query. At sync/index time
|
|
6
|
+
* every memory's keywords are expanded through a curated, checked-in
|
|
7
|
+
* equivalence map and the variants are written into the indexed `aliases`
|
|
8
|
+
* column (FTS + CJK bigram columns both pick them up via sync.ts). The
|
|
9
|
+
* query side stays simple: one FTS round, no trigger, no second chance.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately NOT an embedding model or an LLM call: deterministic,
|
|
12
|
+
* auditable, zero dependencies, and the failure mode (a bad synonym)
|
|
13
|
+
* is a visible line in this file, not a silent vector.
|
|
14
|
+
*
|
|
15
|
+
* Groups are bidirectional — any member expands to the other members.
|
|
16
|
+
* Keep groups tight: an over-broad synonym costs precision on every
|
|
17
|
+
* query, and at index time a bad entry pollutes every document instead
|
|
18
|
+
* of one query. When in doubt, leave it out. A map update only reaches
|
|
19
|
+
* already-indexed memories after a reindex.
|
|
20
|
+
*
|
|
21
|
+
* Design note (D80): the map covers the *precise* slice — EN↔ZH
|
|
22
|
+
* translation pairs and dev-domain equivalents where embeddings are
|
|
23
|
+
* fuzzy (ship/deploy are neighbors in vector space but "单点登录"/SSO
|
|
24
|
+
* is an exact pair no embedding guarantees). Embeddings cover the
|
|
25
|
+
* semantic long tail; this map covers what must never be missed.
|
|
26
|
+
*/
|
|
27
|
+
import { LATIN_STOPWORDS } from "./query.js";
|
|
28
|
+
const CJK_RE = /[\u4e00-\u9fff]/;
|
|
29
|
+
/**
|
|
30
|
+
* Curated equivalence groups. EN dev-domain vocabulary, ZH equivalents,
|
|
31
|
+
* and EN↔ZH translation pairs live in one bidirectional group: any member
|
|
32
|
+
* expands to all the others, in either direction.
|
|
33
|
+
*/
|
|
34
|
+
const GROUPS = [
|
|
35
|
+
["deploy", "ship", "release", "rollout", "publish", "部署", "发布", "上线"],
|
|
36
|
+
["test", "testing", "tests", "测试", "单测", "集成测试"],
|
|
37
|
+
["regression test", "回归测试"],
|
|
38
|
+
["rollback", "revert", "回滚", "撤回", "撤销", "回退"],
|
|
39
|
+
["incident", "outage", "postmortem", "故障", "事故", "宕机"],
|
|
40
|
+
["auth", "authentication", "login", "sso", "single sign-on", "单点登录", "登录"],
|
|
41
|
+
["secret", "secrets", "credential", "credentials", "token", "密钥"],
|
|
42
|
+
["database", "db", "数据库"],
|
|
43
|
+
["backup", "restore", "备份", "恢复"],
|
|
44
|
+
["retention", "保留"],
|
|
45
|
+
["production", "生产", "生产环境"],
|
|
46
|
+
["staging", "预发布环境", "预发布"],
|
|
47
|
+
["wipe", "wipes", "清空", "清除"],
|
|
48
|
+
["nightly", "night", "每晚"],
|
|
49
|
+
["thursday", "周四", "星期四"],
|
|
50
|
+
["mandatory", "强制", "必须"],
|
|
51
|
+
["flaky", "flake", "unreliable", "intermittent"],
|
|
52
|
+
["bug", "defect", "issue"],
|
|
53
|
+
["ci", "pipeline"],
|
|
54
|
+
["cache", "caching", "cached"],
|
|
55
|
+
["invalidate", "invalidation", "expire", "expiry"],
|
|
56
|
+
["oncall", "on-call", "pager"],
|
|
57
|
+
["migration", "migrate", "migrating"],
|
|
58
|
+
["index", "indexing", "indexed"],
|
|
59
|
+
["slow", "latency", "performance", "perf"],
|
|
60
|
+
["refactor", "refactoring"],
|
|
61
|
+
["merge", "merging"],
|
|
62
|
+
["branch", "branches"],
|
|
63
|
+
["commit", "commits"],
|
|
64
|
+
["review", "code review", "评审", "代码评审"],
|
|
65
|
+
["mock", "stub"],
|
|
66
|
+
["debug", "debugging", "troubleshoot", "troubleshooting"],
|
|
67
|
+
["log", "logs", "logging"],
|
|
68
|
+
["monitor", "monitoring", "alert", "alerting"],
|
|
69
|
+
["scale", "scaling"],
|
|
70
|
+
["timeout", "timeouts"],
|
|
71
|
+
["retry", "retries"],
|
|
72
|
+
["queue", "job", "worker"],
|
|
73
|
+
["api", "endpoint"],
|
|
74
|
+
["docker", "container", "image"],
|
|
75
|
+
["kubernetes", "k8s"],
|
|
76
|
+
["env", "environment"],
|
|
77
|
+
["config", "configuration", "setting", "settings"],
|
|
78
|
+
["hotfix", "hot fix"],
|
|
79
|
+
["estimate", "estimation"],
|
|
80
|
+
["乱码", "编码问题"],
|
|
81
|
+
["并发", "并行"],
|
|
82
|
+
["重构"],
|
|
83
|
+
["规范", "约定"],
|
|
84
|
+
];
|
|
85
|
+
const termLookup = new Map();
|
|
86
|
+
const phraseEntries = [];
|
|
87
|
+
const cjkEntries = [];
|
|
88
|
+
for (const g of GROUPS) {
|
|
89
|
+
const cjkKeys = g.filter((m) => CJK_RE.test(m));
|
|
90
|
+
if (cjkKeys.length > 0)
|
|
91
|
+
cjkEntries.push({ keys: cjkKeys, siblings: [...g] });
|
|
92
|
+
for (const m of g) {
|
|
93
|
+
if (CJK_RE.test(m))
|
|
94
|
+
continue;
|
|
95
|
+
const siblings = g.filter((x) => x !== m);
|
|
96
|
+
if (m.includes(" "))
|
|
97
|
+
phraseEntries.push({ phrase: m.toLowerCase(), siblings });
|
|
98
|
+
else
|
|
99
|
+
termLookup.set(m.toLowerCase(), siblings);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Synonym variants for one lowercased Latin term (excluding the term
|
|
104
|
+
* itself). Empty array when the term has no curated synonyms.
|
|
105
|
+
*/
|
|
106
|
+
export function synonymVariants(term) {
|
|
107
|
+
return termLookup.get(term.toLowerCase()) ?? [];
|
|
108
|
+
}
|
|
109
|
+
/** Cap on auto-generated variants per memory — bounds index growth. */
|
|
110
|
+
export const MAX_AUTO_VARIANTS = 32;
|
|
111
|
+
/**
|
|
112
|
+
* Index-time expansion for one memory's text (body + tags). Keyword-
|
|
113
|
+
* centric (Stone): extract content terms, expand each through the
|
|
114
|
+
* curated map, return variants not already present in the text.
|
|
115
|
+
* Pure logic — safe to unit test, runs once per memory at sync time.
|
|
116
|
+
*/
|
|
117
|
+
export function expansionVariantsForDoc(text) {
|
|
118
|
+
const lower = text.toLowerCase();
|
|
119
|
+
// Content terms only: function words carry no signal (same filter the
|
|
120
|
+
// query builder applies, so both sides agree on what a keyword is).
|
|
121
|
+
const latinTokens = new Set((lower.match(/[a-z0-9_.\-]+/g) ?? []).filter((t) => t.length > 1 && !LATIN_STOPWORDS.has(t)));
|
|
122
|
+
const out = [];
|
|
123
|
+
const seen = new Set();
|
|
124
|
+
const push = (v) => {
|
|
125
|
+
const key = v.toLowerCase();
|
|
126
|
+
if (seen.has(key) || out.length >= MAX_AUTO_VARIANTS)
|
|
127
|
+
return;
|
|
128
|
+
seen.add(key);
|
|
129
|
+
out.push(v);
|
|
130
|
+
};
|
|
131
|
+
// A variant is "already present" when the doc has it as a token (Latin),
|
|
132
|
+
// as a substring (Latin phrase), or as a substring (CJK — no tokenizer).
|
|
133
|
+
const present = (v) => CJK_RE.test(v)
|
|
134
|
+
? text.includes(v)
|
|
135
|
+
: v.includes(" ")
|
|
136
|
+
? lower.includes(v.toLowerCase())
|
|
137
|
+
: latinTokens.has(v.toLowerCase());
|
|
138
|
+
for (const t of latinTokens) {
|
|
139
|
+
for (const v of termLookup.get(t) ?? [])
|
|
140
|
+
if (!present(v))
|
|
141
|
+
push(v);
|
|
142
|
+
}
|
|
143
|
+
for (const { phrase, siblings } of phraseEntries) {
|
|
144
|
+
if (!lower.includes(phrase))
|
|
145
|
+
continue;
|
|
146
|
+
for (const v of siblings)
|
|
147
|
+
if (!present(v))
|
|
148
|
+
push(v);
|
|
149
|
+
}
|
|
150
|
+
for (const { keys, siblings } of cjkEntries) {
|
|
151
|
+
if (!keys.some((k) => text.includes(k)))
|
|
152
|
+
continue;
|
|
153
|
+
for (const v of siblings)
|
|
154
|
+
if (!present(v))
|
|
155
|
+
push(v);
|
|
156
|
+
}
|
|
157
|
+
return out;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Expand a raw query string with synonym variants (legacy query-time
|
|
161
|
+
* helper, kept for bench scripts). The product no longer expands at
|
|
162
|
+
* query time — D80 moved expansion to the index.
|
|
163
|
+
*/
|
|
164
|
+
export function expandQueryWithSynonyms(q) {
|
|
165
|
+
const extra = new Set();
|
|
166
|
+
const latinTerms = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
|
|
167
|
+
for (const t of latinTerms) {
|
|
168
|
+
for (const s of synonymVariants(t))
|
|
169
|
+
extra.add(s);
|
|
170
|
+
}
|
|
171
|
+
for (const { keys, siblings } of cjkEntries) {
|
|
172
|
+
if (keys.some((k) => q.includes(k))) {
|
|
173
|
+
for (const s of siblings)
|
|
174
|
+
extra.add(s);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
// Don't re-add terms already in the query.
|
|
178
|
+
const have = new Set(latinTerms);
|
|
179
|
+
const additions = [...extra].filter((s) => !have.has(s.toLowerCase()) && !q.includes(s));
|
|
180
|
+
if (additions.length === 0)
|
|
181
|
+
return q;
|
|
182
|
+
return `${q} ${additions.join(" ")}`;
|
|
183
|
+
}
|
package/dist/store/db.js
CHANGED
|
@@ -35,6 +35,8 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
35
35
|
status TEXT NOT NULL DEFAULT 'active',
|
|
36
36
|
tags TEXT NOT NULL DEFAULT '',
|
|
37
37
|
aliases TEXT NOT NULL DEFAULT '',
|
|
38
|
+
alias TEXT NOT NULL DEFAULT '',
|
|
39
|
+
target TEXT NOT NULL DEFAULT '',
|
|
38
40
|
content TEXT NOT NULL,
|
|
39
41
|
cjk TEXT NOT NULL DEFAULT '',
|
|
40
42
|
content_hash TEXT NOT NULL DEFAULT '',
|
|
@@ -88,7 +90,7 @@ CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
|
|
88
90
|
END;
|
|
89
91
|
`;
|
|
90
92
|
/** Current index schema version. Bump when TABLE_SCHEMA/FTS_SCHEMA change. */
|
|
91
|
-
const SCHEMA_VERSION =
|
|
93
|
+
const SCHEMA_VERSION = 8;
|
|
92
94
|
function userVersion(d) {
|
|
93
95
|
const row = d.prepare("PRAGMA user_version").get();
|
|
94
96
|
return row.user_version;
|
package/dist/store/lifecycle.js
CHANGED
|
@@ -215,6 +215,9 @@ export function supersede(oldId, input) {
|
|
|
215
215
|
tags: input.tags ?? oldMf.fm.tags,
|
|
216
216
|
// D61: aliases carry forward unless the caller replaces them.
|
|
217
217
|
aliases: input.aliases ?? oldMf.fm.aliases,
|
|
218
|
+
// D81: alias vocabulary carries forward the same way.
|
|
219
|
+
alias: input.alias ?? oldMf.fm.alias,
|
|
220
|
+
target: input.target ?? oldMf.fm.target,
|
|
218
221
|
source: input.source ?? oldMf.fm.source,
|
|
219
222
|
status: "active",
|
|
220
223
|
created_at: now,
|
package/dist/store/markdown.js
CHANGED
|
@@ -161,6 +161,13 @@ export function normalizeFrontmatter(raw) {
|
|
|
161
161
|
return aliases.length > 0 ? { aliases } : {};
|
|
162
162
|
})()
|
|
163
163
|
: {}),
|
|
164
|
+
// D81: alias vocabulary — single trimmed strings, dropped when empty.
|
|
165
|
+
...(typeof raw.alias === "string" && raw.alias.trim()
|
|
166
|
+
? { alias: raw.alias.trim() }
|
|
167
|
+
: {}),
|
|
168
|
+
...(typeof raw.target === "string" && raw.target.trim()
|
|
169
|
+
? { target: raw.target.trim() }
|
|
170
|
+
: {}),
|
|
164
171
|
source: typeof raw.source === "string" ? raw.source : "",
|
|
165
172
|
created_at: msToRfc3339(timeToMs(raw.created_at)),
|
|
166
173
|
updated_at: msToRfc3339(timeToMs(raw.updated_at)),
|
package/dist/store/sync.js
CHANGED
|
@@ -2,13 +2,14 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { db } from "./db.js";
|
|
4
4
|
import { cjkIndexText } from "../retrieve/cjk.js";
|
|
5
|
+
import { expansionVariantsForDoc } from "../retrieve/synonyms.js";
|
|
5
6
|
import { contentHash, repairChain } from "./lifecycle.js";
|
|
6
7
|
import { iterMemoryFiles, iterInRepoMemoryFiles, readMemoryFile, timeToMs, } from "./markdown.js";
|
|
7
8
|
import { projectRoot, paths } from "../paths.js";
|
|
8
9
|
import { loadConfig } from "../config.js";
|
|
9
10
|
const UPSERT_SQL = `
|
|
10
|
-
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
11
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
11
|
+
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, alias, target, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
12
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
12
13
|
ON CONFLICT(id) DO UPDATE SET
|
|
13
14
|
scope_key = excluded.scope_key,
|
|
14
15
|
scope = excluded.scope,
|
|
@@ -20,6 +21,8 @@ const UPSERT_SQL = `
|
|
|
20
21
|
status = excluded.status,
|
|
21
22
|
tags = excluded.tags,
|
|
22
23
|
aliases = excluded.aliases,
|
|
24
|
+
alias = excluded.alias,
|
|
25
|
+
target = excluded.target,
|
|
23
26
|
content = excluded.content,
|
|
24
27
|
cjk = excluded.cjk,
|
|
25
28
|
content_hash = excluded.content_hash,
|
|
@@ -54,8 +57,16 @@ export function upsertFromFile(mf) {
|
|
|
54
57
|
function writeRow(mf, mtimeMs) {
|
|
55
58
|
const { fm, body, filePath } = mf;
|
|
56
59
|
const tags = (fm.tags ?? []).join(",");
|
|
57
|
-
|
|
58
|
-
|
|
60
|
+
// D80 index-time expansion: curated synonym/translation variants for the
|
|
61
|
+
// memory's keywords are indexed in the aliases column (and therefore the
|
|
62
|
+
// CJK bigram column below), so any related word matches in round one.
|
|
63
|
+
// Derived at sync time — the Markdown file stays the source of truth.
|
|
64
|
+
const supplied = fm.aliases ?? [];
|
|
65
|
+
const have = new Set(supplied.map((a) => a.toLowerCase()));
|
|
66
|
+
const auto = expansionVariantsForDoc(body + "\n" + tags).filter((a) => !have.has(a.toLowerCase()));
|
|
67
|
+
const aliases = [...supplied, ...auto].join(",");
|
|
68
|
+
// D81: user-defined alias vocabulary (single nickname → referent pair).
|
|
69
|
+
db().prepare(UPSERT_SQL).run(fm.id, fm.scope_key, fm.scope, fm.visibility, fm.project_name, fm.type, fm.role, fm.importance, fm.status, tags, aliases, fm.alias ?? "", fm.target ?? "", body, cjkIndexText(body + "\n" + tags + "\n" + aliases), contentHash(body), fm.superseded_by ?? null, fm.source ?? "", filePath, mtimeMs, timeToMs(fm.created_at), timeToMs(fm.updated_at), fm.review_state ?? "draft");
|
|
59
70
|
}
|
|
60
71
|
export function deleteFromIndex(id) {
|
|
61
72
|
db().prepare(`DELETE FROM memories WHERE id = ?`).run(id);
|
package/dist/tools/ops.js
CHANGED
|
@@ -68,6 +68,14 @@ export const memoryAddArgs = {
|
|
|
68
68
|
scope: scopeArg,
|
|
69
69
|
tags: z.array(z.string()).optional().describe("Optional tags for filtering."),
|
|
70
70
|
aliases: aliasesArg,
|
|
71
|
+
alias: z
|
|
72
|
+
.string()
|
|
73
|
+
.optional()
|
|
74
|
+
.describe("D81: nickname this memory defines (e.g. 香蕉计划). Pairs with `target`: the alias memory expands queries in its scope at search time, both directions, no reindex. Explicit vocabulary — always stored, not gated by capture aliases."),
|
|
75
|
+
target: z
|
|
76
|
+
.string()
|
|
77
|
+
.optional()
|
|
78
|
+
.describe("D81: what `alias` refers to (e.g. 支付系统重构项目). Only meaningful with `alias`."),
|
|
71
79
|
source: z
|
|
72
80
|
.string()
|
|
73
81
|
.optional()
|
|
@@ -106,6 +114,14 @@ export const memorySupersedeArgs = {
|
|
|
106
114
|
.describe("Category of the new memory. Defaults to the old memory's type."),
|
|
107
115
|
tags: z.array(z.string()).optional().describe("Optional tags for the new memory."),
|
|
108
116
|
aliases: aliasesArg,
|
|
117
|
+
alias: z
|
|
118
|
+
.string()
|
|
119
|
+
.optional()
|
|
120
|
+
.describe("D81: nickname the new memory defines (e.g. 香蕉计划). Omitted = inherit the old memory's alias."),
|
|
121
|
+
target: z
|
|
122
|
+
.string()
|
|
123
|
+
.optional()
|
|
124
|
+
.describe("D81: what `alias` refers to (e.g. 支付系统重构项目). Omitted = inherit."),
|
|
109
125
|
};
|
|
110
126
|
export const memoryForgetArgs = {
|
|
111
127
|
id: z.string().min(1),
|
|
@@ -170,6 +186,11 @@ function buildFrontmatter(s, body) {
|
|
|
170
186
|
status: "active",
|
|
171
187
|
tags: body.tags,
|
|
172
188
|
...(body.aliases && body.aliases.length > 0 ? { aliases: body.aliases } : {}),
|
|
189
|
+
// D81: explicit alias vocabulary — single trimmed strings, dropped
|
|
190
|
+
// when empty. Not gated by captureAliases (that gates agent-invented
|
|
191
|
+
// phrasings; this is deliberate vocabulary).
|
|
192
|
+
...(body.alias?.trim() ? { alias: body.alias.trim() } : {}),
|
|
193
|
+
...(body.target?.trim() ? { target: body.target.trim() } : {}),
|
|
173
194
|
source: body.source,
|
|
174
195
|
created_at: rfc,
|
|
175
196
|
updated_at: rfc,
|
|
@@ -228,6 +249,14 @@ export async function addMemory(getScope, cfg, args) {
|
|
|
228
249
|
aliases: cfg.captureAliases
|
|
229
250
|
? normalizeAliases((args.aliases ?? []).map((a) => redact(a, cfg.redactPatterns).content))
|
|
230
251
|
: [],
|
|
252
|
+
// D81: explicit alias vocabulary — same redaction as content, never
|
|
253
|
+
// gated by captureAliases.
|
|
254
|
+
alias: args.alias?.trim()
|
|
255
|
+
? redact(args.alias.trim(), cfg.redactPatterns).content
|
|
256
|
+
: undefined,
|
|
257
|
+
target: args.target?.trim()
|
|
258
|
+
? redact(args.target.trim(), cfg.redactPatterns).content
|
|
259
|
+
: undefined,
|
|
231
260
|
source: args.source ?? "tool",
|
|
232
261
|
});
|
|
233
262
|
const { filePath } = writeMemoryFile(fm, redacted);
|
|
@@ -322,6 +351,14 @@ export async function supersedeMemory(cfg, args) {
|
|
|
322
351
|
aliases: cfg.captureAliases && args.aliases !== undefined
|
|
323
352
|
? normalizeAliases(args.aliases.map((a) => redact(a, cfg.redactPatterns).content))
|
|
324
353
|
: undefined,
|
|
354
|
+
// D81: explicit alias vocabulary — omitted inherits the old memory's
|
|
355
|
+
// pair; same redaction as content; never gated by captureAliases.
|
|
356
|
+
alias: args.alias !== undefined
|
|
357
|
+
? redact(args.alias, cfg.redactPatterns).content || undefined
|
|
358
|
+
: undefined,
|
|
359
|
+
target: args.target !== undefined
|
|
360
|
+
? redact(args.target, cfg.redactPatterns).content || undefined
|
|
361
|
+
: undefined,
|
|
325
362
|
source: "tool",
|
|
326
363
|
});
|
|
327
364
|
upsertFromFile(oldMf);
|
package/docs/V2-DESIGN.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# OpenMemex — Design Document (protocol v0.2)
|
|
2
2
|
|
|
3
|
-
**Status:** FROZEN — protocol v0.2 (2026-09-26). Decisions D1–
|
|
3
|
+
**Status:** FROZEN — protocol v0.2 (2026-09-26). Decisions D1–D79 are settled; open questions are tracked at the end of this document.
|
|
4
4
|
**Author:** Stone, with 小沐
|
|
5
5
|
**Changelog vs v1:** incorporates round-3 review from Perplexity, Grok, Gemini, ChatGPT, DeepSeek.
|
|
6
6
|
Key changes: Design Principles section; `role` separated from `type`; two iron rules;
|
|
@@ -562,6 +562,8 @@ requirement: personal data never touches third-party services). Benchmarks to tr
|
|
|
562
562
|
- **Phase 4 — Future, signal-gated.** Cloud `RemoteProvider` customization only on: multi-private-repo
|
|
563
563
|
sharing needs, fine-grained ACL, audit/compliance mandates · optional API-backed exporters/providers
|
|
564
564
|
for enterprise knowledge systems.
|
|
565
|
+
- **Near-term backlog — committed direction, not yet built (2026-10-07).** Stone's rule: anything said as "later" is written down here. D77-L2: per-memory `retrieved_count`/`useful_count` (`useful_self` vs `useful_team`) + `memory_useful` tool. D77-L3: `memory_status` reuse report (top reused, zero-reuse, per-scope reuse rate) feeding the Spark Tank pilot metrics. D77: external bench — LongMemEval-S run 2026-10-07 (`scripts/bench-longmemeval.ts`, 470 questions, recall_any@k: R@5 97.0% / MRR 0.909 per-question, R@5 40.6% on a pooled 22k corpus; ahead of agentmemory’s BM25-only, level with their BM25+vector). MemoryAgentBench still open as a second external data point. D79 follow-up SUPERSEDED → built as **D80** (2026-10-07): index-time synonym/translation expansion shipped; 41-query fixture recall@5 0.854 → 0.976, MRR 0.791 → 0.859. Embedding-model configurability (Stone 2026-10-07): the multilingual model is his personal need (EN↔ZH), not everyone’s — the model must be user-configurable. Timing: (1) NOW in the bench only — a `--model` flag to measure the multilingual tax (multilingual vs English-only on English data), which decides the DEFAULT. (2) AT SHIP TIME — `embedding.model` in `MyOMemoryConfig` + `SETTABLE_KEYS`, model-agnostic storage from day one (dim + model id stored with vectors, cache dir), download UX and `doctor` checks. Not before the gate passes: building config UX for a layer we haven’t decided to ship is premature. If multilingual tax ≈ 0, default multilingual (covers everyone); else default English-only with a docs note.D78: curate+submit — pre-submit lint, then confidence-tier auto-curation (auto-apply / batch-confirm / explicit human). D79 follow-ups: feedback-weighted ranking once D77 counters land; embeddings gate decision against the published baseline; reranker evaluation. TDQS nit (separate PR): tighten search/list `type` to `z.enum`. Retrieval eval: grow the fixture (project-scope tier cases) as retrieval logic evolves. Retrieval split — local vs share server (Stone 2026-10-07): the two environments emphasize different mechanisms. Local (personal machine): BM25 + curated index-time expansion as the primary — deterministic, zero model download (D7), fast on CPU, index bloat bounded by the small corpus; embeddings opt-in only. Share server (team/org, enterprise): embeddings + ANN + reranker as the primary — server-grade compute with models resident, corpus too large and multi-author for any curated map; the curated map stays as a precision layer plus org glossary. The server additionally aggregates D77 useful_count across users into feedback-weighted ranking (already in D79 follow-ups) — a signal a single machine can never have. Bottom-up loop: curated pairs that many users converge on locally become org-glossary candidates. Economics flip: N users each downloading a model vs one model amortized on the server.
|
|
566
|
+
|
|
565
567
|
- **Phase 5 — Candidates recorded 2026-10-04 (not committed).** From the competitor scan (D75): an offline FTS5 retrieval eval with a published recall number (gates any semantic-layer decision) [promoted to next build, D77] · per-client trust levels on the review ladder · cold-start Markdown import into outbox drafts · `doctor` check that each editor's MCP registration points at the intended data root (mis-rooted servers are a real failure mode elsewhere). Sequencing and scope TBD per candidate.
|
|
566
568
|
|
|
567
569
|
## 19. Migration v1 → v2
|
|
@@ -1598,6 +1600,11 @@ requirement: personal data never touches third-party services). Benchmarks to tr
|
|
|
1598
1600
|
|
|
1599
1601
|
- **D78 — pre-submit curation: human-on-the-loop, not in-the-loop (2026-10-07).** Fragmented captures must be organized before submit, but the submitter's patience is the scarce resource — every manual gate lowers the submit rate, so curation defaults to automatic and the human handles exceptions. Confidence tiers: **auto-apply** (exact duplicates, trivial fragments, inferred type/tags, title normalization); **batch-confirm** (semantic merges, splits — one summary screen, one tap to accept or undo, never per-item approval); **explicit human decision** (dropping substantive content, and submit itself — the D-hard boundary "project scope never auto-submits" stands: automation covers organizing, sharing stays human). The safety net is the supersede chain: every auto-merge preserves provenance and is undoable, so pre-approval is replaced by reversibility (undo-send, not confirm-dialog). Trigger: no new step to remember — `submit` becomes curate+submit in one command, with the curation summary attached to the PR description; session wrap-up is the alternative hook (batch-silent, per the D61 checkpoint posture). Closed loop: human undos feed back into confidence thresholds, consuming the D77 `useful_count` signals. This revises the earlier review-ladder instinct of per-item approval: the human moves from in-the-loop to on-the-loop.
|
|
1600
1602
|
|
|
1603
|
+
- **D79 — query-time synonym expansion with a second-chance round (2026-10-07).** Fourth retrieval layer after D56 (query construction) and D61 (agent-side expansion guidance, capture-time aliases): a curated, checked-in synonym map (`src/retrieve/synonyms.ts` — EN dev-domain groups plus ZH groups, bidirectional) applied at query time. `search()` runs round one unchanged; when it returns fewer candidates than requested, round two re-runs with synonym variants OR-ed in and fills only the gaps — round-one ids and scores are never diluted by the expansion. Deliberately not embeddings and not an LLM call: deterministic, auditable, zero dependencies; a bad synonym is a visible line in the file, not a silent vector. `SearchStats.secondRound` records whether the round ran (surfaced in `search --explain`). Eval proof: 3 synonym-only fixture queries are unanswerable in round one by construction (their words appear in no memory); round two answers 2 of 3 at rank –2. Baseline with the round: recall@1 0.81, recall@5 0.95, MRR 0.87 over 37 queries. Follow-ups, not in this change: feedback-weighted ranking once D77 counters land; the embeddings gate decision against these numbers; reranker evaluation. **Update (2026-10-07, LongMemEval-S):** measured on 470 questions (recall_any@k protocol, per-question isolated corpora): R@1 86.0% / R@5 97.0% / R@10 98.3% / MRR 0.909 — ahead of agentmemory’s published BM25-only (86.2% / 94.6% / 0.715) and level with their BM25+vector (95.2% / 98.6% / 0.882). But the second round fired on **0 of 470** queries: the `raw.length < limit` trigger never trips once the corpus fills the page — the round as designed is a thin-corpus fallback, not a scale answer. A pooled 22k-memory stress run drops R@5 to 40.6% (the lexical ceiling, now measured instead of asserted). Consequence: the trigger needs a redesign (score/coverage-based, or always-expand-with-fusion) before the round can claim the scale story; the pooled number is the baseline an embedding layer must beat. **Ablation (2026-10-07, same bench):** OFAT over the lexical factors, limit=10, effect on MRR: OR semantics **+0.652** (AND collapses to 0.257 — the load-bearing wall); prefix matching **-0.010** (disabling it lifts R@1 86.0%→87.9% — mildly harmful, needs a second dataset before acting); stopword filtering **+0.005** (keep: cheap, principled); synonym round **0.000** at every limit and scale tested. Limit sweep: R@1 is invariant to limit (86.0% at 1–50); returns flatten at limit 5 (R@5 97.0%, +1.3pp to go to 50); round-2 fired 10/470 at limit=50 and rescued nothing (identical to no-synonyms). Pooled 22k at limit=50: R@5 still 40.6%, round-2 fired 0. Reading: the lexical lemon is squeezed — OR does the work, the rest is ±0.01. The next gains must come from ranking under distractors (embedding/reranker layer), not more lexical tweaks. Rescue experiment (same day): fallback-append vs always-expand+RRF-fusion, 470q, stratified by query–gold overlap — **zero rescues** under either design, at either scale; not one question changes outcome. The 8 residual misses are reasoning problems (preference/multi-hop/temporal), not vocabulary gaps. The round’s value lives in a mismatch slice the public data barely contains (proven on constructed queries: 2 of 3 rescued). Keep the curated map; stop spending design budget on the trigger. (Superseded same day: index-time expansion, see backlog.)
|
|
1604
|
+
- **D80 — index-time synonym/translation expansion (2026-10-07).** Supersedes D79's query-time second round (removed from `search()`; `SearchStats.secondRound` gone): the curated map (`src/retrieve/synonyms.ts` — EN dev-domain groups, ZH groups and EN↔ZH translation pairs merged into one bidirectional group list) is applied at sync/index time. `writeRow` expands each memory's keywords via `expansionVariantsForDoc()` and appends the variants to the indexed `aliases` column (FTS and CJK-bigram columns both pick them up), so any related word matches in a plain round-1 query. Query side: single FTS round, no trigger, no second chance. Keyword-centric per Stone ("重要的是关键词"): Latin content terms (same stopword filter as the query builder, so both sides agree on what a keyword is) + CJK substring matching + multi-word Latin phrases (`regression test`); variants already present in the text are skipped; capped at 32 per memory to bound index growth. Derived at sync time — the Markdown file stays the source of truth; a map update needs a reindex to backfill. *Rationale: the precise slice (exact EN↔ZH pairs like 单点登录/SSO) must never be missed, and embeddings are fuzzy where the map is exact; paying the cost once at write time keeps every query simple and trigger-free.* Eval on the 41-query cross-lingual fixture: recall@1 0.732 → 0.756, recall@5 0.854 → **0.976**, MRR 0.791 → **0.859** — all 5 EN↔ZH vocabulary misses rescued at rank 1–3. Known trade-off: a single-keyword query can rank an aliases-only hit below shorter docs sharing the variant (`how do we ship`: rank 2 → 8 — its hit is 1 of 15 variants on a long aliases field); accepted, the old rank depended on the fragile thin-corpus trigger. `expandQueryWithSynonyms` kept as a bench-only export.
|
|
1605
|
+
- **D81 — user-defined alias vocabulary as query-time expansion (2026-10-07).** Stone: an alias should be a *kind of memory* — defined once, remembered forever. Any memory type may carry `alias:` (the nickname, e.g. 香蕉计划) and `target:` (what it refers to, e.g. 支付系统重构项目) frontmatter; the alias memory defines vocabulary once per scope. At query time (`src/retrieve/alias-memory.ts`), query terms are looked up against the alias registry — one cheap `SELECT` per `search()` call over rows with non-empty alias+target (trigger A: exact registry hit, never a count heuristic; the D79 0/470 trigger is not repeated). On a hit the query is rewritten with the other side of the relation — bidirectional (alias→target and target→alias), depth < 3, cycle-safe — and re-run as round 2 (round 3 for chains). Rounds merge by normalized-max (`mergeRounds`): each round's BM25 scores are divided by that round's best (raw cross-round scores are incomparable — °C vs °F), then final = max, so an alias-side champion competes evenly with a literal-side champion — alias hits are never down-weighted for coming from a later round. Merged pool is re-ranked then cut to [1.5K, 2K] (Stone) — never back to K, or round-2 hits would be starved the way D79's slice-to-K starved them. No reindex when an alias changes (query-time). Scope-aware: a project codename stays in its project. *Rationale: D80's curated map is static and shared — personal vocabulary (小宝贝, 香蕉计划) doesn't belong in it; per-memory D61 aliases don't propagate to new memories. Alias memories close the gap: user-defined, scope-aware, effective immediately.* v1: user-explicit creation only (`memory_add` alias/target params, CLI `--alias`/`--target`, frontmatter edit); agent-proposed aliases deferred. D81 also rewords the init `captureAliases` prompt (it is not the synonym switch — D80's expansion always applies). Eval: 3 new fixture queries (香蕉计划↔支付系统重构项目 both directions + the alias memory itself), 44 queries total: recall@1 0.750, recall@5 **0.977** (43/44), MRR 0.857 — the single miss remains `how do we ship` (known D80 trade-off).
|
|
1606
|
+
|
|
1607
|
+
|
|
1601
1608
|
## Open Questions
|
|
1602
1609
|
|
|
1603
1610
|
_Pruned 2026-10-04 (D75): entries decided in D56/D57/D61/D68 and the shipped layers of the retrieval-robustness question were removed — the decisions are the record. What remains below is genuinely open._
|
|
@@ -1608,7 +1615,7 @@ _Pruned 2026-10-04 (D75): entries decided in D56/D57/D61/D68 and the shipped lay
|
|
|
1608
1615
|
|
|
1609
1616
|
- **Novice experience: the 60-second magic moment and the plain-language layer (open, 2026-10-03; trimmed 2026-10-04):** the privacy model is now stated in plain words at init (D56). Still open: init offering to save one sample memory and immediately having the user's agent answer a question with it — the user *sees* recall work before being asked to trust it — and a plain-language layer over the review ladder ("saved on this computer" / "shared with the team") with outbox/submit/promote vocabulary becoming the advanced view. This is the same vocabulary wall the PM/BA question above hits from the other side.
|
|
1610
1617
|
|
|
1611
|
-
- **Retrieval evaluation: measure recall before choosing a semantic layer (open, 2026-10-04):** the retrieval-robustness question closed with all three cheap layers shipped (D56 query construction; D61 agent-side expansion + capture-time aliases), and embeddings stayed deferred *by judgment, without numbers*. A comparable project (agent-memory) publishes an offline recall@5 of 0.98 and wears it as a badge. open-memex should build the same kind of small, fixed eval — a query set over a seeded store, recall@k / MRR for the current FTS5 pipeline — and publish the number. The semantic-layer decision then gets a gate: if FTS5 + aliases is good enough, embeddings stay deferred; if a query class measurably fails, that class defines what an optional local-embedding layer must beat. Eval first, architecture second.
|
|
1618
|
+
- **Retrieval evaluation: measure recall before choosing a semantic layer (open, 2026-10-04):** the retrieval-robustness question closed with all three cheap layers shipped (D56 query construction; D61 agent-side expansion + capture-time aliases), and embeddings stayed deferred *by judgment, without numbers*. A comparable project (agent-memory) publishes an offline recall@5 of 0.98 and wears it as a badge. open-memex should build the same kind of small, fixed eval — a query set over a seeded store, recall@k / MRR for the current FTS5 pipeline — and publish the number. The semantic-layer decision then gets a gate: if FTS5 + aliases is good enough, embeddings stay deferred; if a query class measurably fails, that class defines what an optional local-embedding layer must beat. Eval first, architecture second. **Update (2026-10-07):** the eval shipped (D77, `scripts/retrieval-eval.ts`, baseline recall@1 0.81 / recall@5 0.95 / MRR 0.87); query-time synonym expansion landed as retrieval layer 4 (D79) — embeddings stay deferred, now with numbers behind the judgment.
|
|
1612
1619
|
|
|
1613
1620
|
- **Per-client trust levels (open, 2026-10-04):** the review ladder is currently global: every host's writes face the same gates. A comparable tool (auxly) sets a trust level per agent — one agent's writes land immediately, another's queue for approval, a third is read-only. That maps naturally onto open-memex, since the ladder already exists; the open questions are the config surface (per host? per MCP client name? how does the server learn the client's identity reliably?), the default for unknown clients, and whether trust should also gate *reads* of the personal scope. Signal to watch: one user running several agents with different autonomy expectations.
|
|
1614
1621
|
|
package/docs/release-runbook.md
CHANGED
|
@@ -34,7 +34,7 @@ npm run build
|
|
|
34
34
|
最后跑全量测试。`test-full` 之前必须先 `build`:测试里有 dist 内容指纹守卫,没 build 会误报失败。
|
|
35
35
|
|
|
36
36
|
```powershell
|
|
37
|
-
|
|
37
|
+
node --experimental-strip-types scripts/test-full.ts
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
测试全绿再往下走,中间任何一步报错就停。
|
|
@@ -91,7 +91,13 @@ npm publish
|
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
3. 把版本号告诉小沐(例如"0.7.2 published"):小沐核验 npm latest,然后打 git tag `vX.Y.Z` 并建 GitHub Release。
|
|
94
|
-
4. 同步 MCP Registry。**必须等 npm 新版生效后再跑**:registry 会去 npm 抓新版本的实际包验 `mcpName` 和版本。`npm publish` 成功后先等 1–2
|
|
94
|
+
4. 同步 MCP Registry。**必须等 npm 新版生效后再跑**:registry 会去 npm 抓新版本的实际包验 `mcpName` 和版本。`npm publish` 成功后先等 1–2 分钟,再执行发布(一条一条来)。如果登录态过期(`publish` 报认证错误),先重跑登录:
|
|
95
|
+
|
|
96
|
+
```powershell
|
|
97
|
+
& "$env:USERPROFILE\tools\mcp-publisher\mcp-publisher.exe" login github
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
登录态没过期就跳过这一步,直接发布:
|
|
95
101
|
|
|
96
102
|
```powershell
|
|
97
103
|
& "$env:USERPROFILE\tools\mcp-publisher\mcp-publisher.exe" publish
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# Retrieval Ablation Study: What Actually Moves Recall in open-memex's Lexical Pipeline
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-10-07 · **Status:** complete · **Code:** `scripts/bench-longmemeval.ts` · **Data:** LongMemEval-S (`xiaowu0162/longmemeval-cleaned`, MIT)
|
|
4
|
+
|
|
5
|
+
## Abstract
|
|
6
|
+
|
|
7
|
+
We measured, on unbiased public data, which parts of open-memex's BM25 retrieval pipeline actually move recall — and by how much. Over 470 LongMemEval-S questions: our BM25-only pipeline reaches **R@5 97.0% / MRR 0.909** per-question, ahead of agentmemory's published BM25-only (86.2% / 0.715) and level with their BM25+vector (95.2% / 0.882). A one-factor-at-a-time ablation finds **OR semantics is the load-bearing wall (+0.652 MRR)**; prefix matching is mildly harmful (−0.010); stopword filtering is ~neutral (+0.005); the synonym second round measured **0.000** — its count-based trigger fired 0/470 queries. A limit sweep shows R@1 invariant to limit and returns flattening at limit 5. A pooled 22,419-memory stress run drops R@5 to 40.6%: the lexical ceiling, measured. A rescue-rate experiment (round-1-only vs fallback-append vs always-expand+RRF-fusion, stratified by query–gold vocabulary overlap) finds **zero rescues** — not one of 470 questions changes outcome under either synonym design, at either scale: LongMemEval-S barely contains the vocabulary-mismatch slice synonyms exist for (its residual misses are reasoning problems), while the mechanism is proven on constructed mismatch queries (2 of 3 rescued). The embedding gate (§4.7, local multilingual MiniLM-L12-v2, 384-dim): on the 41-query cross-lingual fixture, vector-only reaches **R@5 92.7% / MRR 0.881** vs BM25's 80.5% / 0.752 — rescuing the cross-lingual misses BM25 cannot see — while naive RRF fusion underperforms vector-only (87.8%); on pooled 22k long sessions the single-vector arm collapses (3.4%) from mean-pooling dilution, a measured methodology artifact on a non-product-representative corpus, not a verdict on embeddings for short memories.
|
|
8
|
+
|
|
9
|
+
## 1. Introduction
|
|
10
|
+
|
|
11
|
+
open-memex's retrieval pipeline accumulated eight lexical optimizations across D56, D61 and D79 (query construction, CJK handling, capture-time aliases, synonym expansion, …). Until now they were only ever measured end-to-end, on a synthetic fixture (37 queries at the time, now 41 with the cross-lingual vocabulary-mismatch slice) written by the same author as the code. Four research questions drove this study:
|
|
12
|
+
|
|
13
|
+
- **RQ1.** How does the pipeline compare to published systems on unbiased data?
|
|
14
|
+
- **RQ2.** Which pipeline factors actually move recall, and what is each factor's effect size?
|
|
15
|
+
- **RQ3.** How does page limit affect recall, and does it interact with the synonym round?
|
|
16
|
+
- **RQ4.** Where can synonym expansion rescue hits that lexical search misses, and which design maximizes rescues?
|
|
17
|
+
|
|
18
|
+
## 2. System under test
|
|
19
|
+
|
|
20
|
+
The pipeline, in order: free-text query → `toFtsQuery` (content-term extraction, function-word filtering, Latin prefix terms OR-ed, CJK bigrams OR-ed against a dedicated column, term cap) → FTS5 `bm25` → lifecycle filtering → review-tier boost + recency boost → top-`limit`. D79 adds a second-chance round: if round one returns fewer than `limit` candidates, the query is re-run with curated synonym variants OR-ed in; round-two rows only fill gaps, never re-rank round one.
|
|
21
|
+
|
|
22
|
+
Eight optimizations were enumerated from the code; five are ablatable on this bench. Tier boost, recency boost and capture-time aliases are constant on the bench (uniform tiers, simultaneous writes, sessions bypass capture) and CJK handling is vacuous on an English dataset — ablating them here would measure nothing, so they are excluded and noted.
|
|
23
|
+
|
|
24
|
+
## 3. Methodology
|
|
25
|
+
|
|
26
|
+
**Dataset.** LongMemEval-S (Wu et al., ICLR 2025): 500 questions, each over ~50 chat sessions (~115k tokens), with gold `answer_session_ids`. We exclude the 30 abstention questions per the benchmark's own retrieval protocol → **470 questions**. Each session becomes one memory (full turns, role-labeled); each question runs against an isolated corpus of its own sessions. A second **pooled** mode puts all 22,419 sessions in one corpus as a scale stress test.
|
|
27
|
+
|
|
28
|
+
**Design.** Standard ablation-study methodology: factors at two levels (on/off), one response, ceteris-paribus comparison. (1) **OFAT ablation** for main effects — baseline vs each factor disabled, one at a time. (2) **Limit sweep** — limit ∈ {1, 3, 5, 10, 20, 50} for sensitivity and the flattening point. (3) **Interaction check** — synonym × limit at 20/50, the only interaction the trigger design makes plausible. The pipeline is deterministic, so no replication is needed; every run is exactly reproducible.
|
|
29
|
+
|
|
30
|
+
**Metrics.** Primary: **MRR** (one number capturing recall and ranking). Secondary: recall_any@k (the protocol agentmemory publishes). Diagnostic: second-round firing rate; rescue rate (§4.6).
|
|
31
|
+
|
|
32
|
+
**Honest framing.** This is the retrieval stage only — no answer generation, no LLM judge — so these are *not* official LongMemEval scores. Corpus construction (full sessions, both roles) may differ from competitors'; the comparison is same-dataset/same-protocol, not a controlled bake-off.
|
|
33
|
+
|
|
34
|
+
## 4. Results
|
|
35
|
+
|
|
36
|
+
### 4.1 Baseline vs published numbers (RQ1)
|
|
37
|
+
|
|
38
|
+
Per-question isolated corpora, limit 10:
|
|
39
|
+
|
|
40
|
+
| System | R@1 | R@5 | R@10 | MRR |
|
|
41
|
+
|---|---|---|---|---|
|
|
42
|
+
| open-memex (this work, BM25-only) | 86.0% | **97.0%** | 98.3% | **0.909** |
|
|
43
|
+
| agentmemory BM25-only (published) | — | 86.2% | 94.6% | 0.715 |
|
|
44
|
+
| agentmemory BM25+vector (published) | — | 95.2% | 98.6% | 0.882 |
|
|
45
|
+
|
|
46
|
+
By question type (ours): single-session-assistant 1.000, knowledge-update 0.981, single-session-user 0.948, multi-session 0.929, temporal-reasoning 0.851, single-session-preference 0.652 MRR. Preference and temporal questions are the weak slices.
|
|
47
|
+
|
|
48
|
+
### 4.2 Scale stress: pooled 22k corpus (RQ1)
|
|
49
|
+
|
|
50
|
+
| limit | R@1 | R@5 | R@10 | MRR |
|
|
51
|
+
|---|---|---|---|---|
|
|
52
|
+
| 10 | 18.9% | 40.6% | 51.1% | 0.284 |
|
|
53
|
+
| 50 | 18.9% | 40.6% | 51.1% | 0.294 |
|
|
54
|
+
|
|
55
|
+
Same-distribution distractors drown lexical matching: R@5 falls 97.0% → 40.6%. This pooled number is the baseline any embedding layer must beat (the D77 gate, now with a measured target).
|
|
56
|
+
|
|
57
|
+
### 4.3 OFAT ablation (RQ2)
|
|
58
|
+
|
|
59
|
+
Limit 10, per-question. Effect = ΔMRR vs baseline (0.909):
|
|
60
|
+
|
|
61
|
+
| Factor disabled | R@1 | R@5 | MRR | ΔMRR |
|
|
62
|
+
|---|---|---|---|---|
|
|
63
|
+
| (baseline) | 86.0% | 97.0% | 0.909 | — |
|
|
64
|
+
| stopword filtering | 85.3% | 96.6% | 0.904 | **+0.005** (keep: cheap, principled) |
|
|
65
|
+
| OR → AND semantics | 25.3% | 26.0% | 0.257 | **+0.652** (the load-bearing wall) |
|
|
66
|
+
| prefix matching | 87.9% | 96.8% | 0.919 | **−0.010** (mildly harmful; needs a second dataset before acting) |
|
|
67
|
+
| synonym round | 86.0% | 97.0% | 0.909 | **0.000** (never fires — see §4.5) |
|
|
68
|
+
|
|
69
|
+
### 4.4 Limit sweep (RQ3)
|
|
70
|
+
|
|
71
|
+
Baseline, per-question:
|
|
72
|
+
|
|
73
|
+
| limit | R@1 | R@3 | R@5 | R@10 | MRR |
|
|
74
|
+
|---|---|---|---|---|---|
|
|
75
|
+
| 1 | 86.0% | — | — | — | 0.860 |
|
|
76
|
+
| 3 | 86.0% | 95.7% | — | — | 0.905 |
|
|
77
|
+
| 5 | 86.0% | 95.7% | 97.0% | — | 0.907 |
|
|
78
|
+
| 10 | 86.0% | 95.7% | 97.0% | 98.3% | 0.909 |
|
|
79
|
+
| 20 | 86.0% | 95.7% | 97.0% | 98.3% | 0.910 |
|
|
80
|
+
| 50 | 86.0% | 95.7% | 97.0% | 98.3% | 0.910 |
|
|
81
|
+
|
|
82
|
+
R@1 is **invariant to limit** (ranking, not paging, decides #1). Returns flatten at limit 5 (+1.3pp going to 50). The default limit of 8 needs no change for recall.
|
|
83
|
+
|
|
84
|
+
### 4.5 Synonym × limit interaction (RQ3)
|
|
85
|
+
|
|
86
|
+
The D79 trigger (`round-1 candidates < limit`) fired on **0/470** queries at limits ≤ 20 and **10/470** at limit 50 — and rescued nothing (identical numbers with the round disabled). Two conclusions: the trigger is a thin-corpus fallback, dead at any realistic scale; and MRR was the wrong metric for it anyway — a fallback round can never take #1, so its value must be measured as **rescue rate**, which §4.6 does.
|
|
87
|
+
|
|
88
|
+
### 4.6 Rescue experiment: fallback vs fusion, stratified by vocabulary overlap (RQ4)
|
|
89
|
+
|
|
90
|
+
*[Results pending — experiment running.]*
|
|
91
|
+
|
|
92
|
+
**Design (fixed).** The flaw in §4.5's measurement: it tested whether the trigger fires, not whether synonyms have value — and LongMemEval-S questions share vocabulary with their gold sessions, the exact case where synonyms are unnecessary. The corrected experiment: three arms per question — **A**: round one only; **B**: fallback append with the trigger forced to fire; **C**: always-expand + RRF fusion (k=60, depth 50). Metric: recall@K and **rescue count** (gold in the arm's top-K but not in A's). Stratification: per-question rows record query–gold lexical overlap (shared content terms); analysis splits into terciles, so the low-overlap slice — synonyms' theoretical home field — is measured separately from the high-overlap slice where they should be inert. Runs: per-question K=10, per-question K=5, pooled K=10.
|
|
93
|
+
|
|
94
|
+
**Results.**
|
|
95
|
+
|
|
96
|
+
| Corpus | K | A: round 1 | B: fallback (+rescues) | C: RRF fusion (+rescues) |
|
|
97
|
+
|---|---|---|---|---|
|
|
98
|
+
| per-question | 10 | 98.3% | 98.3% (+0) | 98.3% (+0) |
|
|
99
|
+
| per-question | 5 | 97.0% | 97.0% (+0) | 97.0% (+0) |
|
|
100
|
+
| pooled 22k | 10 | 51.1% | 51.1% (+0) | 51.1% (+0) |
|
|
101
|
+
|
|
102
|
+
Not one of the 470 questions changes outcome between arms — in any overlap tercile (low 0–5: n=228, mid 6–7, high 8+), at either scale. The expanded query's top-50 is nearly identical to round one's: with OR semantics, adding terms broadens a net that is already wide, and in the pooled corpus the gold drowns further rather than surfacing.
|
|
103
|
+
|
|
104
|
+
Miss autopsy (per-question, the 8 residual): they are **reasoning problems, not vocabulary gaps** — preference inference ("recommend publications" → a session about medical imaging, the user's field), multi-hop ("how many siblings" → count scattered across sessions), temporal reasoning ("bought 10 days ago" → "got a smoker today" needs date arithmetic). No synonym map reaches these.
|
|
105
|
+
|
|
106
|
+
The overlap distribution explains why: median 6 shared content terms between question and gold; the low-overlap tercile still yields 96.5% to arm A alone. **LongMemEval-S barely contains the vocabulary-mismatch slice** — the exact case synonym expansion exists for.
|
|
107
|
+
|
|
108
|
+
Existence proof that the mechanism works when the slice exists: the synthetic fixture's 3 synonym-only queries (vocabulary mismatch by construction) — the fallback round rescued 2 of 3 at rank ≤2 (§4.5's harness measured the trigger, this measures the round).
|
|
109
|
+
|
|
110
|
+
**Reading.** Synonym expansion's value is real but narrow and invisible on available public data: keep the curated map (cheap, harmless, proven on the constructed case), but the trigger redesign is moot — there is nothing measurable to trigger for. The pooled-scale problem (51.1%) is a *ranking-under-distractors* problem; no lexical query trick fixes it. That is the embedding gate's job — run in §4.7.
|
|
111
|
+
|
|
112
|
+
### 4.7 Semantic ranking: multilingual embeddings + RRF hybrid (RQ1, gate)
|
|
113
|
+
|
|
114
|
+
**Setup.** `scripts/bench-semantic.ts`, three arms: **BM25-only** (the §4 pipeline), **vec-only** (local `Xenova/paraphrase-multilingual-MiniLM-L12-v2`, 384-dim, mean pooling + normalize, cosine brute force — no external API), and **hybrid** (RRF k=60, depth 50 over both arms). Two corpora: the 41-query cross-lingual fixture (51 short memories — the product-representative shape) and the pooled LongMemEval-S 22k.
|
|
115
|
+
|
|
116
|
+
**Fixture results (memory-like docs).**
|
|
117
|
+
|
|
118
|
+
| Arm | R@5 | MRR |
|
|
119
|
+
|---|---|---|
|
|
120
|
+
| BM25-only | 80.5% | 0.752 |
|
|
121
|
+
| vec-only | **92.7%** | **0.881** |
|
|
122
|
+
| hybrid RRF | 87.8% | 0.853 |
|
|
123
|
+
|
|
124
|
+
The vector arm rescues 3 of the 5 cross-lingual vocabulary misses — e.g. `production release thursday rule` → mem-021 at **vector rank 1** while BM25 misses it entirely. Two findings: (1) naive RRF **hurts** (87.8% < 92.7%): RRF rewards consensus docs, so BM25 noise demotes vector-#1 hits out of the top-10 — fusion needs design, not default; (2) `how do we ship` → mem-001 is missed by **both** arms, while the curated synonym map caught it (§4.6) — embeddings are fuzzy, curated maps are precise; complements, not substitutes.
|
|
125
|
+
|
|
126
|
+
**Pooled results (22k sessions).**
|
|
127
|
+
|
|
128
|
+
| Arm | R@5 | MRR |
|
|
129
|
+
|---|---|---|
|
|
130
|
+
| BM25-only | 38.5% | 0.277 |
|
|
131
|
+
| vec-only | 3.4% | 0.021 |
|
|
132
|
+
| hybrid RRF | 27.4% | 0.187 |
|
|
133
|
+
|
|
134
|
+
BM25 reproduces the §4.2 lexical ceiling (38.5% vs 40.6%, within implementation variance of the simplified bench arm). The vector arm collapses — and the collapse is measured, not speculated: pooled "documents" are full chat sessions (**median 10,506 chars; 94.9% exceed the 2000-char truncation; questions median 72 chars**), and both arms saw *identical* truncated text (seeding wrote truncated files; FTS indexed those), so truncation does not explain the arm gap. The explanation is mean-pooling dilution: one vector for a 2000-char multi-topic session cannot match a 72-char question about a single factoid, while BM25 still hits exact rare terms when present. Not a bug — the same code scores 92.7% on the fixture, and 3.4% ≫ random (0.02%), so weak signal is present, just diluted. This is the known bi-encoder weakness on long documents; production systems chunk.
|
|
135
|
+
|
|
136
|
+
**Gate verdict: PASS on memory-like docs, methodology artifact on pooled.** The pooled corpus does not resemble open-memex memories (short, focused) — it tested single-vector-per-long-doc, which nobody ships. The fixture is the product-representative test, and there the multilingual vector beats BM25 by **+12pp R@5**, specifically on the cross-lingual slice. A chunked rerun was deliberately skipped (4× compute on a non-representative corpus). The 40.6% lexical ceiling stands as the BM25 bar; the semantic layer's value is proven where the product lives.
|
|
137
|
+
|
|
138
|
+
## 5. Discussion
|
|
139
|
+
|
|
140
|
+
**OR semantics does the work; everything else is trim.** Of +0.66 total explainable MRR, OR contributes +0.652 and the remaining factors ±0.01. The lexical lemon is squeezed — further gains will not come from query-string tweaks.
|
|
141
|
+
|
|
142
|
+
**The synonym round needs no redesign — it needs a measurable problem.** Fallback and RRF fusion both rescue exactly zero on public data; the round's value lives in a vocabulary-mismatch slice that LongMemEval-S barely contains (proven real on constructed queries: 2 of 3 rescued). Keep the curated map — cheap, harmless, correct on its home turf — and stop spending design budget here.
|
|
143
|
+
|
|
144
|
+
**Scale is the real problem.** 97% → 41% R@5 under same-distribution distractors. The embedding gate (§4.7) passes on memory-like docs: multilingual vectors beat BM25 by +12pp R@5 on the cross-lingual fixture slice. Naive RRF fusion, however, hurts (it demotes vector-#1 hits via consensus) — fusion needs design, not default.
|
|
145
|
+
|
|
146
|
+
**Prefix matching is a candidate for removal** (−0.010 MRR, +1.9pp R@1 when disabled) but the effect is small; it needs confirmation on a second dataset before acting.
|
|
147
|
+
|
|
148
|
+
## 6. Threats to validity
|
|
149
|
+
|
|
150
|
+
- Single dataset, synthetic English chit-chat; generalization to real personal memories is unproven.
|
|
151
|
+
- Corpus construction (full sessions, both roles) may differ from agentmemory's; treat the comparison as indicative.
|
|
152
|
+
- OFAT cannot see factor interactions (only synonym×limit was checked).
|
|
153
|
+
- The pooled stress test is adversarial — 22k same-distribution distractors are harsher than a real diverse memory store.
|
|
154
|
+
- The 41-query synthetic fixture (recall@1 0.73) and this bench measure different things; both are reported, neither is "the" number.
|
|
155
|
+
- §4.7's pooled vector arm used single-vector-per-document with no chunking — a known-bad setup for 10k-char sessions, deliberately not re-run chunked (non-representative corpus); the fixture arm is the product-representative result.
|
|
156
|
+
|
|
157
|
+
## 7. Conclusion
|
|
158
|
+
|
|
159
|
+
Measure first, then change: the pipeline's BM25 core is strong (level with published hybrid systems), its dominant factor is OR semantics (+0.652 MRR; everything else ±0.01), its limit behavior is understood (R@1 invariant, flattening at 5), and its synonym round — under either design, at either scale — contributes nothing measurable on public data, because the data barely contains the vocabulary-mismatch slice it exists for. The embedding gate passes where the product lives: multilingual vectors beat BM25 by +12pp R@5 on memory-like docs, rescuing the cross-lingual misses BM25 cannot see; the pooled vector collapse is a single-vector-on-long-docs artifact, not a verdict. What remains is engineering, not research: index-time synonym/translation expansion (§18 backlog), a designed fusion strategy (naive RRF hurts), and a configurable embedding model.
|
|
160
|
+
|
|
161
|
+
## References
|
|
162
|
+
|
|
163
|
+
- Wu et al., *LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory*, ICLR 2025. Data: `xiaowu0162/longmemeval-cleaned` (MIT).
|
|
164
|
+
- Hu et al., *MemoryAgentBench*, arXiv:2507.05257.
|
|
165
|
+
- agentmemory, `benchmark/LONGMEMEVAL.md` (published BM25-only / BM25+vector recall_any@k).
|
|
166
|
+
- RRF: Cormack et al., SIGIR 2009.
|
|
167
|
+
|
|
168
|
+
## Appendix: reproduction
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
# per-question baseline (limit 10)
|
|
172
|
+
node --experimental-strip-types scripts/bench-longmemeval.ts \
|
|
173
|
+
--data <longmemeval_s_cleaned.json> --variant baseline --search-limit 10
|
|
174
|
+
# OFAT: --variant no-stopwords | and | no-prefix
|
|
175
|
+
# (no-synonyms retired by D80 — expansion is index-time now; its 2026-10-07
|
|
176
|
+
# measurement, effect 0.000, stands in the paper)
|
|
177
|
+
# limit sweep: --search-limit 1|3|5|10|20|50
|
|
178
|
+
# pooled stress: add --pooled
|
|
179
|
+
# rescue experiment: --rescue --search-limit <K> --out-rows <tsv> [--pooled]
|
|
180
|
+
# semantic experiment (§4.7): local multilingual embeddings, no API key needed
|
|
181
|
+
# first run downloads the model (~470MB) to $HF_HUB_CACHE; /tmp is only
|
|
182
|
+
# 512MB on small VMs — set TMPDIR to a roomy dir before running
|
|
183
|
+
TMPDIR=~/bench-tmp HF_HUB_CACHE=~/bench-hf-cache \
|
|
184
|
+
node --experimental-strip-types scripts/bench-semantic.ts --fixture
|
|
185
|
+
TMPDIR=~/bench-tmp HF_HUB_CACHE=~/bench-hf-cache \
|
|
186
|
+
node --experimental-strip-types scripts/bench-semantic.ts \
|
|
187
|
+
--pooled --data <longmemeval_s_cleaned.json>
|
|
188
|
+
# --model <hf-id> overrides the default
|
|
189
|
+
# Xenova/paraphrase-multilingual-MiniLM-L12-v2 (doc embeddings cache to
|
|
190
|
+
# <cwd>/emb-<mode>.json and are reused across runs)
|
|
191
|
+
```
|