open-memex 0.7.3 → 0.8.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -3
- package/CHANGELOG.md +14 -0
- package/README.md +61 -5
- package/README.zh-CN.md +14 -1
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +18 -2
- package/dist/init.js +5 -3
- package/dist/retrieve/alias-memory.js +130 -0
- package/dist/retrieve/query.js +20 -9
- package/dist/retrieve/search.js +51 -14
- package/dist/retrieve/synonyms.js +183 -0
- package/dist/store/db.js +3 -1
- package/dist/store/lifecycle.js +3 -0
- package/dist/store/markdown.js +7 -0
- package/dist/store/sync.js +15 -4
- package/dist/tools/ops.js +37 -0
- package/docs/V2-DESIGN.md +9 -2
- package/docs/release-runbook.md +8 -2
- package/docs/retrieval-ablation-study.md +191 -0
- package/package.json +5 -3
- package/scripts/bench-longmemeval.ts +342 -0
- package/scripts/bench-semantic.ts +198 -0
- package/scripts/retrieval-eval/fixture.ts +182 -0
- package/scripts/retrieval-eval.ts +83 -0
- package/scripts/smoke-pure.ts +82 -0
- package/src/cli.ts +18 -2
- package/src/init.ts +5 -3
- package/src/retrieve/alias-memory.ts +154 -0
- package/src/retrieve/query.ts +23 -9
- package/src/retrieve/search.ts +67 -15
- package/src/retrieve/synonyms.ts +184 -0
- package/src/store/db.ts +3 -1
- package/src/store/lifecycle.ts +6 -0
- package/src/store/markdown.ts +18 -0
- package/src/store/sync.ts +18 -3
- package/src/tools/ops.ts +49 -0
package/src/retrieve/query.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import { cjkQueryExpr, hasCjk } from "./cjk.ts";
|
|
15
15
|
|
|
16
16
|
/** English function words that carry no retrieval signal in questions. */
|
|
17
|
-
const LATIN_STOPWORDS = new Set([
|
|
17
|
+
export const LATIN_STOPWORDS = new Set([
|
|
18
18
|
"the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",
|
|
19
19
|
"without", "is", "are", "was", "were", "be", "been", "do", "does", "did",
|
|
20
20
|
"how", "what", "why", "when", "where", "which", "who", "whom", "we", "i",
|
|
@@ -32,9 +32,11 @@ const CJK_QUESTION_BIGRAMS = new Set([
|
|
|
32
32
|
|
|
33
33
|
const MAX_TERMS = 24;
|
|
34
34
|
|
|
35
|
-
function latinTerms(q: string): string[] {
|
|
35
|
+
export function latinTerms(q: string, useStopwords = true): string[] {
|
|
36
36
|
const raw = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
|
|
37
|
-
const kept =
|
|
37
|
+
const kept = useStopwords
|
|
38
|
+
? raw.filter((t) => t.length > 1 && !LATIN_STOPWORDS.has(t))
|
|
39
|
+
: raw.filter((t) => t.length > 1);
|
|
38
40
|
const chosen = kept.length > 0 ? kept : raw;
|
|
39
41
|
return [...new Set(chosen)].slice(0, MAX_TERMS);
|
|
40
42
|
}
|
|
@@ -44,15 +46,27 @@ function latinTerms(q: string): string[] {
|
|
|
44
46
|
* Latin tokens keep the old behavior (prefix match on content/tags/type).
|
|
45
47
|
* CJK runs become an OR of bigrams against the `cjk` column (see cjk.ts).
|
|
46
48
|
* Mixed queries OR the two parts together.
|
|
49
|
+
*
|
|
50
|
+
* Flags are ablation toggles (all default to the shipped behavior):
|
|
51
|
+
* - stopwords: false → skip function-word filtering
|
|
52
|
+
* - orJoin: false → AND the terms instead of OR
|
|
53
|
+
* - prefix: false → exact term match instead of `term*`
|
|
47
54
|
*/
|
|
48
|
-
export function toFtsQuery(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
export function toFtsQuery(
|
|
56
|
+
q: string,
|
|
57
|
+
flags: { stopwords?: boolean; orJoin?: boolean; prefix?: boolean } = {},
|
|
58
|
+
): string {
|
|
59
|
+
const useStopwords = flags.stopwords !== false;
|
|
60
|
+
const useOr = flags.orJoin !== false;
|
|
61
|
+
const usePrefix = flags.prefix !== false;
|
|
62
|
+
const joiner = useOr ? " OR " : " AND ";
|
|
63
|
+
const latin = latinTerms(q, useStopwords)
|
|
64
|
+
.map((t) => (usePrefix ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`))
|
|
65
|
+
.join(joiner);
|
|
52
66
|
const cjk = hasCjk(q)
|
|
53
|
-
? cjkQueryExpr(q, { dropBigrams: CJK_QUESTION_BIGRAMS, maxTerms: MAX_TERMS })
|
|
67
|
+
? cjkQueryExpr(q, { dropBigrams: useStopwords ? CJK_QUESTION_BIGRAMS : undefined, maxTerms: MAX_TERMS })
|
|
54
68
|
: "";
|
|
55
|
-
if (latin && cjk) return `(${latin}) OR {cjk}:(${cjk})`;
|
|
69
|
+
if (latin && cjk) return `(${latin}) ${useOr ? "OR" : "AND"} {cjk}:(${cjk})`;
|
|
56
70
|
if (cjk) return `{cjk}:(${cjk})`;
|
|
57
71
|
return latin;
|
|
58
72
|
}
|
package/src/retrieve/search.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { db } from "../store/db.ts";
|
|
2
2
|
import { toFtsQuery } from "./query.ts";
|
|
3
|
+
import {
|
|
4
|
+
loadAliasEntries,
|
|
5
|
+
planAliasExpansions,
|
|
6
|
+
mergeRounds,
|
|
7
|
+
} from "./alias-memory.ts";
|
|
3
8
|
|
|
4
9
|
export interface SearchHit {
|
|
5
10
|
id: string;
|
|
@@ -88,6 +93,8 @@ export interface SearchStats {
|
|
|
88
93
|
candidates: number;
|
|
89
94
|
hiddenSuperseded: number;
|
|
90
95
|
hiddenExcluded: number;
|
|
96
|
+
/** D81: query-time alias expansions actually run (rounds 2+, raw text). */
|
|
97
|
+
aliasExpansions: string[];
|
|
91
98
|
}
|
|
92
99
|
|
|
93
100
|
/**
|
|
@@ -158,14 +165,25 @@ function resolveVisible(rows: RawRow[], limit: number, stats?: SearchStats): Sea
|
|
|
158
165
|
*/
|
|
159
166
|
export function search(
|
|
160
167
|
query: string,
|
|
161
|
-
opts: {
|
|
168
|
+
opts: {
|
|
169
|
+
scopeKeys?: string[];
|
|
170
|
+
limit?: number;
|
|
171
|
+
type?: string;
|
|
172
|
+
stats?: SearchStats;
|
|
173
|
+
/** Ablation toggles for query construction (default = shipped behavior). */
|
|
174
|
+
stopwords?: boolean;
|
|
175
|
+
orJoin?: boolean;
|
|
176
|
+
prefix?: boolean;
|
|
177
|
+
} = {},
|
|
162
178
|
): SearchHit[] {
|
|
163
|
-
const
|
|
179
|
+
const qflags = { stopwords: opts.stopwords, orJoin: opts.orJoin, prefix: opts.prefix };
|
|
180
|
+
const q = toFtsQuery(query, qflags);
|
|
164
181
|
if (opts.stats) {
|
|
165
182
|
opts.stats.ftsQuery = q;
|
|
166
183
|
opts.stats.candidates = 0;
|
|
167
184
|
opts.stats.hiddenSuperseded = 0;
|
|
168
185
|
opts.stats.hiddenExcluded = 0;
|
|
186
|
+
opts.stats.aliasExpansions = [];
|
|
169
187
|
}
|
|
170
188
|
if (!q) return [];
|
|
171
189
|
const limit = Math.max(1, Math.min(opts.limit ?? 8, 50));
|
|
@@ -190,14 +208,7 @@ export function search(
|
|
|
190
208
|
ORDER BY score ASC
|
|
191
209
|
LIMIT ?
|
|
192
210
|
`;
|
|
193
|
-
|
|
194
|
-
if (opts.scopeKeys && opts.scopeKeys.length > 0) params.push(...opts.scopeKeys);
|
|
195
|
-
if (opts.type) params.push(opts.type);
|
|
196
|
-
params.push(fetchLimit);
|
|
197
|
-
|
|
198
|
-
const rows = db()
|
|
199
|
-
.prepare(sql)
|
|
200
|
-
.all(...(params as any[])) as Array<{
|
|
211
|
+
type Row = {
|
|
201
212
|
id: string;
|
|
202
213
|
scope_key: string;
|
|
203
214
|
project_name: string;
|
|
@@ -211,12 +222,53 @@ export function search(
|
|
|
211
222
|
created_at: number;
|
|
212
223
|
snippet: string;
|
|
213
224
|
score: number;
|
|
214
|
-
}
|
|
225
|
+
};
|
|
226
|
+
const fetchFts = (matchExpr: string): RawRow[] => {
|
|
227
|
+
const params: unknown[] = [matchExpr];
|
|
228
|
+
if (opts.scopeKeys && opts.scopeKeys.length > 0) params.push(...opts.scopeKeys);
|
|
229
|
+
if (opts.type) params.push(opts.type);
|
|
230
|
+
params.push(fetchLimit);
|
|
231
|
+
const rows = db()
|
|
232
|
+
.prepare(sql)
|
|
233
|
+
.all(...(params as any[])) as Row[];
|
|
234
|
+
// FTS5 bm25: lower = better; invert for intuition.
|
|
235
|
+
return rows.map((r) => ({ ...r, score: -r.score }));
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
const raw: RawRow[] = fetchFts(q);
|
|
239
|
+
|
|
240
|
+
// D81: user-defined alias vocabulary — query-time expansion rounds.
|
|
241
|
+
// Round 1 is the query as-is; each further round rewrites alias terms to
|
|
242
|
+
// the other side of the relation (bidirectional, depth < 3). The trigger
|
|
243
|
+
// is an exact registry hit (loadAliasEntries), not a count heuristic.
|
|
244
|
+
const aliasEntries = loadAliasEntries(opts.scopeKeys);
|
|
245
|
+
const expansions = planAliasExpansions(query, aliasEntries);
|
|
246
|
+
let merged: RawRow[] = raw;
|
|
247
|
+
let finalLimit = limit;
|
|
248
|
+
if (expansions.length > 0) {
|
|
249
|
+
const rounds: RawRow[][] = [raw];
|
|
250
|
+
for (const eq of expansions) {
|
|
251
|
+
const mq = toFtsQuery(eq, qflags);
|
|
252
|
+
if (mq) rounds.push(fetchFts(mq));
|
|
253
|
+
}
|
|
254
|
+
// Normalized-max merge (alias-memory.ts): each round's scores are
|
|
255
|
+
// divided by that round's best, so an alias-side champion competes
|
|
256
|
+
// evenly with a literal-side champion — no down-weighting for being
|
|
257
|
+
// from a later round.
|
|
258
|
+
merged = mergeRounds(rounds).map((m) => ({ ...m.doc, score: m.score }));
|
|
259
|
+
// Cut to [1.5K, 2K] — never back to K, or round-2 hits would be
|
|
260
|
+
// starved the way D79's slice-to-K starved them.
|
|
261
|
+
const pool = merged.length;
|
|
262
|
+
finalLimit = Math.min(pool, 2 * limit);
|
|
263
|
+
finalLimit = Math.max(finalLimit, Math.min(pool, Math.ceil(1.5 * limit)));
|
|
264
|
+
if (opts.stats) opts.stats.aliasExpansions = expansions;
|
|
265
|
+
}
|
|
215
266
|
|
|
216
|
-
//
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
267
|
+
// D80: the curated static map is solved at index time (synonyms.ts
|
|
268
|
+
// expands variants into the aliases column at sync time), so without a
|
|
269
|
+
// user alias the query side stays a single FTS round.
|
|
270
|
+
if (opts.stats) opts.stats.candidates = merged.length;
|
|
271
|
+
return resolveVisible(merged, finalLimit, opts.stats);
|
|
220
272
|
}
|
|
221
273
|
|
|
222
274
|
export function list(
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index-time synonym/translation expansion (D80).
|
|
3
|
+
*
|
|
4
|
+
* Stone's rule: solve vocabulary mismatch at write time, so any related
|
|
5
|
+
* word finds the answer with a plain round-1 query. At sync/index time
|
|
6
|
+
* every memory's keywords are expanded through a curated, checked-in
|
|
7
|
+
* equivalence map and the variants are written into the indexed `aliases`
|
|
8
|
+
* column (FTS + CJK bigram columns both pick them up via sync.ts). The
|
|
9
|
+
* query side stays simple: one FTS round, no trigger, no second chance.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately NOT an embedding model or an LLM call: deterministic,
|
|
12
|
+
* auditable, zero dependencies, and the failure mode (a bad synonym)
|
|
13
|
+
* is a visible line in this file, not a silent vector.
|
|
14
|
+
*
|
|
15
|
+
* Groups are bidirectional — any member expands to the other members.
|
|
16
|
+
* Keep groups tight: an over-broad synonym costs precision on every
|
|
17
|
+
* query, and at index time a bad entry pollutes every document instead
|
|
18
|
+
* of one query. When in doubt, leave it out. A map update only reaches
|
|
19
|
+
* already-indexed memories after a reindex.
|
|
20
|
+
*
|
|
21
|
+
* Design note (D80): the map covers the *precise* slice — EN↔ZH
|
|
22
|
+
* translation pairs and dev-domain equivalents where embeddings are
|
|
23
|
+
* fuzzy (ship/deploy are neighbors in vector space but "单点登录"/SSO
|
|
24
|
+
* is an exact pair no embedding guarantees). Embeddings cover the
|
|
25
|
+
* semantic long tail; this map covers what must never be missed.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { LATIN_STOPWORDS } from "./query.ts";
|
|
29
|
+
|
|
30
|
+
type Group = string[];
|
|
31
|
+
|
|
32
|
+
const CJK_RE = /[\u4e00-\u9fff]/;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Curated equivalence groups. EN dev-domain vocabulary, ZH equivalents,
|
|
36
|
+
* and EN↔ZH translation pairs live in one bidirectional group: any member
|
|
37
|
+
* expands to all the others, in either direction.
|
|
38
|
+
*/
|
|
39
|
+
const GROUPS: Group[] = [
|
|
40
|
+
["deploy", "ship", "release", "rollout", "publish", "部署", "发布", "上线"],
|
|
41
|
+
["test", "testing", "tests", "测试", "单测", "集成测试"],
|
|
42
|
+
["regression test", "回归测试"],
|
|
43
|
+
["rollback", "revert", "回滚", "撤回", "撤销", "回退"],
|
|
44
|
+
["incident", "outage", "postmortem", "故障", "事故", "宕机"],
|
|
45
|
+
["auth", "authentication", "login", "sso", "single sign-on", "单点登录", "登录"],
|
|
46
|
+
["secret", "secrets", "credential", "credentials", "token", "密钥"],
|
|
47
|
+
["database", "db", "数据库"],
|
|
48
|
+
["backup", "restore", "备份", "恢复"],
|
|
49
|
+
["retention", "保留"],
|
|
50
|
+
["production", "生产", "生产环境"],
|
|
51
|
+
["staging", "预发布环境", "预发布"],
|
|
52
|
+
["wipe", "wipes", "清空", "清除"],
|
|
53
|
+
["nightly", "night", "每晚"],
|
|
54
|
+
["thursday", "周四", "星期四"],
|
|
55
|
+
["mandatory", "强制", "必须"],
|
|
56
|
+
["flaky", "flake", "unreliable", "intermittent"],
|
|
57
|
+
["bug", "defect", "issue"],
|
|
58
|
+
["ci", "pipeline"],
|
|
59
|
+
["cache", "caching", "cached"],
|
|
60
|
+
["invalidate", "invalidation", "expire", "expiry"],
|
|
61
|
+
["oncall", "on-call", "pager"],
|
|
62
|
+
["migration", "migrate", "migrating"],
|
|
63
|
+
["index", "indexing", "indexed"],
|
|
64
|
+
["slow", "latency", "performance", "perf"],
|
|
65
|
+
["refactor", "refactoring"],
|
|
66
|
+
["merge", "merging"],
|
|
67
|
+
["branch", "branches"],
|
|
68
|
+
["commit", "commits"],
|
|
69
|
+
["review", "code review", "评审", "代码评审"],
|
|
70
|
+
["mock", "stub"],
|
|
71
|
+
["debug", "debugging", "troubleshoot", "troubleshooting"],
|
|
72
|
+
["log", "logs", "logging"],
|
|
73
|
+
["monitor", "monitoring", "alert", "alerting"],
|
|
74
|
+
["scale", "scaling"],
|
|
75
|
+
["timeout", "timeouts"],
|
|
76
|
+
["retry", "retries"],
|
|
77
|
+
["queue", "job", "worker"],
|
|
78
|
+
["api", "endpoint"],
|
|
79
|
+
["docker", "container", "image"],
|
|
80
|
+
["kubernetes", "k8s"],
|
|
81
|
+
["env", "environment"],
|
|
82
|
+
["config", "configuration", "setting", "settings"],
|
|
83
|
+
["hotfix", "hot fix"],
|
|
84
|
+
["estimate", "estimation"],
|
|
85
|
+
["乱码", "编码问题"],
|
|
86
|
+
["并发", "并行"],
|
|
87
|
+
["重构"],
|
|
88
|
+
["规范", "约定"],
|
|
89
|
+
];
|
|
90
|
+
|
|
91
|
+
const termLookup = new Map<string, string[]>();
|
|
92
|
+
const phraseEntries: Array<{ phrase: string; siblings: string[] }> = [];
|
|
93
|
+
const cjkEntries: Array<{ keys: string[]; siblings: string[] }> = [];
|
|
94
|
+
|
|
95
|
+
for (const g of GROUPS) {
|
|
96
|
+
const cjkKeys = g.filter((m) => CJK_RE.test(m));
|
|
97
|
+
if (cjkKeys.length > 0) cjkEntries.push({ keys: cjkKeys, siblings: [...g] });
|
|
98
|
+
for (const m of g) {
|
|
99
|
+
if (CJK_RE.test(m)) continue;
|
|
100
|
+
const siblings = g.filter((x) => x !== m);
|
|
101
|
+
if (m.includes(" ")) phraseEntries.push({ phrase: m.toLowerCase(), siblings });
|
|
102
|
+
else termLookup.set(m.toLowerCase(), siblings);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Synonym variants for one lowercased Latin term (excluding the term
|
|
108
|
+
* itself). Empty array when the term has no curated synonyms.
|
|
109
|
+
*/
|
|
110
|
+
export function synonymVariants(term: string): string[] {
|
|
111
|
+
return termLookup.get(term.toLowerCase()) ?? [];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Cap on auto-generated variants per memory — bounds index growth. */
|
|
115
|
+
export const MAX_AUTO_VARIANTS = 32;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Index-time expansion for one memory's text (body + tags). Keyword-
|
|
119
|
+
* centric (Stone): extract content terms, expand each through the
|
|
120
|
+
* curated map, return variants not already present in the text.
|
|
121
|
+
* Pure logic — safe to unit test, runs once per memory at sync time.
|
|
122
|
+
*/
|
|
123
|
+
export function expansionVariantsForDoc(text: string): string[] {
|
|
124
|
+
const lower = text.toLowerCase();
|
|
125
|
+
// Content terms only: function words carry no signal (same filter the
|
|
126
|
+
// query builder applies, so both sides agree on what a keyword is).
|
|
127
|
+
const latinTokens = new Set(
|
|
128
|
+
(lower.match(/[a-z0-9_.\-]+/g) ?? []).filter(
|
|
129
|
+
(t) => t.length > 1 && !LATIN_STOPWORDS.has(t),
|
|
130
|
+
),
|
|
131
|
+
);
|
|
132
|
+
const out: string[] = [];
|
|
133
|
+
const seen = new Set<string>();
|
|
134
|
+
const push = (v: string) => {
|
|
135
|
+
const key = v.toLowerCase();
|
|
136
|
+
if (seen.has(key) || out.length >= MAX_AUTO_VARIANTS) return;
|
|
137
|
+
seen.add(key);
|
|
138
|
+
out.push(v);
|
|
139
|
+
};
|
|
140
|
+
// A variant is "already present" when the doc has it as a token (Latin),
|
|
141
|
+
// as a substring (Latin phrase), or as a substring (CJK — no tokenizer).
|
|
142
|
+
const present = (v: string): boolean =>
|
|
143
|
+
CJK_RE.test(v)
|
|
144
|
+
? text.includes(v)
|
|
145
|
+
: v.includes(" ")
|
|
146
|
+
? lower.includes(v.toLowerCase())
|
|
147
|
+
: latinTokens.has(v.toLowerCase());
|
|
148
|
+
|
|
149
|
+
for (const t of latinTokens) {
|
|
150
|
+
for (const v of termLookup.get(t) ?? []) if (!present(v)) push(v);
|
|
151
|
+
}
|
|
152
|
+
for (const { phrase, siblings } of phraseEntries) {
|
|
153
|
+
if (!lower.includes(phrase)) continue;
|
|
154
|
+
for (const v of siblings) if (!present(v)) push(v);
|
|
155
|
+
}
|
|
156
|
+
for (const { keys, siblings } of cjkEntries) {
|
|
157
|
+
if (!keys.some((k) => text.includes(k))) continue;
|
|
158
|
+
for (const v of siblings) if (!present(v)) push(v);
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Expand a raw query string with synonym variants (legacy query-time
|
|
165
|
+
* helper, kept for bench scripts). The product no longer expands at
|
|
166
|
+
* query time — D80 moved expansion to the index.
|
|
167
|
+
*/
|
|
168
|
+
export function expandQueryWithSynonyms(q: string): string {
|
|
169
|
+
const extra = new Set<string>();
|
|
170
|
+
const latinTerms = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
|
|
171
|
+
for (const t of latinTerms) {
|
|
172
|
+
for (const s of synonymVariants(t)) extra.add(s);
|
|
173
|
+
}
|
|
174
|
+
for (const { keys, siblings } of cjkEntries) {
|
|
175
|
+
if (keys.some((k) => q.includes(k))) {
|
|
176
|
+
for (const s of siblings) extra.add(s);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
// Don't re-add terms already in the query.
|
|
180
|
+
const have = new Set(latinTerms);
|
|
181
|
+
const additions = [...extra].filter((s) => !have.has(s.toLowerCase()) && !q.includes(s));
|
|
182
|
+
if (additions.length === 0) return q;
|
|
183
|
+
return `${q} ${additions.join(" ")}`;
|
|
184
|
+
}
|
package/src/store/db.ts
CHANGED
|
@@ -48,6 +48,8 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
48
48
|
status TEXT NOT NULL DEFAULT 'active',
|
|
49
49
|
tags TEXT NOT NULL DEFAULT '',
|
|
50
50
|
aliases TEXT NOT NULL DEFAULT '',
|
|
51
|
+
alias TEXT NOT NULL DEFAULT '',
|
|
52
|
+
target TEXT NOT NULL DEFAULT '',
|
|
51
53
|
content TEXT NOT NULL,
|
|
52
54
|
cjk TEXT NOT NULL DEFAULT '',
|
|
53
55
|
content_hash TEXT NOT NULL DEFAULT '',
|
|
@@ -103,7 +105,7 @@ END;
|
|
|
103
105
|
`;
|
|
104
106
|
|
|
105
107
|
/** Current index schema version. Bump when TABLE_SCHEMA/FTS_SCHEMA change. */
|
|
106
|
-
const SCHEMA_VERSION =
|
|
108
|
+
const SCHEMA_VERSION = 8;
|
|
107
109
|
|
|
108
110
|
function userVersion(d: AnyDatabase): number {
|
|
109
111
|
const row = d.prepare("PRAGMA user_version").get() as { user_version: number };
|
package/src/store/lifecycle.ts
CHANGED
|
@@ -260,6 +260,9 @@ export interface SupersedeInput {
|
|
|
260
260
|
type?: string;
|
|
261
261
|
tags?: string[];
|
|
262
262
|
aliases?: string[];
|
|
263
|
+
/** D81: undefined = inherit the old memory's pair. */
|
|
264
|
+
alias?: string;
|
|
265
|
+
target?: string;
|
|
263
266
|
source?: string;
|
|
264
267
|
}
|
|
265
268
|
|
|
@@ -288,6 +291,9 @@ export function supersede(
|
|
|
288
291
|
tags: input.tags ?? oldMf.fm.tags,
|
|
289
292
|
// D61: aliases carry forward unless the caller replaces them.
|
|
290
293
|
aliases: input.aliases ?? oldMf.fm.aliases,
|
|
294
|
+
// D81: alias vocabulary carries forward the same way.
|
|
295
|
+
alias: input.alias ?? oldMf.fm.alias,
|
|
296
|
+
target: input.target ?? oldMf.fm.target,
|
|
291
297
|
source: input.source ?? oldMf.fm.source,
|
|
292
298
|
status: "active",
|
|
293
299
|
created_at: now,
|
package/src/store/markdown.ts
CHANGED
|
@@ -60,6 +60,17 @@ export interface Frontmatter {
|
|
|
60
60
|
* Optional and additive: files without it parse unchanged.
|
|
61
61
|
*/
|
|
62
62
|
aliases?: string[];
|
|
63
|
+
/**
|
|
64
|
+
* D81: user-defined alias vocabulary — `alias` is the nickname
|
|
65
|
+
* (e.g. 香蕉计划), `target` is what it refers to
|
|
66
|
+
* (e.g. 支付系统重构项目). Unlike D61 per-memory aliases (alternate
|
|
67
|
+
* phrasings of one memory), an alias memory defines vocabulary once and
|
|
68
|
+
* the query-time expansion (retrieve/alias-memory.ts) applies it to every
|
|
69
|
+
* search in its scope — no reindex when the alias changes. Optional;
|
|
70
|
+
* files without it parse unchanged.
|
|
71
|
+
*/
|
|
72
|
+
alias?: string;
|
|
73
|
+
target?: string;
|
|
63
74
|
source: string;
|
|
64
75
|
created_at: string; // RFC 3339, never bare epoch (§3)
|
|
65
76
|
updated_at: string; // RFC 3339
|
|
@@ -247,6 +258,13 @@ export function normalizeFrontmatter(
|
|
|
247
258
|
return aliases.length > 0 ? { aliases } : {};
|
|
248
259
|
})()
|
|
249
260
|
: {}),
|
|
261
|
+
// D81: alias vocabulary — single trimmed strings, dropped when empty.
|
|
262
|
+
...(typeof raw.alias === "string" && raw.alias.trim()
|
|
263
|
+
? { alias: raw.alias.trim() }
|
|
264
|
+
: {}),
|
|
265
|
+
...(typeof raw.target === "string" && raw.target.trim()
|
|
266
|
+
? { target: raw.target.trim() }
|
|
267
|
+
: {}),
|
|
250
268
|
source: typeof raw.source === "string" ? raw.source : "",
|
|
251
269
|
created_at: msToRfc3339(timeToMs(raw.created_at)),
|
|
252
270
|
updated_at: msToRfc3339(timeToMs(raw.updated_at)),
|
package/src/store/sync.ts
CHANGED
|
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { db } from "./db.ts";
|
|
4
4
|
import { cjkIndexText } from "../retrieve/cjk.ts";
|
|
5
|
+
import { expansionVariantsForDoc } from "../retrieve/synonyms.ts";
|
|
5
6
|
import { contentHash, repairChain } from "./lifecycle.ts";
|
|
6
7
|
import {
|
|
7
8
|
iterMemoryFiles,
|
|
@@ -14,8 +15,8 @@ import { projectRoot, paths } from "../paths.ts";
|
|
|
14
15
|
import { loadConfig } from "../config.ts";
|
|
15
16
|
|
|
16
17
|
const UPSERT_SQL = `
|
|
17
|
-
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
18
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
18
|
+
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, alias, target, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
19
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
19
20
|
ON CONFLICT(id) DO UPDATE SET
|
|
20
21
|
scope_key = excluded.scope_key,
|
|
21
22
|
scope = excluded.scope,
|
|
@@ -27,6 +28,8 @@ const UPSERT_SQL = `
|
|
|
27
28
|
status = excluded.status,
|
|
28
29
|
tags = excluded.tags,
|
|
29
30
|
aliases = excluded.aliases,
|
|
31
|
+
alias = excluded.alias,
|
|
32
|
+
target = excluded.target,
|
|
30
33
|
content = excluded.content,
|
|
31
34
|
cjk = excluded.cjk,
|
|
32
35
|
content_hash = excluded.content_hash,
|
|
@@ -63,7 +66,17 @@ export function upsertFromFile(mf: MemoryFile): void {
|
|
|
63
66
|
function writeRow(mf: MemoryFile, mtimeMs: number): void {
|
|
64
67
|
const { fm, body, filePath } = mf;
|
|
65
68
|
const tags = (fm.tags ?? []).join(",");
|
|
66
|
-
|
|
69
|
+
// D80 index-time expansion: curated synonym/translation variants for the
|
|
70
|
+
// memory's keywords are indexed in the aliases column (and therefore the
|
|
71
|
+
// CJK bigram column below), so any related word matches in round one.
|
|
72
|
+
// Derived at sync time — the Markdown file stays the source of truth.
|
|
73
|
+
const supplied = fm.aliases ?? [];
|
|
74
|
+
const have = new Set(supplied.map((a) => a.toLowerCase()));
|
|
75
|
+
const auto = expansionVariantsForDoc(body + "\n" + tags).filter(
|
|
76
|
+
(a) => !have.has(a.toLowerCase()),
|
|
77
|
+
);
|
|
78
|
+
const aliases = [...supplied, ...auto].join(",");
|
|
79
|
+
// D81: user-defined alias vocabulary (single nickname → referent pair).
|
|
67
80
|
db().prepare(UPSERT_SQL).run(
|
|
68
81
|
fm.id,
|
|
69
82
|
fm.scope_key,
|
|
@@ -76,6 +89,8 @@ function writeRow(mf: MemoryFile, mtimeMs: number): void {
|
|
|
76
89
|
fm.status,
|
|
77
90
|
tags,
|
|
78
91
|
aliases,
|
|
92
|
+
fm.alias ?? "",
|
|
93
|
+
fm.target ?? "",
|
|
79
94
|
body,
|
|
80
95
|
cjkIndexText(body + "\n" + tags + "\n" + aliases),
|
|
81
96
|
contentHash(body),
|
package/src/tools/ops.ts
CHANGED
|
@@ -113,6 +113,18 @@ export const memoryAddArgs = {
|
|
|
113
113
|
scope: scopeArg,
|
|
114
114
|
tags: z.array(z.string()).optional().describe("Optional tags for filtering."),
|
|
115
115
|
aliases: aliasesArg,
|
|
116
|
+
alias: z
|
|
117
|
+
.string()
|
|
118
|
+
.optional()
|
|
119
|
+
.describe(
|
|
120
|
+
"D81: nickname this memory defines (e.g. 香蕉计划). Pairs with `target`: the alias memory expands queries in its scope at search time, both directions, no reindex. Explicit vocabulary — always stored, not gated by capture aliases.",
|
|
121
|
+
),
|
|
122
|
+
target: z
|
|
123
|
+
.string()
|
|
124
|
+
.optional()
|
|
125
|
+
.describe(
|
|
126
|
+
"D81: what `alias` refers to (e.g. 支付系统重构项目). Only meaningful with `alias`.",
|
|
127
|
+
),
|
|
116
128
|
source: z
|
|
117
129
|
.string()
|
|
118
130
|
.optional()
|
|
@@ -163,6 +175,18 @@ export const memorySupersedeArgs = {
|
|
|
163
175
|
.describe("Category of the new memory. Defaults to the old memory's type."),
|
|
164
176
|
tags: z.array(z.string()).optional().describe("Optional tags for the new memory."),
|
|
165
177
|
aliases: aliasesArg,
|
|
178
|
+
alias: z
|
|
179
|
+
.string()
|
|
180
|
+
.optional()
|
|
181
|
+
.describe(
|
|
182
|
+
"D81: nickname the new memory defines (e.g. 香蕉计划). Omitted = inherit the old memory's alias.",
|
|
183
|
+
),
|
|
184
|
+
target: z
|
|
185
|
+
.string()
|
|
186
|
+
.optional()
|
|
187
|
+
.describe(
|
|
188
|
+
"D81: what `alias` refers to (e.g. 支付系统重构项目). Omitted = inherit.",
|
|
189
|
+
),
|
|
166
190
|
};
|
|
167
191
|
export type MemorySupersedeArgs = z.infer<z.ZodObject<typeof memorySupersedeArgs>>;
|
|
168
192
|
|
|
@@ -242,6 +266,8 @@ function buildFrontmatter(
|
|
|
242
266
|
type: Frontmatter["type"];
|
|
243
267
|
tags: string[];
|
|
244
268
|
aliases?: string[];
|
|
269
|
+
alias?: string;
|
|
270
|
+
target?: string;
|
|
245
271
|
source: Frontmatter["source"];
|
|
246
272
|
},
|
|
247
273
|
): Frontmatter {
|
|
@@ -259,6 +285,11 @@ function buildFrontmatter(
|
|
|
259
285
|
status: "active",
|
|
260
286
|
tags: body.tags,
|
|
261
287
|
...(body.aliases && body.aliases.length > 0 ? { aliases: body.aliases } : {}),
|
|
288
|
+
// D81: explicit alias vocabulary — single trimmed strings, dropped
|
|
289
|
+
// when empty. Not gated by captureAliases (that gates agent-invented
|
|
290
|
+
// phrasings; this is deliberate vocabulary).
|
|
291
|
+
...(body.alias?.trim() ? { alias: body.alias.trim() } : {}),
|
|
292
|
+
...(body.target?.trim() ? { target: body.target.trim() } : {}),
|
|
262
293
|
source: body.source,
|
|
263
294
|
created_at: rfc,
|
|
264
295
|
updated_at: rfc,
|
|
@@ -328,6 +359,14 @@ export async function addMemory(
|
|
|
328
359
|
(args.aliases ?? []).map((a) => redact(a, cfg.redactPatterns).content),
|
|
329
360
|
)
|
|
330
361
|
: [],
|
|
362
|
+
// D81: explicit alias vocabulary — same redaction as content, never
|
|
363
|
+
// gated by captureAliases.
|
|
364
|
+
alias: args.alias?.trim()
|
|
365
|
+
? redact(args.alias.trim(), cfg.redactPatterns).content
|
|
366
|
+
: undefined,
|
|
367
|
+
target: args.target?.trim()
|
|
368
|
+
? redact(args.target.trim(), cfg.redactPatterns).content
|
|
369
|
+
: undefined,
|
|
331
370
|
source: args.source ?? "tool",
|
|
332
371
|
});
|
|
333
372
|
const { filePath } = writeMemoryFile(fm, redacted);
|
|
@@ -447,6 +486,16 @@ export async function supersedeMemory(
|
|
|
447
486
|
args.aliases.map((a) => redact(a, cfg.redactPatterns).content),
|
|
448
487
|
)
|
|
449
488
|
: undefined,
|
|
489
|
+
// D81: explicit alias vocabulary — omitted inherits the old memory's
|
|
490
|
+
// pair; same redaction as content; never gated by captureAliases.
|
|
491
|
+
alias:
|
|
492
|
+
args.alias !== undefined
|
|
493
|
+
? redact(args.alias, cfg.redactPatterns).content || undefined
|
|
494
|
+
: undefined,
|
|
495
|
+
target:
|
|
496
|
+
args.target !== undefined
|
|
497
|
+
? redact(args.target, cfg.redactPatterns).content || undefined
|
|
498
|
+
: undefined,
|
|
450
499
|
source: "tool",
|
|
451
500
|
});
|
|
452
501
|
upsertFromFile(oldMf);
|