open-memex 0.8.0 → 0.8.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +12 -0
- package/README.md +21 -8
- package/README.zh-CN.md +9 -2
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +13 -0
- package/dist/plugin-core.js +1 -1
- package/dist/retrieve/search.js +53 -16
- package/dist/retrieve/weights.js +35 -0
- package/dist/store/db.js +42 -9
- package/dist/store/sync.js +13 -8
- package/dist/tools/ops.js +8 -2
- package/docs/V2-DESIGN.md +15 -1
- package/docs/writing-for-memory.md +94 -0
- package/package.json +1 -1
- package/scripts/smoke-mcp.ts +12 -0
- package/scripts/smoke-pure.ts +16 -0
- package/scripts/test-full.ts +49 -0
- package/server.json +2 -2
- package/src/cli.ts +15 -0
- package/src/plugin-core.ts +1 -1
- package/src/retrieve/search.ts +78 -17
- package/src/retrieve/weights.ts +37 -0
- package/src/store/db.ts +41 -9
- package/src/store/sync.ts +14 -8
- package/src/tools/ops.ts +12 -2
package/src/store/db.ts
CHANGED
|
@@ -48,6 +48,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
48
48
|
status TEXT NOT NULL DEFAULT 'active',
|
|
49
49
|
tags TEXT NOT NULL DEFAULT '',
|
|
50
50
|
aliases TEXT NOT NULL DEFAULT '',
|
|
51
|
+
expansions TEXT NOT NULL DEFAULT '',
|
|
51
52
|
alias TEXT NOT NULL DEFAULT '',
|
|
52
53
|
target TEXT NOT NULL DEFAULT '',
|
|
53
54
|
content TEXT NOT NULL,
|
|
@@ -66,6 +67,24 @@ CREATE INDEX IF NOT EXISTS idx_memories_scope_updated
|
|
|
66
67
|
ON memories(scope_key, updated_at DESC);
|
|
67
68
|
CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
|
|
68
69
|
CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
|
|
70
|
+
-- D81: the alias registry (retrieve/alias-memory.ts) runs on every search().
|
|
71
|
+
-- The predicate is a full table scan without this — the rows carrying a
|
|
72
|
+
-- defined alias pair are a tiny subset of a corpus that may be tens of
|
|
73
|
+
-- thousands. A partial index keeps the lookup proportional to the number of
|
|
74
|
+
-- aliases defined, not the size of the store.
|
|
75
|
+
--
|
|
76
|
+
-- No SCHEMA_VERSION bump: TABLE_SCHEMA runs on every db open (see db()), so
|
|
77
|
+
-- existing installs pick this up on next open. Deliberately NOT a result
|
|
78
|
+
-- cache — the alias registry must be re-read per query so a newly saved
|
|
79
|
+
-- alias memory applies on the very next query. This index accelerates the
|
|
80
|
+
-- lookup without caching results.
|
|
81
|
+
-- D87 correction to the "no bump" reasoning above: it holds only for stores
|
|
82
|
+
-- whose tables already have the alias/target columns. A pre-D81 store's
|
|
83
|
+
-- table lacks them, CREATE TABLE IF NOT EXISTS adds none, and creating this
|
|
84
|
+
-- index throws "no such column: alias" before the version check runs. db()
|
|
85
|
+
-- now rebuilds a stale table before creating the schema.
|
|
86
|
+
CREATE INDEX IF NOT EXISTS idx_memories_alias
|
|
87
|
+
ON memories(scope_key) WHERE alias <> '' AND target <> '';
|
|
69
88
|
`;
|
|
70
89
|
|
|
71
90
|
// The `cjk` column holds pre-tokenized CJK unigrams+bigrams (see
|
|
@@ -78,6 +97,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
|
78
97
|
content,
|
|
79
98
|
tags,
|
|
80
99
|
aliases,
|
|
100
|
+
expansions,
|
|
81
101
|
type,
|
|
82
102
|
cjk,
|
|
83
103
|
scope_key UNINDEXED,
|
|
@@ -87,25 +107,25 @@ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
|
87
107
|
);
|
|
88
108
|
|
|
89
109
|
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
90
|
-
INSERT INTO memories_fts(rowid, content, tags, aliases, type, cjk, scope_key)
|
|
91
|
-
VALUES (new.rowid, new.content, new.tags, new.aliases, new.type, new.cjk, new.scope_key);
|
|
110
|
+
INSERT INTO memories_fts(rowid, content, tags, aliases, expansions, type, cjk, scope_key)
|
|
111
|
+
VALUES (new.rowid, new.content, new.tags, new.aliases, new.expansions, new.type, new.cjk, new.scope_key);
|
|
92
112
|
END;
|
|
93
113
|
|
|
94
114
|
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
|
95
|
-
INSERT INTO memories_fts(memories_fts, rowid, content, tags, aliases, type, cjk, scope_key)
|
|
96
|
-
VALUES ('delete', old.rowid, old.content, old.tags, old.aliases, old.type, old.cjk, old.scope_key);
|
|
115
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tags, aliases, expansions, type, cjk, scope_key)
|
|
116
|
+
VALUES ('delete', old.rowid, old.content, old.tags, old.aliases, old.expansions, old.type, old.cjk, old.scope_key);
|
|
97
117
|
END;
|
|
98
118
|
|
|
99
119
|
CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
|
100
|
-
INSERT INTO memories_fts(memories_fts, rowid, content, tags, aliases, type, cjk, scope_key)
|
|
101
|
-
VALUES ('delete', old.rowid, old.content, old.tags, old.aliases, old.type, old.cjk, old.scope_key);
|
|
102
|
-
INSERT INTO memories_fts(rowid, content, tags, aliases, type, cjk, scope_key)
|
|
103
|
-
VALUES (new.rowid, new.content, new.tags, new.aliases, new.type, new.cjk, new.scope_key);
|
|
120
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tags, aliases, expansions, type, cjk, scope_key)
|
|
121
|
+
VALUES ('delete', old.rowid, old.content, old.tags, old.aliases, old.expansions, old.type, old.cjk, old.scope_key);
|
|
122
|
+
INSERT INTO memories_fts(rowid, content, tags, aliases, expansions, type, cjk, scope_key)
|
|
123
|
+
VALUES (new.rowid, new.content, new.tags, new.aliases, new.expansions, new.type, new.cjk, new.scope_key);
|
|
104
124
|
END;
|
|
105
125
|
`;
|
|
106
126
|
|
|
107
127
|
/** Current index schema version. Bump when TABLE_SCHEMA/FTS_SCHEMA change. */
|
|
108
|
-
const SCHEMA_VERSION =
|
|
128
|
+
const SCHEMA_VERSION = 9;
|
|
109
129
|
|
|
110
130
|
function userVersion(d: AnyDatabase): number {
|
|
111
131
|
const row = d.prepare("PRAGMA user_version").get() as { user_version: number };
|
|
@@ -147,6 +167,18 @@ export function db(): AnyDatabase {
|
|
|
147
167
|
d.exec("PRAGMA busy_timeout = 5000;");
|
|
148
168
|
d.exec("PRAGMA synchronous = NORMAL;");
|
|
149
169
|
d.exec("PRAGMA foreign_keys = ON;");
|
|
170
|
+
// D87: a stale on-disk table predates columns that TABLE_SCHEMA's partial
|
|
171
|
+
// index references (D85's idx_memories_alias needs alias/target, added by
|
|
172
|
+
// D81). CREATE TABLE IF NOT EXISTS adds no columns to an existing table,
|
|
173
|
+
// so creating that index first throws "no such column: alias" — and the
|
|
174
|
+
// version check below never runs, wedging every open on a pre-D81 store.
|
|
175
|
+
// When the table exists but the version is old, rebuild first: the table
|
|
176
|
+
// exists, so rebuild's DELETE has something to delete. Fresh stores have
|
|
177
|
+
// no table and fall through to creation below, exactly as before.
|
|
178
|
+
const hasTable = d
|
|
179
|
+
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memories'")
|
|
180
|
+
.get() as { name: string } | undefined;
|
|
181
|
+
if (hasTable && userVersion(d) < SCHEMA_VERSION) rebuildIndexSchema(d);
|
|
150
182
|
d.exec(TABLE_SCHEMA);
|
|
151
183
|
d.exec(FTS_SCHEMA);
|
|
152
184
|
if (userVersion(d) < SCHEMA_VERSION) rebuildIndexSchema(d);
|
package/src/store/sync.ts
CHANGED
|
@@ -15,8 +15,8 @@ import { projectRoot, paths } from "../paths.ts";
|
|
|
15
15
|
import { loadConfig } from "../config.ts";
|
|
16
16
|
|
|
17
17
|
const UPSERT_SQL = `
|
|
18
|
-
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, alias, target, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
19
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
18
|
+
INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, aliases, expansions, alias, target, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
|
|
19
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
20
20
|
ON CONFLICT(id) DO UPDATE SET
|
|
21
21
|
scope_key = excluded.scope_key,
|
|
22
22
|
scope = excluded.scope,
|
|
@@ -28,6 +28,7 @@ const UPSERT_SQL = `
|
|
|
28
28
|
status = excluded.status,
|
|
29
29
|
tags = excluded.tags,
|
|
30
30
|
aliases = excluded.aliases,
|
|
31
|
+
expansions = excluded.expansions,
|
|
31
32
|
alias = excluded.alias,
|
|
32
33
|
target = excluded.target,
|
|
33
34
|
content = excluded.content,
|
|
@@ -66,16 +67,20 @@ export function upsertFromFile(mf: MemoryFile): void {
|
|
|
66
67
|
function writeRow(mf: MemoryFile, mtimeMs: number): void {
|
|
67
68
|
const { fm, body, filePath } = mf;
|
|
68
69
|
const tags = (fm.tags ?? []).join(",");
|
|
69
|
-
// D80 index-time expansion: curated synonym/translation variants for
|
|
70
|
-
// memory's keywords are indexed in
|
|
71
|
-
//
|
|
72
|
-
//
|
|
70
|
+
// D80 index-time expansion, D83: curated synonym/translation variants for
|
|
71
|
+
// the memory's keywords are indexed in their own `expansions` column —
|
|
72
|
+
// never mixed into `aliases`, which holds only user/authored alternate
|
|
73
|
+
// phrasings (so `(aka: …)` attribution stays honest). The CJK bigram
|
|
74
|
+
// column still covers expansions, so CJK variants stay substring-
|
|
75
|
+
// searchable. Derived at sync time — the Markdown file stays the source
|
|
76
|
+
// of truth.
|
|
73
77
|
const supplied = fm.aliases ?? [];
|
|
74
78
|
const have = new Set(supplied.map((a) => a.toLowerCase()));
|
|
75
79
|
const auto = expansionVariantsForDoc(body + "\n" + tags).filter(
|
|
76
80
|
(a) => !have.has(a.toLowerCase()),
|
|
77
81
|
);
|
|
78
|
-
const aliases =
|
|
82
|
+
const aliases = supplied.join(",");
|
|
83
|
+
const expansions = auto.join(",");
|
|
79
84
|
// D81: user-defined alias vocabulary (single nickname → referent pair).
|
|
80
85
|
db().prepare(UPSERT_SQL).run(
|
|
81
86
|
fm.id,
|
|
@@ -89,10 +94,11 @@ function writeRow(mf: MemoryFile, mtimeMs: number): void {
|
|
|
89
94
|
fm.status,
|
|
90
95
|
tags,
|
|
91
96
|
aliases,
|
|
97
|
+
expansions,
|
|
92
98
|
fm.alias ?? "",
|
|
93
99
|
fm.target ?? "",
|
|
94
100
|
body,
|
|
95
|
-
cjkIndexText(body + "\n" + tags + "\n" + aliases),
|
|
101
|
+
cjkIndexText(body + "\n" + tags + "\n" + aliases + "\n" + expansions),
|
|
96
102
|
contentHash(body),
|
|
97
103
|
fm.superseded_by ?? null,
|
|
98
104
|
fm.source ?? "",
|
package/src/tools/ops.ts
CHANGED
|
@@ -143,7 +143,12 @@ export const memorySearchArgs = {
|
|
|
143
143
|
.enum(["project", "personal", "both"])
|
|
144
144
|
.optional()
|
|
145
145
|
.describe("Which scope(s) to search. Default: both."),
|
|
146
|
-
type: z
|
|
146
|
+
type: z
|
|
147
|
+
.enum(MEMORY_TYPE_TAXONOMY)
|
|
148
|
+
.optional()
|
|
149
|
+
.describe(
|
|
150
|
+
"Restrict to memories of this type. Closed taxonomy — a misspelled type is rejected, never silently empty.",
|
|
151
|
+
),
|
|
147
152
|
limit: z.number().int().min(1).max(50).optional().describe("Max results to return (default 8)."),
|
|
148
153
|
};
|
|
149
154
|
export type MemorySearchArgs = z.infer<z.ZodObject<typeof memorySearchArgs>>;
|
|
@@ -155,7 +160,12 @@ export const memoryListArgs = {
|
|
|
155
160
|
.describe(
|
|
156
161
|
"Which scope(s) to list. `project` = tied to this repo. `personal` = global across all your projects. `both` lists each scope's newest. Default: project.",
|
|
157
162
|
),
|
|
158
|
-
type: z
|
|
163
|
+
type: z
|
|
164
|
+
.enum(MEMORY_TYPE_TAXONOMY)
|
|
165
|
+
.optional()
|
|
166
|
+
.describe(
|
|
167
|
+
"Only list memories of this type. Closed taxonomy — a misspelled type is rejected, never silently empty.",
|
|
168
|
+
),
|
|
159
169
|
limit: z.number().int().min(1).max(100).optional().describe("Max entries to list, newest first (default 20)."),
|
|
160
170
|
include: z
|
|
161
171
|
.enum(["active", "all"])
|