@context-vault/core 2.10.3 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/capture/file-ops.js +4 -0
- package/src/capture/index.js +26 -0
- package/src/constants.js +13 -0
- package/src/core/categories.js +11 -0
- package/src/core/config.js +32 -0
- package/src/core/frontmatter.js +2 -0
- package/src/core/status.js +165 -0
- package/src/core/telemetry.js +90 -0
- package/src/index/db.js +86 -9
- package/src/index/index.js +37 -1
- package/src/index.js +1 -1
- package/src/retrieve/index.js +81 -4
- package/src/server/tools/context-status.js +37 -3
- package/src/server/tools/get-context.js +23 -2
- package/src/server/tools/ingest-url.js +0 -11
- package/src/server/tools/list-context.js +7 -3
- package/src/server/tools/save-context.js +144 -13
- package/src/server/tools.js +13 -0
package/src/index/db.js
CHANGED
|
@@ -55,7 +55,9 @@ export const SCHEMA_DDL = `
|
|
|
55
55
|
file_path TEXT UNIQUE,
|
|
56
56
|
identity_key TEXT,
|
|
57
57
|
expires_at TEXT,
|
|
58
|
+
superseded_by TEXT,
|
|
58
59
|
created_at TEXT DEFAULT (datetime('now')),
|
|
60
|
+
updated_at TEXT,
|
|
59
61
|
user_id TEXT,
|
|
60
62
|
team_id TEXT,
|
|
61
63
|
body_encrypted BLOB,
|
|
@@ -67,9 +69,11 @@ export const SCHEMA_DDL = `
|
|
|
67
69
|
CREATE INDEX IF NOT EXISTS idx_vault_kind ON vault(kind);
|
|
68
70
|
CREATE INDEX IF NOT EXISTS idx_vault_category ON vault(category);
|
|
69
71
|
CREATE INDEX IF NOT EXISTS idx_vault_category_created ON vault(category, created_at DESC);
|
|
72
|
+
CREATE INDEX IF NOT EXISTS idx_vault_updated ON vault(updated_at DESC);
|
|
70
73
|
CREATE INDEX IF NOT EXISTS idx_vault_user ON vault(user_id);
|
|
71
74
|
CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id);
|
|
72
75
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(user_id, kind, identity_key) WHERE identity_key IS NOT NULL;
|
|
76
|
+
CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL;
|
|
73
77
|
|
|
74
78
|
-- Single FTS5 table
|
|
75
79
|
CREATE VIRTUAL TABLE IF NOT EXISTS vault_fts USING fts5(
|
|
@@ -147,13 +151,13 @@ export async function initDatabase(dbPath) {
|
|
|
147
151
|
|
|
148
152
|
const freshDb = createDb(dbPath);
|
|
149
153
|
freshDb.exec(SCHEMA_DDL);
|
|
150
|
-
freshDb.exec("PRAGMA user_version =
|
|
154
|
+
freshDb.exec("PRAGMA user_version = 9");
|
|
151
155
|
return freshDb;
|
|
152
156
|
}
|
|
153
157
|
|
|
154
158
|
if (version < 5) {
|
|
155
159
|
db.exec(SCHEMA_DDL);
|
|
156
|
-
db.exec("PRAGMA user_version =
|
|
160
|
+
db.exec("PRAGMA user_version = 9");
|
|
157
161
|
} else if (version === 5) {
|
|
158
162
|
// v5 -> v6 migration: add multi-tenancy + encryption columns
|
|
159
163
|
// Wrapped in transaction with duplicate-column guards for idempotent retry
|
|
@@ -171,24 +175,91 @@ export async function initDatabase(dbPath) {
|
|
|
171
175
|
addColumnSafe(`ALTER TABLE vault ADD COLUMN meta_encrypted BLOB`);
|
|
172
176
|
addColumnSafe(`ALTER TABLE vault ADD COLUMN iv BLOB`);
|
|
173
177
|
addColumnSafe(`ALTER TABLE vault ADD COLUMN team_id TEXT`);
|
|
178
|
+
addColumnSafe(`ALTER TABLE vault ADD COLUMN updated_at TEXT`);
|
|
179
|
+
addColumnSafe(`ALTER TABLE vault ADD COLUMN superseded_by TEXT`);
|
|
174
180
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_user ON vault(user_id)`);
|
|
175
181
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id)`);
|
|
176
182
|
db.exec(`DROP INDEX IF EXISTS idx_vault_identity`);
|
|
177
183
|
db.exec(
|
|
178
184
|
`CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(user_id, kind, identity_key) WHERE identity_key IS NOT NULL`,
|
|
179
185
|
);
|
|
180
|
-
db.exec(
|
|
186
|
+
db.exec(
|
|
187
|
+
`UPDATE vault SET updated_at = created_at WHERE updated_at IS NULL`,
|
|
188
|
+
);
|
|
189
|
+
db.exec(
|
|
190
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_updated ON vault(updated_at DESC)`,
|
|
191
|
+
);
|
|
192
|
+
db.exec(
|
|
193
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
|
|
194
|
+
);
|
|
195
|
+
db.exec("PRAGMA user_version = 9");
|
|
181
196
|
});
|
|
182
197
|
} else if (version === 6) {
|
|
183
|
-
// v6 -> v7 migration: add team_id
|
|
198
|
+
// v6 -> v7+v8+v9 migration: add team_id, updated_at, superseded_by columns
|
|
184
199
|
runTransaction(db, () => {
|
|
185
200
|
try {
|
|
186
201
|
db.exec(`ALTER TABLE vault ADD COLUMN team_id TEXT`);
|
|
187
202
|
} catch (e) {
|
|
188
203
|
if (!e.message.includes("duplicate column")) throw e;
|
|
189
204
|
}
|
|
205
|
+
try {
|
|
206
|
+
db.exec(`ALTER TABLE vault ADD COLUMN updated_at TEXT`);
|
|
207
|
+
} catch (e) {
|
|
208
|
+
if (!e.message.includes("duplicate column")) throw e;
|
|
209
|
+
}
|
|
210
|
+
try {
|
|
211
|
+
db.exec(`ALTER TABLE vault ADD COLUMN superseded_by TEXT`);
|
|
212
|
+
} catch (e) {
|
|
213
|
+
if (!e.message.includes("duplicate column")) throw e;
|
|
214
|
+
}
|
|
190
215
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id)`);
|
|
191
|
-
db.exec(
|
|
216
|
+
db.exec(
|
|
217
|
+
`UPDATE vault SET updated_at = created_at WHERE updated_at IS NULL`,
|
|
218
|
+
);
|
|
219
|
+
db.exec(
|
|
220
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_updated ON vault(updated_at DESC)`,
|
|
221
|
+
);
|
|
222
|
+
db.exec(
|
|
223
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
|
|
224
|
+
);
|
|
225
|
+
db.exec("PRAGMA user_version = 9");
|
|
226
|
+
});
|
|
227
|
+
} else if (version === 7) {
|
|
228
|
+
// v7 -> v8+v9 migration: add updated_at, superseded_by columns
|
|
229
|
+
runTransaction(db, () => {
|
|
230
|
+
try {
|
|
231
|
+
db.exec(`ALTER TABLE vault ADD COLUMN updated_at TEXT`);
|
|
232
|
+
} catch (e) {
|
|
233
|
+
if (!e.message.includes("duplicate column")) throw e;
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
db.exec(`ALTER TABLE vault ADD COLUMN superseded_by TEXT`);
|
|
237
|
+
} catch (e) {
|
|
238
|
+
if (!e.message.includes("duplicate column")) throw e;
|
|
239
|
+
}
|
|
240
|
+
db.exec(
|
|
241
|
+
`UPDATE vault SET updated_at = created_at WHERE updated_at IS NULL`,
|
|
242
|
+
);
|
|
243
|
+
db.exec(
|
|
244
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_updated ON vault(updated_at DESC)`,
|
|
245
|
+
);
|
|
246
|
+
db.exec(
|
|
247
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
|
|
248
|
+
);
|
|
249
|
+
db.exec("PRAGMA user_version = 9");
|
|
250
|
+
});
|
|
251
|
+
} else if (version === 8) {
|
|
252
|
+
// v8 -> v9 migration: add superseded_by column
|
|
253
|
+
runTransaction(db, () => {
|
|
254
|
+
try {
|
|
255
|
+
db.exec(`ALTER TABLE vault ADD COLUMN superseded_by TEXT`);
|
|
256
|
+
} catch (e) {
|
|
257
|
+
if (!e.message.includes("duplicate column")) throw e;
|
|
258
|
+
}
|
|
259
|
+
db.exec(
|
|
260
|
+
`CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
|
|
261
|
+
);
|
|
262
|
+
db.exec("PRAGMA user_version = 9");
|
|
192
263
|
});
|
|
193
264
|
}
|
|
194
265
|
|
|
@@ -199,13 +270,13 @@ export function prepareStatements(db) {
|
|
|
199
270
|
try {
|
|
200
271
|
return {
|
|
201
272
|
insertEntry: db.prepare(
|
|
202
|
-
`INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
273
|
+
`INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
203
274
|
),
|
|
204
275
|
insertEntryEncrypted: db.prepare(
|
|
205
|
-
`INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, body_encrypted, title_encrypted, meta_encrypted, iv) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
276
|
+
`INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at, body_encrypted, title_encrypted, meta_encrypted, iv) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
206
277
|
),
|
|
207
278
|
updateEntry: db.prepare(
|
|
208
|
-
`UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, identity_key = ?, expires_at =
|
|
279
|
+
`UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, identity_key = ?, expires_at = ?, updated_at = datetime('now') WHERE file_path = ?`,
|
|
209
280
|
),
|
|
210
281
|
deleteEntry: db.prepare(`DELETE FROM vault WHERE id = ?`),
|
|
211
282
|
getRowid: db.prepare(`SELECT rowid FROM vault WHERE id = ?`),
|
|
@@ -215,12 +286,18 @@ export function prepareStatements(db) {
|
|
|
215
286
|
`SELECT * FROM vault WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
|
|
216
287
|
),
|
|
217
288
|
upsertByIdentityKey: db.prepare(
|
|
218
|
-
`UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at =
|
|
289
|
+
`UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at = ?, updated_at = datetime('now') WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
|
|
219
290
|
),
|
|
220
291
|
insertVecStmt: db.prepare(
|
|
221
292
|
`INSERT INTO vault_vec (rowid, embedding) VALUES (?, ?)`,
|
|
222
293
|
),
|
|
223
294
|
deleteVecStmt: db.prepare(`DELETE FROM vault_vec WHERE rowid = ?`),
|
|
295
|
+
updateSupersededBy: db.prepare(
|
|
296
|
+
`UPDATE vault SET superseded_by = ? WHERE id = ?`,
|
|
297
|
+
),
|
|
298
|
+
clearSupersededByRef: db.prepare(
|
|
299
|
+
`UPDATE vault SET superseded_by = NULL WHERE superseded_by = ?`,
|
|
300
|
+
),
|
|
224
301
|
};
|
|
225
302
|
} catch (e) {
|
|
226
303
|
throw new Error(
|
package/src/index/index.js
CHANGED
|
@@ -111,6 +111,7 @@ export async function indexEntry(
|
|
|
111
111
|
identity_key || null,
|
|
112
112
|
expires_at || null,
|
|
113
113
|
createdAt,
|
|
114
|
+
createdAt,
|
|
114
115
|
encrypted.body_encrypted,
|
|
115
116
|
encrypted.title_encrypted,
|
|
116
117
|
encrypted.meta_encrypted,
|
|
@@ -131,6 +132,7 @@ export async function indexEntry(
|
|
|
131
132
|
identity_key || null,
|
|
132
133
|
expires_at || null,
|
|
133
134
|
createdAt,
|
|
135
|
+
createdAt,
|
|
134
136
|
);
|
|
135
137
|
}
|
|
136
138
|
} catch (e) {
|
|
@@ -186,6 +188,39 @@ export async function indexEntry(
|
|
|
186
188
|
}
|
|
187
189
|
}
|
|
188
190
|
|
|
191
|
+
/**
|
|
192
|
+
* Prune expired entries: delete files, vec rows, and DB rows for all entries
|
|
193
|
+
* where expires_at <= now(). Safe to call on startup or CLI — non-destructive
|
|
194
|
+
* to active data.
|
|
195
|
+
*
|
|
196
|
+
* @param {import('../server/types.js').BaseCtx} ctx
|
|
197
|
+
* @returns {Promise<number>} count of pruned entries
|
|
198
|
+
*/
|
|
199
|
+
export async function pruneExpired(ctx) {
|
|
200
|
+
const expired = ctx.db
|
|
201
|
+
.prepare(
|
|
202
|
+
"SELECT id, file_path FROM vault WHERE expires_at IS NOT NULL AND expires_at <= datetime('now')",
|
|
203
|
+
)
|
|
204
|
+
.all();
|
|
205
|
+
|
|
206
|
+
for (const row of expired) {
|
|
207
|
+
if (row.file_path) {
|
|
208
|
+
try {
|
|
209
|
+
unlinkSync(row.file_path);
|
|
210
|
+
} catch {}
|
|
211
|
+
}
|
|
212
|
+
const vRowid = ctx.stmts.getRowid.get(row.id)?.rowid;
|
|
213
|
+
if (vRowid) {
|
|
214
|
+
try {
|
|
215
|
+
ctx.deleteVec(Number(vRowid));
|
|
216
|
+
} catch {}
|
|
217
|
+
}
|
|
218
|
+
ctx.stmts.deleteEntry.run(row.id);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
return expired.length;
|
|
222
|
+
}
|
|
223
|
+
|
|
189
224
|
/**
|
|
190
225
|
* Bulk reindex: sync vault directory into the database.
|
|
191
226
|
* P2: Wrapped in a transaction for atomicity.
|
|
@@ -205,7 +240,7 @@ export async function reindex(ctx, opts = {}) {
|
|
|
205
240
|
// Use INSERT OR IGNORE for reindex — handles files with duplicate frontmatter IDs
|
|
206
241
|
// user_id is NULL for reindex (always local mode)
|
|
207
242
|
const upsertEntry = ctx.db.prepare(
|
|
208
|
-
`INSERT OR IGNORE INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at) VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
243
|
+
`INSERT OR IGNORE INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at) VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
209
244
|
);
|
|
210
245
|
|
|
211
246
|
// Auto-discover kind directories, supporting both:
|
|
@@ -309,6 +344,7 @@ export async function reindex(ctx, opts = {}) {
|
|
|
309
344
|
identity_key,
|
|
310
345
|
expires_at,
|
|
311
346
|
created,
|
|
347
|
+
fmMeta.updated || created,
|
|
312
348
|
);
|
|
313
349
|
if (result.changes > 0) {
|
|
314
350
|
const rowidResult = ctx.stmts.getRowid.get(id);
|
package/src/index.js
CHANGED
|
@@ -47,7 +47,7 @@ export {
|
|
|
47
47
|
deleteVec,
|
|
48
48
|
} from "./index/db.js";
|
|
49
49
|
export { embed, embedBatch, resetEmbedPipeline } from "./index/embed.js";
|
|
50
|
-
export { indexEntry, reindex } from "./index/index.js";
|
|
50
|
+
export { indexEntry, reindex, pruneExpired } from "./index/index.js";
|
|
51
51
|
|
|
52
52
|
// Retrieve layer
|
|
53
53
|
export { hybridSearch } from "./retrieve/index.js";
|
package/src/retrieve/index.js
CHANGED
|
@@ -9,10 +9,23 @@
|
|
|
9
9
|
|
|
10
10
|
const FTS_WEIGHT = 0.4;
|
|
11
11
|
const VEC_WEIGHT = 0.6;
|
|
12
|
+
const NEAR_DUP_THRESHOLD = 0.92;
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
|
-
*
|
|
15
|
-
|
|
15
|
+
* Dot product of two Float32Array vectors (cosine similarity for unit vectors).
|
|
16
|
+
*/
|
|
17
|
+
export function dotProduct(a, b) {
|
|
18
|
+
let sum = 0;
|
|
19
|
+
for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
|
|
20
|
+
return sum;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Build a tiered FTS5 query that prioritises phrase match, then proximity,
|
|
25
|
+
* then AND. Multi-word queries become:
|
|
26
|
+
* "word1 word2" OR NEAR("word1" "word2", 10) OR "word1" AND "word2"
|
|
27
|
+
* Single-word queries remain a simple quoted term.
|
|
28
|
+
* Returns null if no valid words remain after stripping FTS5 metacharacters.
|
|
16
29
|
*/
|
|
17
30
|
export function buildFtsQuery(query) {
|
|
18
31
|
const words = query
|
|
@@ -20,7 +33,11 @@ export function buildFtsQuery(query) {
|
|
|
20
33
|
.map((w) => w.replace(/[*"():^~{}]/g, ""))
|
|
21
34
|
.filter((w) => w.length > 0);
|
|
22
35
|
if (!words.length) return null;
|
|
23
|
-
|
|
36
|
+
if (words.length === 1) return `"${words[0]}"`;
|
|
37
|
+
const phrase = `"${words.join(" ")}"`;
|
|
38
|
+
const near = `NEAR(${words.map((w) => `"${w}"`).join(" ")}, 10)`;
|
|
39
|
+
const and = words.map((w) => `"${w}"`).join(" AND ");
|
|
40
|
+
return `${phrase} OR ${near} OR ${and}`;
|
|
24
41
|
}
|
|
25
42
|
|
|
26
43
|
/**
|
|
@@ -44,6 +61,7 @@ export function buildFilterClauses({
|
|
|
44
61
|
until,
|
|
45
62
|
userIdFilter,
|
|
46
63
|
teamIdFilter,
|
|
64
|
+
includeSuperseeded = false,
|
|
47
65
|
}) {
|
|
48
66
|
const clauses = [];
|
|
49
67
|
const params = [];
|
|
@@ -68,6 +86,9 @@ export function buildFilterClauses({
|
|
|
68
86
|
params.push(until);
|
|
69
87
|
}
|
|
70
88
|
clauses.push("(e.expires_at IS NULL OR e.expires_at > datetime('now'))");
|
|
89
|
+
if (!includeSuperseeded) {
|
|
90
|
+
clauses.push("e.superseded_by IS NULL");
|
|
91
|
+
}
|
|
71
92
|
return { clauses, params };
|
|
72
93
|
}
|
|
73
94
|
|
|
@@ -92,15 +113,19 @@ export async function hybridSearch(
|
|
|
92
113
|
decayDays = 30,
|
|
93
114
|
userIdFilter,
|
|
94
115
|
teamIdFilter = null,
|
|
116
|
+
includeSuperseeded = false,
|
|
95
117
|
} = {},
|
|
96
118
|
) {
|
|
97
119
|
const results = new Map();
|
|
120
|
+
const idToRowid = new Map();
|
|
121
|
+
let queryVec = null;
|
|
98
122
|
const extraFilters = buildFilterClauses({
|
|
99
123
|
categoryFilter,
|
|
100
124
|
since,
|
|
101
125
|
until,
|
|
102
126
|
userIdFilter,
|
|
103
127
|
teamIdFilter,
|
|
128
|
+
includeSuperseeded,
|
|
104
129
|
});
|
|
105
130
|
|
|
106
131
|
// FTS5 search
|
|
@@ -144,7 +169,7 @@ export async function hybridSearch(
|
|
|
144
169
|
.prepare("SELECT COUNT(*) as c FROM vault_vec")
|
|
145
170
|
.get().c;
|
|
146
171
|
if (vecCount > 0) {
|
|
147
|
-
|
|
172
|
+
queryVec = await ctx.embed(query);
|
|
148
173
|
if (queryVec) {
|
|
149
174
|
// Increase limits in hosted mode to compensate for post-filtering
|
|
150
175
|
const hasPostFilter = userIdFilter !== undefined || teamIdFilter;
|
|
@@ -188,6 +213,7 @@ export async function hybridSearch(
|
|
|
188
213
|
continue;
|
|
189
214
|
|
|
190
215
|
const { rowid: _rowid, ...cleanRow } = row;
|
|
216
|
+
idToRowid.set(cleanRow.id, Number(row.rowid));
|
|
191
217
|
// sqlite-vec returns L2 distance [0, 2] for normalized vectors.
|
|
192
218
|
// Convert to similarity [1, 0] with: 1 - distance/2
|
|
193
219
|
const vecScore = Math.max(0, 1 - vr.distance / 2) * VEC_WEIGHT;
|
|
@@ -215,5 +241,56 @@ export async function hybridSearch(
|
|
|
215
241
|
}
|
|
216
242
|
|
|
217
243
|
const sorted = [...results.values()].sort((a, b) => b.score - a.score);
|
|
244
|
+
|
|
245
|
+
// Near-duplicate suppression: when embeddings are available and we have more
|
|
246
|
+
// candidates than needed, skip results that are too similar to already-selected ones.
|
|
247
|
+
if (queryVec && idToRowid.size > 0 && sorted.length > limit) {
|
|
248
|
+
const rowidsToFetch = sorted
|
|
249
|
+
.filter((c) => idToRowid.has(c.id))
|
|
250
|
+
.map((c) => idToRowid.get(c.id));
|
|
251
|
+
|
|
252
|
+
const embeddingMap = new Map();
|
|
253
|
+
if (rowidsToFetch.length > 0) {
|
|
254
|
+
try {
|
|
255
|
+
const placeholders = rowidsToFetch.map(() => "?").join(",");
|
|
256
|
+
const vecData = ctx.db
|
|
257
|
+
.prepare(
|
|
258
|
+
`SELECT rowid, embedding FROM vault_vec WHERE rowid IN (${placeholders})`,
|
|
259
|
+
)
|
|
260
|
+
.all(...rowidsToFetch);
|
|
261
|
+
for (const row of vecData) {
|
|
262
|
+
const buf = row.embedding;
|
|
263
|
+
if (buf) {
|
|
264
|
+
embeddingMap.set(
|
|
265
|
+
Number(row.rowid),
|
|
266
|
+
new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4),
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
} catch (_) {
|
|
271
|
+
return sorted.slice(offset, offset + limit);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
const selected = [];
|
|
276
|
+
const selectedVecs = [];
|
|
277
|
+
for (const candidate of sorted) {
|
|
278
|
+
if (selected.length >= offset + limit) break;
|
|
279
|
+
const rowid = idToRowid.get(candidate.id);
|
|
280
|
+
const vec = rowid !== undefined ? embeddingMap.get(rowid) : null;
|
|
281
|
+
if (vec && selectedVecs.length > 0) {
|
|
282
|
+
let maxSim = 0;
|
|
283
|
+
for (const sv of selectedVecs) {
|
|
284
|
+
const sim = dotProduct(sv, vec);
|
|
285
|
+
if (sim > maxSim) maxSim = sim;
|
|
286
|
+
}
|
|
287
|
+
if (maxSim > NEAR_DUP_THRESHOLD) continue;
|
|
288
|
+
}
|
|
289
|
+
selected.push(candidate);
|
|
290
|
+
if (vec) selectedVecs.push(vec);
|
|
291
|
+
}
|
|
292
|
+
return selected.slice(offset, offset + limit);
|
|
293
|
+
}
|
|
294
|
+
|
|
218
295
|
return sorted.slice(offset, offset + limit);
|
|
219
296
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { gatherVaultStatus } from "../../core/status.js";
|
|
1
|
+
import { gatherVaultStatus, computeGrowthWarnings } from "../../core/status.js";
|
|
2
2
|
import { errorLogPath, errorLogCount } from "../../core/error-log.js";
|
|
3
3
|
import { ok } from "../helpers.js";
|
|
4
4
|
|
|
@@ -40,7 +40,7 @@ export function handler(_args, ctx) {
|
|
|
40
40
|
`Data dir: ${config.dataDir}`,
|
|
41
41
|
`Config: ${config.configPath}`,
|
|
42
42
|
`Resolved via: ${status.resolvedFrom}`,
|
|
43
|
-
`Schema:
|
|
43
|
+
`Schema: v9 (updated_at, superseded_by)`,
|
|
44
44
|
];
|
|
45
45
|
|
|
46
46
|
if (status.embeddingStatus) {
|
|
@@ -58,7 +58,7 @@ export function handler(_args, ctx) {
|
|
|
58
58
|
lines.push(`Decay: ${config.eventDecayDays} days (event recency window)`);
|
|
59
59
|
if (status.expiredCount > 0) {
|
|
60
60
|
lines.push(
|
|
61
|
-
`Expired: ${status.expiredCount} entries (
|
|
61
|
+
`Expired: ${status.expiredCount} entries pending prune (run \`context-vault prune\` to remove now)`,
|
|
62
62
|
);
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -93,6 +93,25 @@ export function handler(_args, ctx) {
|
|
|
93
93
|
lines.push(`Auto-reindex will fix this on next search or save.`);
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
if (status.staleKnowledge?.length > 0) {
|
|
97
|
+
lines.push(``);
|
|
98
|
+
lines.push(`### ⚠ Potentially Stale Knowledge`);
|
|
99
|
+
lines.push(
|
|
100
|
+
`Not updated within kind staleness window (pattern: 180d, decision: 365d, reference: 90d):`,
|
|
101
|
+
);
|
|
102
|
+
for (const entry of status.staleKnowledge) {
|
|
103
|
+
const lastUpdated = entry.last_updated
|
|
104
|
+
? entry.last_updated.split("T")[0]
|
|
105
|
+
: "unknown";
|
|
106
|
+
lines.push(
|
|
107
|
+
`- "${entry.title}" (${entry.kind}) — last updated ${lastUpdated}`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
lines.push(
|
|
111
|
+
`Use save_context to refresh or add expires_at to retire stale entries.`,
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
96
115
|
// Error log
|
|
97
116
|
const logPath = errorLogPath(config.dataDir);
|
|
98
117
|
const logCount = errorLogCount(config.dataDir);
|
|
@@ -120,6 +139,21 @@ export function handler(_args, ctx) {
|
|
|
120
139
|
}
|
|
121
140
|
}
|
|
122
141
|
|
|
142
|
+
// Growth warnings
|
|
143
|
+
const growth = computeGrowthWarnings(status, config.thresholds);
|
|
144
|
+
if (growth.hasWarnings) {
|
|
145
|
+
lines.push("", "### ⚠ Vault Growth Warning");
|
|
146
|
+
for (const w of growth.warnings) {
|
|
147
|
+
lines.push(` ${w.message}`);
|
|
148
|
+
}
|
|
149
|
+
if (growth.actions.length) {
|
|
150
|
+
lines.push("", "Suggested growth actions:");
|
|
151
|
+
for (const a of growth.actions) {
|
|
152
|
+
lines.push(` • ${a}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
123
157
|
// Suggested actions
|
|
124
158
|
const actions = [];
|
|
125
159
|
if (status.stalePaths)
|
|
@@ -42,6 +42,12 @@ export const inputSchema = {
|
|
|
42
42
|
.optional()
|
|
43
43
|
.describe("ISO date, return entries created before this"),
|
|
44
44
|
limit: z.number().optional().describe("Max results to return (default 10)"),
|
|
45
|
+
include_superseded: z
|
|
46
|
+
.boolean()
|
|
47
|
+
.optional()
|
|
48
|
+
.describe(
|
|
49
|
+
"If true, include entries that have been superseded by newer ones. Default: false.",
|
|
50
|
+
),
|
|
45
51
|
};
|
|
46
52
|
|
|
47
53
|
/**
|
|
@@ -50,7 +56,17 @@ export const inputSchema = {
|
|
|
50
56
|
* @param {import('../types.js').ToolShared} shared
|
|
51
57
|
*/
|
|
52
58
|
export async function handler(
|
|
53
|
-
{
|
|
59
|
+
{
|
|
60
|
+
query,
|
|
61
|
+
kind,
|
|
62
|
+
category,
|
|
63
|
+
identity_key,
|
|
64
|
+
tags,
|
|
65
|
+
since,
|
|
66
|
+
until,
|
|
67
|
+
limit,
|
|
68
|
+
include_superseded,
|
|
69
|
+
},
|
|
54
70
|
ctx,
|
|
55
71
|
{ ensureIndexed, reindexFailed },
|
|
56
72
|
) {
|
|
@@ -126,6 +142,7 @@ export async function handler(
|
|
|
126
142
|
limit: fetchLimit,
|
|
127
143
|
decayDays: config.eventDecayDays || 30,
|
|
128
144
|
userIdFilter: userId,
|
|
145
|
+
includeSuperseeded: include_superseded ?? false,
|
|
129
146
|
});
|
|
130
147
|
|
|
131
148
|
// Post-filter by tags if provided, then apply requested limit
|
|
@@ -238,8 +255,12 @@ export async function handler(
|
|
|
238
255
|
lines.push(
|
|
239
256
|
`### [${i + 1}/${filtered.length}] ${r.title || "(untitled)"} [${r.kind}/${r.category}]`,
|
|
240
257
|
);
|
|
258
|
+
const dateStr =
|
|
259
|
+
r.updated_at && r.updated_at !== r.created_at
|
|
260
|
+
? `${r.created_at} (updated ${r.updated_at})`
|
|
261
|
+
: r.created_at || "";
|
|
241
262
|
lines.push(
|
|
242
|
-
`${r.score.toFixed(3)} · ${tagStr} · ${relPath} · id: \`${r.id}\``,
|
|
263
|
+
`${r.score.toFixed(3)} · ${tagStr} · ${relPath} · ${dateStr} · id: \`${r.id}\``,
|
|
243
264
|
);
|
|
244
265
|
lines.push(r.body?.slice(0, 300) + (r.body?.length > 300 ? "..." : ""));
|
|
245
266
|
lines.push("");
|
|
@@ -65,17 +65,6 @@ export async function handler(
|
|
|
65
65
|
|
|
66
66
|
await ensureIndexed();
|
|
67
67
|
|
|
68
|
-
// Hosted tier limit enforcement
|
|
69
|
-
if (ctx.checkLimits) {
|
|
70
|
-
const usage = ctx.checkLimits();
|
|
71
|
-
if (usage.entryCount >= usage.maxEntries) {
|
|
72
|
-
return err(
|
|
73
|
-
`Entry limit reached (${usage.maxEntries}). Upgrade to Pro for unlimited entries.`,
|
|
74
|
-
"LIMIT_EXCEEDED",
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
68
|
try {
|
|
80
69
|
const { ingestUrl } = await import("../../capture/ingest-url.js");
|
|
81
70
|
const entryData = await ingestUrl(targetUrl, { kind, tags });
|
|
@@ -6,7 +6,7 @@ import { ok } from "../helpers.js";
|
|
|
6
6
|
export const name = "list_context";
|
|
7
7
|
|
|
8
8
|
export const description =
|
|
9
|
-
"Browse vault entries without a search query. Returns id, title, kind, category, tags, created_at. Use get_context with a query for semantic search. Use this to browse by tags or find recent entries.";
|
|
9
|
+
"Browse vault entries without a search query. Returns id, title, kind, category, tags, created_at, updated_at. Use get_context with a query for semantic search. Use this to browse by tags or find recent entries.";
|
|
10
10
|
|
|
11
11
|
export const inputSchema = {
|
|
12
12
|
kind: z
|
|
@@ -101,7 +101,7 @@ export async function handler(
|
|
|
101
101
|
params.push(fetchLimit, effectiveOffset);
|
|
102
102
|
const rows = ctx.db
|
|
103
103
|
.prepare(
|
|
104
|
-
`SELECT id, title, kind, category, tags, created_at, SUBSTR(body, 1, 120) as preview FROM vault ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
|
104
|
+
`SELECT id, title, kind, category, tags, created_at, updated_at, SUBSTR(body, 1, 120) as preview FROM vault ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
|
105
105
|
)
|
|
106
106
|
.all(...params);
|
|
107
107
|
|
|
@@ -140,8 +140,12 @@ export async function handler(
|
|
|
140
140
|
for (const r of filtered) {
|
|
141
141
|
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
142
142
|
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
143
|
+
const dateStr =
|
|
144
|
+
r.updated_at && r.updated_at !== r.created_at
|
|
145
|
+
? `${r.created_at} (updated ${r.updated_at})`
|
|
146
|
+
: r.created_at;
|
|
143
147
|
lines.push(
|
|
144
|
-
`- **${r.title || "(untitled)"}** [${r.kind}/${r.category}] — ${tagStr} — ${
|
|
148
|
+
`- **${r.title || "(untitled)"}** [${r.kind}/${r.category}] — ${tagStr} — ${dateStr} — \`${r.id}\``,
|
|
145
149
|
);
|
|
146
150
|
if (r.preview)
|
|
147
151
|
lines.push(
|