@context-vault/core 2.15.0 → 2.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -3
- package/src/capture/file-ops.js +2 -0
- package/src/capture/index.js +14 -0
- package/src/constants.js +7 -2
- package/src/core/categories.js +1 -0
- package/src/core/config.js +9 -2
- package/src/core/files.js +6 -29
- package/src/core/frontmatter.js +1 -0
- package/src/core/linking.js +161 -0
- package/src/core/migrate-dirs.js +196 -0
- package/src/core/status.js +28 -2
- package/src/core/temporal.js +146 -0
- package/src/index/db.js +178 -8
- package/src/index/index.js +113 -48
- package/src/index.js +5 -0
- package/src/retrieve/index.js +16 -136
- package/src/server/tools/context-status.js +7 -0
- package/src/server/tools/create-snapshot.js +37 -68
- package/src/server/tools/get-context.js +120 -19
- package/src/server/tools/save-context.js +29 -6
- package/src/server/tools.js +0 -2
- package/src/server/tools/submit-feedback.js +0 -55
package/src/retrieve/index.js
CHANGED
|
@@ -11,8 +11,6 @@ const NEAR_DUP_THRESHOLD = 0.92;
|
|
|
11
11
|
|
|
12
12
|
const RRF_K = 60;
|
|
13
13
|
|
|
14
|
-
const MMR_LAMBDA = 0.7;
|
|
15
|
-
|
|
16
14
|
/**
|
|
17
15
|
* Exponential recency decay score based on updated_at timestamp.
|
|
18
16
|
* Returns e^(-decayRate * ageDays) for valid dates, or 0.5 as a neutral
|
|
@@ -74,6 +72,7 @@ export function recencyBoost(createdAt, category, decayDays = 30) {
|
|
|
74
72
|
*/
|
|
75
73
|
export function buildFilterClauses({
|
|
76
74
|
categoryFilter,
|
|
75
|
+
excludeEvents = false,
|
|
77
76
|
since,
|
|
78
77
|
until,
|
|
79
78
|
userIdFilter,
|
|
@@ -94,6 +93,9 @@ export function buildFilterClauses({
|
|
|
94
93
|
clauses.push("e.category = ?");
|
|
95
94
|
params.push(categoryFilter);
|
|
96
95
|
}
|
|
96
|
+
if (excludeEvents && !categoryFilter) {
|
|
97
|
+
clauses.push("e.category != 'event'");
|
|
98
|
+
}
|
|
97
99
|
if (since) {
|
|
98
100
|
clauses.push("e.created_at >= ?");
|
|
99
101
|
params.push(since);
|
|
@@ -128,108 +130,16 @@ export function reciprocalRankFusion(rankedLists, k = RRF_K) {
|
|
|
128
130
|
return scores;
|
|
129
131
|
}
|
|
130
132
|
|
|
131
|
-
/**
|
|
132
|
-
* Jaccard similarity between two strings based on word sets.
|
|
133
|
-
* Used as a fallback for MMR when embedding vectors are unavailable.
|
|
134
|
-
*
|
|
135
|
-
* @param {string} a
|
|
136
|
-
* @param {string} b
|
|
137
|
-
* @returns {number} Similarity in [0, 1].
|
|
138
|
-
*/
|
|
139
|
-
export function jaccardSimilarity(a, b) {
|
|
140
|
-
const wordsA = new Set((a ?? "").toLowerCase().split(/\W+/).filter(Boolean));
|
|
141
|
-
const wordsB = new Set((b ?? "").toLowerCase().split(/\W+/).filter(Boolean));
|
|
142
|
-
if (wordsA.size === 0 && wordsB.size === 0) return 1;
|
|
143
|
-
if (wordsA.size === 0 || wordsB.size === 0) return 0;
|
|
144
|
-
let intersection = 0;
|
|
145
|
-
for (const w of wordsA) if (wordsB.has(w)) intersection++;
|
|
146
|
-
return intersection / (wordsA.size + wordsB.size - intersection);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
/**
|
|
150
|
-
* Maximal Marginal Relevance reranking.
|
|
151
|
-
*
|
|
152
|
-
* Selects up to n candidates that balance relevance to the query and
|
|
153
|
-
* diversity from already-selected results.
|
|
154
|
-
*
|
|
155
|
-
* MMR_score = lambda * querySim(doc) - (1 - lambda) * max(sim(doc, selected))
|
|
156
|
-
*
|
|
157
|
-
* @param {Array<object>} candidates - Entries with at least {id, title, body}.
|
|
158
|
-
* @param {Map<string, number>} querySimMap - Map of id -> relevance score.
|
|
159
|
-
* @param {Map<string, Float32Array|null>} embeddingMap - Map of id -> embedding (null if unavailable).
|
|
160
|
-
* @param {number} n - Number of results to select.
|
|
161
|
-
* @param {number} lambda - Trade-off weight (default MMR_LAMBDA = 0.7).
|
|
162
|
-
* @returns {Array<object>} Reranked subset of candidates (length <= n).
|
|
163
|
-
*/
|
|
164
|
-
export function maximalMarginalRelevance(
|
|
165
|
-
candidates,
|
|
166
|
-
querySimMap,
|
|
167
|
-
embeddingMap,
|
|
168
|
-
n,
|
|
169
|
-
lambda = MMR_LAMBDA,
|
|
170
|
-
) {
|
|
171
|
-
if (candidates.length === 0) return [];
|
|
172
|
-
|
|
173
|
-
const remaining = [...candidates];
|
|
174
|
-
const selected = [];
|
|
175
|
-
const selectedVecs = [];
|
|
176
|
-
const selectedEntries = [];
|
|
177
|
-
|
|
178
|
-
while (selected.length < n && remaining.length > 0) {
|
|
179
|
-
let bestIdx = -1;
|
|
180
|
-
let bestScore = -Infinity;
|
|
181
|
-
|
|
182
|
-
for (let i = 0; i < remaining.length; i++) {
|
|
183
|
-
const candidate = remaining[i];
|
|
184
|
-
const relevance = querySimMap.get(candidate.id) ?? 0;
|
|
185
|
-
|
|
186
|
-
let maxRedundancy = 0;
|
|
187
|
-
if (selectedVecs.length > 0) {
|
|
188
|
-
const vec = embeddingMap.get(candidate.id);
|
|
189
|
-
for (let j = 0; j < selectedVecs.length; j++) {
|
|
190
|
-
let sim;
|
|
191
|
-
if (vec && selectedVecs[j]) {
|
|
192
|
-
sim = dotProduct(vec, selectedVecs[j]);
|
|
193
|
-
} else {
|
|
194
|
-
const selEntry = selectedEntries[j];
|
|
195
|
-
sim = jaccardSimilarity(
|
|
196
|
-
`${candidate.title} ${candidate.body}`,
|
|
197
|
-
`${selEntry.title} ${selEntry.body}`,
|
|
198
|
-
);
|
|
199
|
-
}
|
|
200
|
-
if (sim > maxRedundancy) maxRedundancy = sim;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
const score = lambda * relevance - (1 - lambda) * maxRedundancy;
|
|
205
|
-
if (score > bestScore) {
|
|
206
|
-
bestScore = score;
|
|
207
|
-
bestIdx = i;
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
if (bestIdx === -1) break;
|
|
212
|
-
|
|
213
|
-
const chosen = remaining.splice(bestIdx, 1)[0];
|
|
214
|
-
selected.push(chosen);
|
|
215
|
-
selectedVecs.push(embeddingMap.get(chosen.id) ?? null);
|
|
216
|
-
selectedEntries.push(chosen);
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
return selected;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
133
|
/**
|
|
223
134
|
* Hybrid search combining FTS5 text matching and vector similarity,
|
|
224
|
-
* with RRF merging
|
|
135
|
+
* with RRF merging, recency decay, and near-duplicate suppression.
|
|
225
136
|
*
|
|
226
137
|
* Pipeline:
|
|
227
138
|
* 1. FTS5 ranked list
|
|
228
139
|
* 2. Vector (semantic) ranked list
|
|
229
140
|
* 3. RRF: merge the two ranked lists into a single score
|
|
230
|
-
* 4.
|
|
231
|
-
* 5.
|
|
232
|
-
* 6. Near-duplicate suppression on the final selection
|
|
141
|
+
* 4. Recency decay: penalise old events (knowledge/entity entries unaffected)
|
|
142
|
+
* 5. Near-duplicate suppression (cosine similarity > 0.92 threshold)
|
|
233
143
|
*
|
|
234
144
|
* @param {import('../server/types.js').BaseCtx} ctx
|
|
235
145
|
* @param {string} query
|
|
@@ -242,6 +152,7 @@ export async function hybridSearch(
|
|
|
242
152
|
{
|
|
243
153
|
kindFilter = null,
|
|
244
154
|
categoryFilter = null,
|
|
155
|
+
excludeEvents = false,
|
|
245
156
|
since = null,
|
|
246
157
|
until = null,
|
|
247
158
|
limit = 20,
|
|
@@ -258,6 +169,7 @@ export async function hybridSearch(
|
|
|
258
169
|
|
|
259
170
|
const extraFilters = buildFilterClauses({
|
|
260
171
|
categoryFilter,
|
|
172
|
+
excludeEvents,
|
|
261
173
|
since,
|
|
262
174
|
until,
|
|
263
175
|
userIdFilter,
|
|
@@ -340,6 +252,7 @@ export async function hybridSearch(
|
|
|
340
252
|
if (teamIdFilter && row.team_id !== teamIdFilter) continue;
|
|
341
253
|
if (kindFilter && row.kind !== kindFilter) continue;
|
|
342
254
|
if (categoryFilter && row.category !== categoryFilter) continue;
|
|
255
|
+
if (excludeEvents && row.category === "event") continue;
|
|
343
256
|
if (since && row.created_at < since) continue;
|
|
344
257
|
if (until && row.created_at > until) continue;
|
|
345
258
|
if (row.expires_at && new Date(row.expires_at) <= new Date())
|
|
@@ -376,20 +289,6 @@ export async function hybridSearch(
|
|
|
376
289
|
rrfScores.set(id, (rrfScores.get(id) ?? 0) * boost);
|
|
377
290
|
}
|
|
378
291
|
|
|
379
|
-
// Stage 3b: Frequency signal — log(1 + hit_count) / log(1 + max_hit_count)
|
|
380
|
-
const allRows = [...rowMap.values()];
|
|
381
|
-
const maxHitCount = Math.max(...allRows.map((e) => e.hit_count || 0), 0);
|
|
382
|
-
if (maxHitCount > 0) {
|
|
383
|
-
const logMax = Math.log(1 + maxHitCount);
|
|
384
|
-
for (const entry of allRows) {
|
|
385
|
-
const freqScore = Math.log(1 + (entry.hit_count || 0)) / logMax;
|
|
386
|
-
rrfScores.set(
|
|
387
|
-
entry.id,
|
|
388
|
-
(rrfScores.get(entry.id) ?? 0) + freqScore * 0.13,
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
|
|
393
292
|
// Attach final score to each entry and sort by RRF score descending
|
|
394
293
|
const candidates = [...rowMap.values()].map((entry) => ({
|
|
395
294
|
...entry,
|
|
@@ -397,7 +296,7 @@ export async function hybridSearch(
|
|
|
397
296
|
}));
|
|
398
297
|
candidates.sort((a, b) => b.score - a.score);
|
|
399
298
|
|
|
400
|
-
// Stage 4: Fetch embeddings for
|
|
299
|
+
// Stage 4: Fetch embeddings for near-duplicate suppression
|
|
401
300
|
const embeddingMap = new Map();
|
|
402
301
|
if (queryVec && idToRowid.size > 0) {
|
|
403
302
|
const rowidToId = new Map();
|
|
@@ -422,34 +321,15 @@ export async function hybridSearch(
|
|
|
422
321
|
}
|
|
423
322
|
}
|
|
424
323
|
} catch (_) {
|
|
425
|
-
// Embeddings unavailable —
|
|
324
|
+
// Embeddings unavailable — near-dup suppression skipped
|
|
426
325
|
}
|
|
427
326
|
}
|
|
428
327
|
|
|
429
|
-
//
|
|
430
|
-
|
|
431
|
-
for (const candidate of candidates) {
|
|
432
|
-
querySimMap.set(
|
|
433
|
-
candidate.id,
|
|
434
|
-
vecSimMap.has(candidate.id)
|
|
435
|
-
? vecSimMap.get(candidate.id)
|
|
436
|
-
: candidate.score,
|
|
437
|
-
);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
// Stage 5: MMR — rerank for diversity using embeddings or Jaccard fallback
|
|
441
|
-
const mmrSelected = maximalMarginalRelevance(
|
|
442
|
-
candidates,
|
|
443
|
-
querySimMap,
|
|
444
|
-
embeddingMap,
|
|
445
|
-
offset + limit,
|
|
446
|
-
);
|
|
447
|
-
|
|
448
|
-
// Stage 6: Near-duplicate suppression (hard filter, not reorder)
|
|
449
|
-
if (queryVec && embeddingMap.size > 0 && mmrSelected.length > limit) {
|
|
328
|
+
// Stage 5: Near-duplicate suppression (cosine similarity > 0.92 threshold)
|
|
329
|
+
if (queryVec && embeddingMap.size > 0) {
|
|
450
330
|
const selected = [];
|
|
451
331
|
const selectedVecs = [];
|
|
452
|
-
for (const candidate of
|
|
332
|
+
for (const candidate of candidates) {
|
|
453
333
|
if (selected.length >= offset + limit) break;
|
|
454
334
|
const vec = embeddingMap.get(candidate.id);
|
|
455
335
|
if (vec && selectedVecs.length > 0) {
|
|
@@ -468,7 +348,7 @@ export async function hybridSearch(
|
|
|
468
348
|
return dedupedPage;
|
|
469
349
|
}
|
|
470
350
|
|
|
471
|
-
const finalPage =
|
|
351
|
+
const finalPage = candidates.slice(offset, offset + limit);
|
|
472
352
|
trackAccess(ctx.db, finalPage);
|
|
473
353
|
return finalPage;
|
|
474
354
|
}
|
|
@@ -146,6 +146,13 @@ export function handler(_args, ctx) {
|
|
|
146
146
|
for (const w of growth.warnings) {
|
|
147
147
|
lines.push(` ${w.message}`);
|
|
148
148
|
}
|
|
149
|
+
if (growth.kindBreakdown.length) {
|
|
150
|
+
lines.push("");
|
|
151
|
+
lines.push(" Breakdown by kind:");
|
|
152
|
+
for (const { kind, count, pct } of growth.kindBreakdown) {
|
|
153
|
+
lines.push(` ${kind}: ${count.toLocaleString()} (${pct}%)`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
149
156
|
if (growth.actions.length) {
|
|
150
157
|
lines.push("", "Suggested growth actions:");
|
|
151
158
|
for (const a of growth.actions) {
|
|
@@ -5,14 +5,13 @@ import { normalizeKind } from "../../core/files.js";
|
|
|
5
5
|
import { ok, err, ensureVaultExists } from "../helpers.js";
|
|
6
6
|
|
|
7
7
|
const NOISE_KINDS = new Set(["prompt-history", "task-notification"]);
|
|
8
|
-
const
|
|
9
|
-
const MAX_ENTRIES_FOR_SYNTHESIS = 40;
|
|
8
|
+
const MAX_ENTRIES_FOR_GATHER = 40;
|
|
10
9
|
const MAX_BODY_PER_ENTRY = 600;
|
|
11
10
|
|
|
12
11
|
export const name = "create_snapshot";
|
|
13
12
|
|
|
14
13
|
export const description =
|
|
15
|
-
"Pull all relevant vault entries matching a topic,
|
|
14
|
+
"Pull all relevant vault entries matching a topic, deduplicate, and save them as a structured context brief (kind: 'brief'). Entries are formatted as markdown — no external API or LLM call required. The calling agent can synthesize the gathered content directly. Retrieve with: get_context(kind: 'brief', identity_key: '<key>').";
|
|
16
15
|
|
|
17
16
|
export const inputSchema = {
|
|
18
17
|
topic: z.string().describe("The topic or project name to snapshot"),
|
|
@@ -38,62 +37,42 @@ export const inputSchema = {
|
|
|
38
37
|
),
|
|
39
38
|
};
|
|
40
39
|
|
|
41
|
-
function
|
|
42
|
-
const
|
|
40
|
+
function formatGatheredEntries(topic, entries) {
|
|
41
|
+
const header = [
|
|
42
|
+
`# ${topic} — Context Brief`,
|
|
43
|
+
"",
|
|
44
|
+
`*Gathered from ${entries.length} vault ${entries.length === 1 ? "entry" : "entries"}. Synthesize the content below to extract key decisions, patterns, and constraints.*`,
|
|
45
|
+
"",
|
|
46
|
+
"---",
|
|
47
|
+
"",
|
|
48
|
+
].join("\n");
|
|
49
|
+
|
|
50
|
+
const body = entries
|
|
43
51
|
.map((e, i) => {
|
|
44
52
|
const tags = e.tags ? JSON.parse(e.tags) : [];
|
|
45
53
|
const tagStr = tags.length ? tags.join(", ") : "none";
|
|
46
|
-
const
|
|
54
|
+
const updated = e.updated_at || e.created_at || "unknown";
|
|
55
|
+
const bodyText = e.body
|
|
47
56
|
? e.body.slice(0, MAX_BODY_PER_ENTRY) +
|
|
48
57
|
(e.body.length > MAX_BODY_PER_ENTRY ? "…" : "")
|
|
49
58
|
: "(no body)";
|
|
59
|
+
const title = e.title || `Entry ${i + 1}`;
|
|
50
60
|
return [
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
61
|
+
`## ${i + 1}. [${e.kind}] ${title}`,
|
|
62
|
+
"",
|
|
63
|
+
`**Tags:** ${tagStr}`,
|
|
64
|
+
`**Updated:** ${updated}`,
|
|
65
|
+
`**ID:** \`${e.id}\``,
|
|
66
|
+
"",
|
|
67
|
+
bodyText,
|
|
68
|
+
"",
|
|
69
|
+
"---",
|
|
70
|
+
"",
|
|
55
71
|
].join("\n");
|
|
56
72
|
})
|
|
57
|
-
.join("
|
|
58
|
-
|
|
59
|
-
return `You are a knowledge synthesis assistant. Given the following vault entries about "${topic}", produce a structured context brief.
|
|
60
|
-
|
|
61
|
-
Deduplicate overlapping information, resolve any contradictions (note them in Audit Notes), and organise the content into the sections below. Keep each section concise and actionable. Omit sections that have no relevant content.
|
|
62
|
-
|
|
63
|
-
Output ONLY the markdown document — no preamble, no explanation.
|
|
64
|
-
|
|
65
|
-
Required format:
|
|
66
|
-
# ${topic} — Context Brief
|
|
67
|
-
## Status
|
|
68
|
-
(current state of the topic)
|
|
69
|
-
## Key Decisions
|
|
70
|
-
(architectural or strategic decisions made)
|
|
71
|
-
## Patterns & Conventions
|
|
72
|
-
(recurring patterns, coding conventions, standards)
|
|
73
|
-
## Active Constraints
|
|
74
|
-
(known limitations, hard requirements, deadlines)
|
|
75
|
-
## Open Questions
|
|
76
|
-
(unresolved questions or areas needing investigation)
|
|
77
|
-
## Audit Notes
|
|
78
|
-
(contradictions detected, stale entries flagged with their ids)
|
|
73
|
+
.join("");
|
|
79
74
|
|
|
80
|
-
|
|
81
|
-
VAULT ENTRIES:
|
|
82
|
-
|
|
83
|
-
${entriesBlock}`;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
async function callLlm(prompt) {
|
|
87
|
-
const { Anthropic } = await import("@anthropic-ai/sdk");
|
|
88
|
-
const client = new Anthropic();
|
|
89
|
-
const message = await client.messages.create({
|
|
90
|
-
model: SYNTHESIS_MODEL,
|
|
91
|
-
max_tokens: 2048,
|
|
92
|
-
messages: [{ role: "user", content: prompt }],
|
|
93
|
-
});
|
|
94
|
-
const block = message.content.find((b) => b.type === "text");
|
|
95
|
-
if (!block) throw new Error("LLM returned no text content");
|
|
96
|
-
return block.text;
|
|
75
|
+
return header + body;
|
|
97
76
|
}
|
|
98
77
|
|
|
99
78
|
function slugifyTopic(topic) {
|
|
@@ -122,7 +101,6 @@ export async function handler(
|
|
|
122
101
|
await ensureIndexed();
|
|
123
102
|
|
|
124
103
|
const normalizedKinds = kinds?.map(normalizeKind) ?? [];
|
|
125
|
-
// Expand buckets to bucket: prefixed tags and merge with explicit tags
|
|
126
104
|
const bucketTags = buckets?.length ? buckets.map((b) => `bucket:${b}`) : [];
|
|
127
105
|
const effectiveTags = [...(tags ?? []), ...bucketTags];
|
|
128
106
|
|
|
@@ -132,7 +110,7 @@ export async function handler(
|
|
|
132
110
|
for (const kindFilter of normalizedKinds) {
|
|
133
111
|
const rows = await hybridSearch(ctx, topic, {
|
|
134
112
|
kindFilter,
|
|
135
|
-
limit: Math.ceil(
|
|
113
|
+
limit: Math.ceil(MAX_ENTRIES_FOR_GATHER / normalizedKinds.length),
|
|
136
114
|
userIdFilter: userId,
|
|
137
115
|
includeSuperseeded: false,
|
|
138
116
|
});
|
|
@@ -146,7 +124,7 @@ export async function handler(
|
|
|
146
124
|
});
|
|
147
125
|
} else {
|
|
148
126
|
candidates = await hybridSearch(ctx, topic, {
|
|
149
|
-
limit:
|
|
127
|
+
limit: MAX_ENTRIES_FOR_GATHER,
|
|
150
128
|
userIdFilter: userId,
|
|
151
129
|
includeSuperseeded: false,
|
|
152
130
|
});
|
|
@@ -163,25 +141,16 @@ export async function handler(
|
|
|
163
141
|
.filter((r) => NOISE_KINDS.has(r.kind))
|
|
164
142
|
.map((r) => r.id);
|
|
165
143
|
|
|
166
|
-
const
|
|
144
|
+
const gatherEntries = candidates.filter((r) => !NOISE_KINDS.has(r.kind));
|
|
167
145
|
|
|
168
|
-
if (
|
|
146
|
+
if (gatherEntries.length === 0) {
|
|
169
147
|
return err(
|
|
170
|
-
`No entries found for topic "${topic}"
|
|
148
|
+
`No entries found for topic "${topic}". Try a broader topic or different tags.`,
|
|
171
149
|
"NO_ENTRIES",
|
|
172
150
|
);
|
|
173
151
|
}
|
|
174
152
|
|
|
175
|
-
|
|
176
|
-
try {
|
|
177
|
-
const prompt = buildSynthesisPrompt(topic, synthesisEntries);
|
|
178
|
-
briefBody = await callLlm(prompt);
|
|
179
|
-
} catch (e) {
|
|
180
|
-
return err(
|
|
181
|
-
`LLM synthesis failed: ${e.message}. Ensure ANTHROPIC_API_KEY is set.`,
|
|
182
|
-
"LLM_ERROR",
|
|
183
|
-
);
|
|
184
|
-
}
|
|
153
|
+
const briefBody = formatGatheredEntries(topic, gatherEntries);
|
|
185
154
|
|
|
186
155
|
const effectiveIdentityKey =
|
|
187
156
|
identity_key ?? `snapshot-${slugifyTopic(topic)}`;
|
|
@@ -205,9 +174,9 @@ export async function handler(
|
|
|
205
174
|
userId,
|
|
206
175
|
meta: {
|
|
207
176
|
topic,
|
|
208
|
-
entry_count:
|
|
177
|
+
entry_count: gatherEntries.length,
|
|
209
178
|
noise_superseded: noiseIds.length,
|
|
210
|
-
synthesized_from:
|
|
179
|
+
synthesized_from: gatherEntries.map((e) => e.id),
|
|
211
180
|
},
|
|
212
181
|
});
|
|
213
182
|
|
|
@@ -215,7 +184,7 @@ export async function handler(
|
|
|
215
184
|
`✓ Snapshot created → id: ${entry.id}`,
|
|
216
185
|
` title: ${entry.title}`,
|
|
217
186
|
` identity_key: ${effectiveIdentityKey}`,
|
|
218
|
-
` synthesized from: ${
|
|
187
|
+
` synthesized from: ${gatherEntries.length} entries`,
|
|
219
188
|
noiseIds.length > 0
|
|
220
189
|
? ` noise superseded: ${noiseIds.length} entries`
|
|
221
190
|
: null,
|
|
@@ -5,6 +5,8 @@ import { resolve } from "node:path";
|
|
|
5
5
|
import { hybridSearch } from "../../retrieve/index.js";
|
|
6
6
|
import { categoryFor } from "../../core/categories.js";
|
|
7
7
|
import { normalizeKind } from "../../core/files.js";
|
|
8
|
+
import { resolveTemporalParams } from "../../core/temporal.js";
|
|
9
|
+
import { collectLinkedEntries } from "../../core/linking.js";
|
|
8
10
|
import { ok, err } from "../helpers.js";
|
|
9
11
|
import { isEmbedAvailable } from "../../index/embed.js";
|
|
10
12
|
|
|
@@ -152,8 +154,9 @@ export function detectConsolidationHints(entries, db, userId, opts = {}) {
|
|
|
152
154
|
for (const tag of candidateTags) {
|
|
153
155
|
let vaultCount = 0;
|
|
154
156
|
try {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
+
// When userId is defined (hosted mode), scope to that user.
|
|
158
|
+
// When userId is undefined (local mode), no user scoping — column may not exist.
|
|
159
|
+
const userClause = userId !== undefined ? " AND user_id = ?" : "";
|
|
157
160
|
const countParams =
|
|
158
161
|
userId !== undefined ? [`%"${tag}"%`, userId] : [`%"${tag}"%`];
|
|
159
162
|
const countRow = db
|
|
@@ -170,8 +173,7 @@ export function detectConsolidationHints(entries, db, userId, opts = {}) {
|
|
|
170
173
|
|
|
171
174
|
let lastSnapshotAgeDays = null;
|
|
172
175
|
try {
|
|
173
|
-
const userClause =
|
|
174
|
-
userId !== undefined ? " AND user_id = ?" : " AND user_id IS NULL";
|
|
176
|
+
const userClause = userId !== undefined ? " AND user_id = ?" : "";
|
|
175
177
|
const params =
|
|
176
178
|
userId !== undefined ? [`%"${tag}"%`, userId] : [`%"${tag}"%`];
|
|
177
179
|
const recentBrief = db
|
|
@@ -280,11 +282,15 @@ export const inputSchema = {
|
|
|
280
282
|
since: z
|
|
281
283
|
.string()
|
|
282
284
|
.optional()
|
|
283
|
-
.describe(
|
|
285
|
+
.describe(
|
|
286
|
+
"Return entries created after this date. Accepts ISO date strings (e.g. '2025-01-01') or natural shortcuts: 'today', 'yesterday', 'this_week', 'this_month', 'last_3_days', 'last_2_weeks', 'last_1_month'. Spaces and underscores are interchangeable.",
|
|
287
|
+
),
|
|
284
288
|
until: z
|
|
285
289
|
.string()
|
|
286
290
|
.optional()
|
|
287
|
-
.describe(
|
|
291
|
+
.describe(
|
|
292
|
+
"Return entries created before this date. Accepts ISO date strings or the same natural shortcuts as `since`. When `since` is 'yesterday' and `until` is omitted, `until` is automatically set to the end of yesterday.",
|
|
293
|
+
),
|
|
288
294
|
limit: z.number().optional().describe("Max results to return (default 10)"),
|
|
289
295
|
include_superseded: z
|
|
290
296
|
.boolean()
|
|
@@ -316,6 +322,24 @@ export const inputSchema = {
|
|
|
316
322
|
.describe(
|
|
317
323
|
"If true, include ephemeral tier entries in results. Default: false — only working and durable tiers are returned.",
|
|
318
324
|
),
|
|
325
|
+
include_events: z
|
|
326
|
+
.boolean()
|
|
327
|
+
.optional()
|
|
328
|
+
.describe(
|
|
329
|
+
"If true, include event category entries in semantic search results. Default: false — events are excluded from query-based search but remain accessible via category/tag filters. Deprecated: prefer scope parameter.",
|
|
330
|
+
),
|
|
331
|
+
scope: z
|
|
332
|
+
.enum(["hot", "events", "all"])
|
|
333
|
+
.optional()
|
|
334
|
+
.describe(
|
|
335
|
+
"Index scope: 'hot' (default) — knowledge + entity entries only; 'events' — event entries only (cold index); 'all' — entire vault including events. Overrides include_events when set.",
|
|
336
|
+
),
|
|
337
|
+
follow_links: z
|
|
338
|
+
.boolean()
|
|
339
|
+
.optional()
|
|
340
|
+
.describe(
|
|
341
|
+
"If true, follow related_to links from result entries and include linked entries (forward links) and backlinks (entries that reference the results). Enables bidirectional graph traversal.",
|
|
342
|
+
),
|
|
319
343
|
};
|
|
320
344
|
|
|
321
345
|
/**
|
|
@@ -339,6 +363,9 @@ export async function handler(
|
|
|
339
363
|
max_tokens,
|
|
340
364
|
pivot_count,
|
|
341
365
|
include_ephemeral,
|
|
366
|
+
include_events,
|
|
367
|
+
scope,
|
|
368
|
+
follow_links,
|
|
342
369
|
},
|
|
343
370
|
ctx,
|
|
344
371
|
{ ensureIndexed, reindexFailed },
|
|
@@ -346,12 +373,37 @@ export async function handler(
|
|
|
346
373
|
const { config } = ctx;
|
|
347
374
|
const userId = ctx.userId !== undefined ? ctx.userId : undefined;
|
|
348
375
|
|
|
376
|
+
// Resolve natural-language temporal shortcuts → ISO date strings
|
|
377
|
+
const resolved = resolveTemporalParams({ since, until });
|
|
378
|
+
since = resolved.since;
|
|
379
|
+
until = resolved.until;
|
|
380
|
+
|
|
349
381
|
const hasQuery = query?.trim();
|
|
382
|
+
|
|
383
|
+
// Resolve effective scope — explicit scope param wins over include_events legacy flag.
|
|
384
|
+
// scope "hot" (default): knowledge + entity only — events excluded from search
|
|
385
|
+
// scope "events": force category filter to "event" (cold index query)
|
|
386
|
+
// scope "all": no category restriction — full vault
|
|
387
|
+
let effectiveScope = scope;
|
|
388
|
+
if (!effectiveScope) {
|
|
389
|
+
effectiveScope = include_events ? "all" : "hot";
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// Scope "events" forces category to "event" unless caller already set a narrower category
|
|
393
|
+
const scopedCategory =
|
|
394
|
+
!category && effectiveScope === "events" ? "event" : category;
|
|
395
|
+
const shouldExcludeEvents =
|
|
396
|
+
hasQuery && effectiveScope === "hot" && !scopedCategory;
|
|
350
397
|
// Expand buckets to bucket: prefixed tags and merge with explicit tags
|
|
351
398
|
const bucketTags = buckets?.length ? buckets.map((b) => `bucket:${b}`) : [];
|
|
352
399
|
const effectiveTags = [...(tags ?? []), ...bucketTags];
|
|
353
400
|
const hasFilters =
|
|
354
|
-
kind ||
|
|
401
|
+
kind ||
|
|
402
|
+
scopedCategory ||
|
|
403
|
+
effectiveTags.length ||
|
|
404
|
+
since ||
|
|
405
|
+
until ||
|
|
406
|
+
identity_key;
|
|
355
407
|
if (!hasQuery && !hasFilters)
|
|
356
408
|
return err(
|
|
357
409
|
"Required: query or at least one filter (kind, category, tags, since, until, identity_key)",
|
|
@@ -365,11 +417,16 @@ export async function handler(
|
|
|
365
417
|
if (identity_key) {
|
|
366
418
|
if (!kindFilter)
|
|
367
419
|
return err("identity_key requires kind to be specified", "INVALID_INPUT");
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
420
|
+
// Local mode: getByIdentityKey takes 2 params (no user_id).
|
|
421
|
+
// Hosted mode: 3 params — (kind, identity_key, userId).
|
|
422
|
+
const match =
|
|
423
|
+
ctx.stmts._mode === "local"
|
|
424
|
+
? ctx.stmts.getByIdentityKey.get(kindFilter, identity_key)
|
|
425
|
+
: ctx.stmts.getByIdentityKey.get(
|
|
426
|
+
kindFilter,
|
|
427
|
+
identity_key,
|
|
428
|
+
userId !== undefined ? userId : null,
|
|
429
|
+
);
|
|
373
430
|
if (match) {
|
|
374
431
|
const entryTags = match.tags ? JSON.parse(match.tags) : [];
|
|
375
432
|
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
@@ -390,7 +447,7 @@ export async function handler(
|
|
|
390
447
|
|
|
391
448
|
// Gap 2: Event default time-window
|
|
392
449
|
const effectiveCategory =
|
|
393
|
-
|
|
450
|
+
scopedCategory || (kindFilter ? categoryFor(kindFilter) : null);
|
|
394
451
|
let effectiveSince = since || null;
|
|
395
452
|
let effectiveUntil = until || null;
|
|
396
453
|
let autoWindowed = false;
|
|
@@ -412,7 +469,8 @@ export async function handler(
|
|
|
412
469
|
// Hybrid search mode
|
|
413
470
|
const sorted = await hybridSearch(ctx, query, {
|
|
414
471
|
kindFilter,
|
|
415
|
-
categoryFilter:
|
|
472
|
+
categoryFilter: scopedCategory || null,
|
|
473
|
+
excludeEvents: shouldExcludeEvents,
|
|
416
474
|
since: effectiveSince,
|
|
417
475
|
until: effectiveUntil,
|
|
418
476
|
limit: fetchLimit,
|
|
@@ -442,9 +500,9 @@ export async function handler(
|
|
|
442
500
|
clauses.push("kind = ?");
|
|
443
501
|
params.push(kindFilter);
|
|
444
502
|
}
|
|
445
|
-
if (
|
|
503
|
+
if (scopedCategory) {
|
|
446
504
|
clauses.push("category = ?");
|
|
447
|
-
params.push(
|
|
505
|
+
params.push(scopedCategory);
|
|
448
506
|
}
|
|
449
507
|
if (effectiveSince) {
|
|
450
508
|
clauses.push("created_at >= ?");
|
|
@@ -490,6 +548,11 @@ export async function handler(
|
|
|
490
548
|
filtered = filtered.filter((r) => r.tier !== "ephemeral");
|
|
491
549
|
}
|
|
492
550
|
|
|
551
|
+
// Event category filter: exclude events from semantic search by default
|
|
552
|
+
if (shouldExcludeEvents) {
|
|
553
|
+
filtered = filtered.filter((r) => r.category !== "event");
|
|
554
|
+
}
|
|
555
|
+
|
|
493
556
|
if (!filtered.length) {
|
|
494
557
|
if (autoWindowed) {
|
|
495
558
|
const days = config.eventDecayDays || 30;
|
|
@@ -618,6 +681,45 @@ export async function handler(
|
|
|
618
681
|
}
|
|
619
682
|
}
|
|
620
683
|
|
|
684
|
+
// Graph traversal: follow related_to links bidirectionally
|
|
685
|
+
if (follow_links) {
|
|
686
|
+
const { forward, backward } = collectLinkedEntries(
|
|
687
|
+
ctx.db,
|
|
688
|
+
filtered,
|
|
689
|
+
userId,
|
|
690
|
+
);
|
|
691
|
+
const allLinked = [...forward, ...backward];
|
|
692
|
+
const seen = new Set();
|
|
693
|
+
const uniqueLinked = allLinked.filter((e) => {
|
|
694
|
+
if (seen.has(e.id)) return false;
|
|
695
|
+
seen.add(e.id);
|
|
696
|
+
return true;
|
|
697
|
+
});
|
|
698
|
+
|
|
699
|
+
if (uniqueLinked.length > 0) {
|
|
700
|
+
lines.push(`## Linked Entries (${uniqueLinked.length} via related_to)\n`);
|
|
701
|
+
for (const r of uniqueLinked) {
|
|
702
|
+
const direction = forward.some((f) => f.id === r.id)
|
|
703
|
+
? "→ forward"
|
|
704
|
+
: "← backlink";
|
|
705
|
+
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
706
|
+
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
707
|
+
const relPath =
|
|
708
|
+
r.file_path && config.vaultDir
|
|
709
|
+
? r.file_path.replace(config.vaultDir + "/", "")
|
|
710
|
+
: r.file_path || "n/a";
|
|
711
|
+
lines.push(
|
|
712
|
+
`### ${r.title || "(untitled)"} [${r.kind}/${r.category}] ${direction}`,
|
|
713
|
+
);
|
|
714
|
+
lines.push(`${tagStr} · ${relPath} · id: \`${r.id}\``);
|
|
715
|
+
lines.push(r.body?.slice(0, 200) + (r.body?.length > 200 ? "..." : ""));
|
|
716
|
+
lines.push("");
|
|
717
|
+
}
|
|
718
|
+
} else {
|
|
719
|
+
lines.push(`## Linked Entries\n\nNo related entries found.\n`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
621
723
|
// Consolidation suggestion detection — lazy, opportunistic, vault-wide
|
|
622
724
|
const consolidationOpts = {
|
|
623
725
|
tagThreshold:
|
|
@@ -647,6 +749,7 @@ export async function handler(
|
|
|
647
749
|
|
|
648
750
|
const result = ok(lines.join("\n"));
|
|
649
751
|
const meta = {};
|
|
752
|
+
meta.scope = effectiveScope;
|
|
650
753
|
if (tokensBudget != null) {
|
|
651
754
|
meta.tokens_used = tokensUsed;
|
|
652
755
|
meta.tokens_budget = tokensBudget;
|
|
@@ -657,8 +760,6 @@ export async function handler(
|
|
|
657
760
|
if (consolidationSuggestions.length > 0) {
|
|
658
761
|
meta.consolidation_suggestions = consolidationSuggestions;
|
|
659
762
|
}
|
|
660
|
-
|
|
661
|
-
result._meta = meta;
|
|
662
|
-
}
|
|
763
|
+
result._meta = meta;
|
|
663
764
|
return result;
|
|
664
765
|
}
|