@context-vault/core 2.17.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capture.d.ts +21 -0
- package/dist/capture.d.ts.map +1 -0
- package/dist/capture.js +269 -0
- package/dist/capture.js.map +1 -0
- package/dist/categories.d.ts +6 -0
- package/dist/categories.d.ts.map +1 -0
- package/dist/categories.js +50 -0
- package/dist/categories.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +190 -0
- package/dist/config.js.map +1 -0
- package/dist/constants.d.ts +33 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +23 -0
- package/dist/constants.js.map +1 -0
- package/dist/db.d.ts +13 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +191 -0
- package/dist/db.js.map +1 -0
- package/dist/embed.d.ts +5 -0
- package/dist/embed.d.ts.map +1 -0
- package/dist/embed.js +78 -0
- package/dist/embed.js.map +1 -0
- package/dist/files.d.ts +13 -0
- package/dist/files.d.ts.map +1 -0
- package/dist/files.js +66 -0
- package/dist/files.js.map +1 -0
- package/dist/formatters.d.ts +8 -0
- package/dist/formatters.d.ts.map +1 -0
- package/dist/formatters.js +18 -0
- package/dist/formatters.js.map +1 -0
- package/dist/frontmatter.d.ts +12 -0
- package/dist/frontmatter.d.ts.map +1 -0
- package/dist/frontmatter.js +101 -0
- package/dist/frontmatter.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +297 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest-url.d.ts +20 -0
- package/dist/ingest-url.d.ts.map +1 -0
- package/dist/ingest-url.js +113 -0
- package/dist/ingest-url.js.map +1 -0
- package/dist/main.d.ts +14 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +25 -0
- package/dist/main.js.map +1 -0
- package/dist/search.d.ts +18 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +238 -0
- package/dist/search.js.map +1 -0
- package/dist/types.d.ts +176 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +66 -16
- package/src/capture.ts +308 -0
- package/src/categories.ts +54 -0
- package/src/{core/config.js → config.ts} +34 -33
- package/src/{constants.js → constants.ts} +6 -3
- package/src/db.ts +229 -0
- package/src/{index/embed.js → embed.ts} +10 -35
- package/src/{core/files.js → files.ts} +15 -20
- package/src/{capture/formatters.js → formatters.ts} +13 -11
- package/src/{core/frontmatter.js → frontmatter.ts} +26 -33
- package/src/index.ts +351 -0
- package/src/ingest-url.ts +99 -0
- package/src/main.ts +111 -0
- package/src/{retrieve/index.js → search.ts} +62 -150
- package/src/types.ts +166 -0
- package/src/capture/file-ops.js +0 -99
- package/src/capture/import-pipeline.js +0 -46
- package/src/capture/importers.js +0 -387
- package/src/capture/index.js +0 -250
- package/src/capture/ingest-url.js +0 -252
- package/src/consolidation/index.js +0 -112
- package/src/core/categories.js +0 -73
- package/src/core/error-log.js +0 -54
- package/src/core/linking.js +0 -161
- package/src/core/migrate-dirs.js +0 -196
- package/src/core/status.js +0 -350
- package/src/core/telemetry.js +0 -90
- package/src/core/temporal.js +0 -146
- package/src/index/db.js +0 -586
- package/src/index/index.js +0 -583
- package/src/index.js +0 -71
- package/src/server/helpers.js +0 -44
- package/src/server/tools/clear-context.js +0 -47
- package/src/server/tools/context-status.js +0 -182
- package/src/server/tools/create-snapshot.js +0 -200
- package/src/server/tools/delete-context.js +0 -60
- package/src/server/tools/get-context.js +0 -765
- package/src/server/tools/ingest-project.js +0 -244
- package/src/server/tools/ingest-url.js +0 -88
- package/src/server/tools/list-buckets.js +0 -116
- package/src/server/tools/list-context.js +0 -163
- package/src/server/tools/save-context.js +0 -632
- package/src/server/tools/session-start.js +0 -285
- package/src/server/tools.js +0 -172
- package/src/sync/sync.js +0 -235
|
@@ -1,765 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
|
-
import { readFileSync, existsSync } from "node:fs";
|
|
4
|
-
import { resolve } from "node:path";
|
|
5
|
-
import { hybridSearch } from "../../retrieve/index.js";
|
|
6
|
-
import { categoryFor } from "../../core/categories.js";
|
|
7
|
-
import { normalizeKind } from "../../core/files.js";
|
|
8
|
-
import { resolveTemporalParams } from "../../core/temporal.js";
|
|
9
|
-
import { collectLinkedEntries } from "../../core/linking.js";
|
|
10
|
-
import { ok, err } from "../helpers.js";
|
|
11
|
-
import { isEmbedAvailable } from "../../index/embed.js";
|
|
12
|
-
|
|
13
|
-
const STALE_DUPLICATE_DAYS = 7;
|
|
14
|
-
const DEFAULT_PIVOT_COUNT = 2;
|
|
15
|
-
const SKELETON_BODY_CHARS = 100;
|
|
16
|
-
const CONSOLIDATION_TAG_THRESHOLD = 10;
|
|
17
|
-
const CONSOLIDATION_SNAPSHOT_MAX_AGE_DAYS = 7;
|
|
18
|
-
const BRIEF_SCORE_BOOST = 0.05;
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Truncate a body string to ~SKELETON_BODY_CHARS, breaking at sentence or
|
|
22
|
-
* word boundary. Returns the truncated string with "..." appended.
|
|
23
|
-
*/
|
|
24
|
-
export function skeletonBody(body) {
|
|
25
|
-
if (!body) return "";
|
|
26
|
-
if (body.length <= SKELETON_BODY_CHARS) return body;
|
|
27
|
-
const slice = body.slice(0, SKELETON_BODY_CHARS);
|
|
28
|
-
const sentenceEnd = Math.max(
|
|
29
|
-
slice.lastIndexOf(". "),
|
|
30
|
-
slice.lastIndexOf(".\n"),
|
|
31
|
-
);
|
|
32
|
-
if (sentenceEnd > SKELETON_BODY_CHARS * 0.4) {
|
|
33
|
-
return slice.slice(0, sentenceEnd + 1) + "...";
|
|
34
|
-
}
|
|
35
|
-
const wordEnd = slice.lastIndexOf(" ");
|
|
36
|
-
if (wordEnd > SKELETON_BODY_CHARS * 0.4) {
|
|
37
|
-
return slice.slice(0, wordEnd) + "...";
|
|
38
|
-
}
|
|
39
|
-
return slice + "...";
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Detect conflicts among a set of search result entries.
|
|
44
|
-
*
|
|
45
|
-
* Two checks are performed:
|
|
46
|
-
* 1. Supersession: if entry A's `superseded_by` points to any entry B in the
|
|
47
|
-
* result set, A is stale and should be discarded in favour of B.
|
|
48
|
-
* 2. Stale duplicate: two entries share the same kind and at least one common
|
|
49
|
-
* tag, but their `updated_at` timestamps differ by more than
|
|
50
|
-
* STALE_DUPLICATE_DAYS days — suggesting the older one may be outdated.
|
|
51
|
-
*
|
|
52
|
-
* No LLM calls, no new dependencies — pure in-memory set operations on the
|
|
53
|
-
* rows already fetched from the DB.
|
|
54
|
-
*
|
|
55
|
-
* @param {Array} entries - Result rows (as returned by hybridSearch / filter-only mode)
|
|
56
|
-
* @param {import('../types.js').BaseCtx} _ctx - Unused for now; reserved for future DB look-ups
|
|
57
|
-
* @returns {Array<{entry_a_id: string, entry_b_id: string, reason: string, recommendation: string}>}
|
|
58
|
-
*/
|
|
59
|
-
export function detectConflicts(entries, _ctx) {
|
|
60
|
-
const conflicts = [];
|
|
61
|
-
const idSet = new Set(entries.map((e) => e.id));
|
|
62
|
-
|
|
63
|
-
for (const entry of entries) {
|
|
64
|
-
if (entry.superseded_by && idSet.has(entry.superseded_by)) {
|
|
65
|
-
conflicts.push({
|
|
66
|
-
entry_a_id: entry.id,
|
|
67
|
-
entry_b_id: entry.superseded_by,
|
|
68
|
-
reason: "superseded",
|
|
69
|
-
recommendation: `Discard \`${entry.id}\` — it has been explicitly superseded by \`${entry.superseded_by}\`.`,
|
|
70
|
-
});
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
const supersededConflictPairs = new Set(
|
|
75
|
-
conflicts.map((c) => `${c.entry_a_id}|${c.entry_b_id}`),
|
|
76
|
-
);
|
|
77
|
-
|
|
78
|
-
for (let i = 0; i < entries.length; i++) {
|
|
79
|
-
for (let j = i + 1; j < entries.length; j++) {
|
|
80
|
-
const a = entries[i];
|
|
81
|
-
const b = entries[j];
|
|
82
|
-
|
|
83
|
-
if (
|
|
84
|
-
supersededConflictPairs.has(`${a.id}|${b.id}`) ||
|
|
85
|
-
supersededConflictPairs.has(`${b.id}|${a.id}`)
|
|
86
|
-
) {
|
|
87
|
-
continue;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
if (a.kind !== b.kind) continue;
|
|
91
|
-
|
|
92
|
-
const tagsA = a.tags ? JSON.parse(a.tags) : [];
|
|
93
|
-
const tagsB = b.tags ? JSON.parse(b.tags) : [];
|
|
94
|
-
|
|
95
|
-
if (!tagsA.length || !tagsB.length) continue;
|
|
96
|
-
|
|
97
|
-
const tagsSetA = new Set(tagsA);
|
|
98
|
-
const sharedTag = tagsB.some((t) => tagsSetA.has(t));
|
|
99
|
-
if (!sharedTag) continue;
|
|
100
|
-
|
|
101
|
-
const dateA = new Date(a.updated_at || a.created_at);
|
|
102
|
-
const dateB = new Date(b.updated_at || b.created_at);
|
|
103
|
-
if (isNaN(dateA.getTime()) || isNaN(dateB.getTime())) continue;
|
|
104
|
-
|
|
105
|
-
const diffDays = Math.abs(dateA - dateB) / 86400000;
|
|
106
|
-
if (diffDays <= STALE_DUPLICATE_DAYS) continue;
|
|
107
|
-
|
|
108
|
-
const [older, newer] = dateA < dateB ? [a, b] : [b, a];
|
|
109
|
-
conflicts.push({
|
|
110
|
-
entry_a_id: older.id,
|
|
111
|
-
entry_b_id: newer.id,
|
|
112
|
-
reason: "stale_duplicate",
|
|
113
|
-
recommendation: `Verify \`${older.id}\` is still accurate — it shares kind "${older.kind}" and tags with \`${newer.id}\` but was last updated ${Math.round(diffDays)} days earlier.`,
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
return conflicts;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Detect tag clusters that would benefit from consolidation via create_snapshot.
|
|
123
|
-
* A suggestion is emitted when a tag appears on threshold+ entries in the full
|
|
124
|
-
* vault AND no recent brief (kind='brief') exists for that tag within the
|
|
125
|
-
* staleness window.
|
|
126
|
-
*
|
|
127
|
-
* Tag counts are derived from the full vault (not just the search result set)
|
|
128
|
-
* so the check reflects the true size of the knowledge cluster. Only tags that
|
|
129
|
-
* appear in the current search results are evaluated — this keeps the check
|
|
130
|
-
* targeted to what the user is actually working with.
|
|
131
|
-
*
|
|
132
|
-
* @param {Array} entries - Search result rows (used to select candidate tags)
|
|
133
|
-
* @param {import('node:sqlite').DatabaseSync} db - Database handle for vault-wide counts and brief lookups
|
|
134
|
-
* @param {number|undefined} userId - Optional user_id scope
|
|
135
|
-
* @param {{ tagThreshold?: number, maxAgeDays?: number }} opts - Configurable thresholds
|
|
136
|
-
* @returns {Array<{tag: string, entry_count: number, last_snapshot_age_days: number|null}>}
|
|
137
|
-
*/
|
|
138
|
-
export function detectConsolidationHints(entries, db, userId, opts = {}) {
|
|
139
|
-
const tagThreshold = opts.tagThreshold ?? CONSOLIDATION_TAG_THRESHOLD;
|
|
140
|
-
const maxAgeDays = opts.maxAgeDays ?? CONSOLIDATION_SNAPSHOT_MAX_AGE_DAYS;
|
|
141
|
-
|
|
142
|
-
const candidateTags = new Set();
|
|
143
|
-
for (const entry of entries) {
|
|
144
|
-
if (entry.kind === "brief") continue;
|
|
145
|
-
const entryTags = entry.tags ? JSON.parse(entry.tags) : [];
|
|
146
|
-
for (const tag of entryTags) candidateTags.add(tag);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
if (candidateTags.size === 0) return [];
|
|
150
|
-
|
|
151
|
-
const suggestions = [];
|
|
152
|
-
const cutoff = new Date(Date.now() - maxAgeDays * 86400000).toISOString();
|
|
153
|
-
|
|
154
|
-
for (const tag of candidateTags) {
|
|
155
|
-
let vaultCount = 0;
|
|
156
|
-
try {
|
|
157
|
-
// When userId is defined (hosted mode), scope to that user.
|
|
158
|
-
// When userId is undefined (local mode), no user scoping — column may not exist.
|
|
159
|
-
const userClause = userId !== undefined ? " AND user_id = ?" : "";
|
|
160
|
-
const countParams =
|
|
161
|
-
userId !== undefined ? [`%"${tag}"%`, userId] : [`%"${tag}"%`];
|
|
162
|
-
const countRow = db
|
|
163
|
-
.prepare(
|
|
164
|
-
`SELECT COUNT(*) as c FROM vault WHERE kind != 'brief' AND tags LIKE ?${userClause} AND (expires_at IS NULL OR expires_at > datetime('now')) AND superseded_by IS NULL`,
|
|
165
|
-
)
|
|
166
|
-
.get(...countParams);
|
|
167
|
-
vaultCount = countRow?.c ?? 0;
|
|
168
|
-
} catch {
|
|
169
|
-
continue;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
if (vaultCount < tagThreshold) continue;
|
|
173
|
-
|
|
174
|
-
let lastSnapshotAgeDays = null;
|
|
175
|
-
try {
|
|
176
|
-
const userClause = userId !== undefined ? " AND user_id = ?" : "";
|
|
177
|
-
const params =
|
|
178
|
-
userId !== undefined ? [`%"${tag}"%`, userId] : [`%"${tag}"%`];
|
|
179
|
-
const recentBrief = db
|
|
180
|
-
.prepare(
|
|
181
|
-
`SELECT created_at FROM vault WHERE kind = 'brief' AND tags LIKE ?${userClause} ORDER BY created_at DESC LIMIT 1`,
|
|
182
|
-
)
|
|
183
|
-
.get(...params);
|
|
184
|
-
|
|
185
|
-
if (recentBrief) {
|
|
186
|
-
lastSnapshotAgeDays = Math.round(
|
|
187
|
-
(Date.now() - new Date(recentBrief.created_at).getTime()) / 86400000,
|
|
188
|
-
);
|
|
189
|
-
if (recentBrief.created_at >= cutoff) continue;
|
|
190
|
-
}
|
|
191
|
-
} catch {
|
|
192
|
-
continue;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
suggestions.push({
|
|
196
|
-
tag,
|
|
197
|
-
entry_count: vaultCount,
|
|
198
|
-
last_snapshot_age_days: lastSnapshotAgeDays,
|
|
199
|
-
});
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
return suggestions;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
/**
|
|
206
|
-
* Check if an entry's source files have changed since the entry was saved.
|
|
207
|
-
* Returns { stale: true, stale_reason } if stale, or null if fresh.
|
|
208
|
-
* Best-effort: any read/parse failure returns null (no crash).
|
|
209
|
-
*
|
|
210
|
-
* @param {object} entry - DB row with source_files JSON column
|
|
211
|
-
* @returns {{ stale: boolean, stale_reason: string } | null}
|
|
212
|
-
*/
|
|
213
|
-
function checkStaleness(entry) {
|
|
214
|
-
if (!entry.source_files) return null;
|
|
215
|
-
let sourceFiles;
|
|
216
|
-
try {
|
|
217
|
-
sourceFiles = JSON.parse(entry.source_files);
|
|
218
|
-
} catch {
|
|
219
|
-
return null;
|
|
220
|
-
}
|
|
221
|
-
if (!Array.isArray(sourceFiles) || sourceFiles.length === 0) return null;
|
|
222
|
-
|
|
223
|
-
for (const sf of sourceFiles) {
|
|
224
|
-
try {
|
|
225
|
-
const absPath = sf.path.startsWith("/")
|
|
226
|
-
? sf.path
|
|
227
|
-
: resolve(process.cwd(), sf.path);
|
|
228
|
-
if (!existsSync(absPath)) {
|
|
229
|
-
return { stale: true, stale_reason: "source file not found" };
|
|
230
|
-
}
|
|
231
|
-
const contents = readFileSync(absPath);
|
|
232
|
-
const currentHash = createHash("sha256").update(contents).digest("hex");
|
|
233
|
-
if (currentHash !== sf.hash) {
|
|
234
|
-
return {
|
|
235
|
-
stale: true,
|
|
236
|
-
stale_reason: "source file modified since observation",
|
|
237
|
-
};
|
|
238
|
-
}
|
|
239
|
-
} catch {
|
|
240
|
-
// skip this file on any error — best-effort
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
return null;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
export const name = "get_context";
|
|
247
|
-
|
|
248
|
-
export const description =
|
|
249
|
-
"Search your knowledge vault. Returns entries ranked by relevance using hybrid full-text + semantic search. Use this to find insights, decisions, patterns, or any saved context. Each result includes an `id` you can use with save_context or delete_context.";
|
|
250
|
-
|
|
251
|
-
export const inputSchema = {
|
|
252
|
-
query: z
|
|
253
|
-
.string()
|
|
254
|
-
.optional()
|
|
255
|
-
.describe(
|
|
256
|
-
"Search query (natural language or keywords). Optional if filters (tags, kind, category) are provided.",
|
|
257
|
-
),
|
|
258
|
-
kind: z
|
|
259
|
-
.string()
|
|
260
|
-
.optional()
|
|
261
|
-
.describe("Filter by kind (e.g. 'insight', 'decision', 'pattern')"),
|
|
262
|
-
category: z
|
|
263
|
-
.enum(["knowledge", "entity", "event"])
|
|
264
|
-
.optional()
|
|
265
|
-
.describe("Filter by category"),
|
|
266
|
-
identity_key: z
|
|
267
|
-
.string()
|
|
268
|
-
.optional()
|
|
269
|
-
.describe("For entity lookup: exact match on identity key. Requires kind."),
|
|
270
|
-
tags: z
|
|
271
|
-
.array(z.string())
|
|
272
|
-
.optional()
|
|
273
|
-
.describe(
|
|
274
|
-
"Filter by tags (entries must match at least one). Use 'bucket:' prefixed tags for project-scoped retrieval (e.g., ['bucket:autohub']).",
|
|
275
|
-
),
|
|
276
|
-
buckets: z
|
|
277
|
-
.array(z.string())
|
|
278
|
-
.optional()
|
|
279
|
-
.describe(
|
|
280
|
-
"Filter by project-scoped buckets. Each name expands to a 'bucket:<name>' tag. Composes with 'tags' via OR (entries matching any tag or any bucket are included).",
|
|
281
|
-
),
|
|
282
|
-
since: z
|
|
283
|
-
.string()
|
|
284
|
-
.optional()
|
|
285
|
-
.describe(
|
|
286
|
-
"Return entries created after this date. Accepts ISO date strings (e.g. '2025-01-01') or natural shortcuts: 'today', 'yesterday', 'this_week', 'this_month', 'last_3_days', 'last_2_weeks', 'last_1_month'. Spaces and underscores are interchangeable.",
|
|
287
|
-
),
|
|
288
|
-
until: z
|
|
289
|
-
.string()
|
|
290
|
-
.optional()
|
|
291
|
-
.describe(
|
|
292
|
-
"Return entries created before this date. Accepts ISO date strings or the same natural shortcuts as `since`. When `since` is 'yesterday' and `until` is omitted, `until` is automatically set to the end of yesterday.",
|
|
293
|
-
),
|
|
294
|
-
limit: z.number().optional().describe("Max results to return (default 10)"),
|
|
295
|
-
include_superseded: z
|
|
296
|
-
.boolean()
|
|
297
|
-
.optional()
|
|
298
|
-
.describe(
|
|
299
|
-
"If true, include entries that have been superseded by newer ones. Default: false.",
|
|
300
|
-
),
|
|
301
|
-
detect_conflicts: z
|
|
302
|
-
.boolean()
|
|
303
|
-
.optional()
|
|
304
|
-
.describe(
|
|
305
|
-
"If true, compare results for contradicting entries and append a conflicts array. Flags superseded entries still in results and stale duplicates (same kind+tags, updated_at >7 days apart). No LLM calls — pure DB logic.",
|
|
306
|
-
),
|
|
307
|
-
max_tokens: z
|
|
308
|
-
.number()
|
|
309
|
-
.optional()
|
|
310
|
-
.describe(
|
|
311
|
-
"Limit output to entries that fit within this token budget (rough estimate: 1 token ≈ 4 chars). Entries are packed greedily by relevance rank. At least 1 result is always returned. Response metadata includes tokens_used and tokens_budget.",
|
|
312
|
-
),
|
|
313
|
-
pivot_count: z
|
|
314
|
-
.number()
|
|
315
|
-
.optional()
|
|
316
|
-
.describe(
|
|
317
|
-
"Skeleton mode: top pivot_count entries by relevance are returned with full body. Remaining entries are returned as skeletons (title + tags + first ~100 chars of body). Default: 2. Set to 0 to skeleton all results, or a high number to disable.",
|
|
318
|
-
),
|
|
319
|
-
include_ephemeral: z
|
|
320
|
-
.boolean()
|
|
321
|
-
.optional()
|
|
322
|
-
.describe(
|
|
323
|
-
"If true, include ephemeral tier entries in results. Default: false — only working and durable tiers are returned.",
|
|
324
|
-
),
|
|
325
|
-
include_events: z
|
|
326
|
-
.boolean()
|
|
327
|
-
.optional()
|
|
328
|
-
.describe(
|
|
329
|
-
"If true, include event category entries in semantic search results. Default: false — events are excluded from query-based search but remain accessible via category/tag filters. Deprecated: prefer scope parameter.",
|
|
330
|
-
),
|
|
331
|
-
scope: z
|
|
332
|
-
.enum(["hot", "events", "all"])
|
|
333
|
-
.optional()
|
|
334
|
-
.describe(
|
|
335
|
-
"Index scope: 'hot' (default) — knowledge + entity entries only; 'events' — event entries only (cold index); 'all' — entire vault including events. Overrides include_events when set.",
|
|
336
|
-
),
|
|
337
|
-
follow_links: z
|
|
338
|
-
.boolean()
|
|
339
|
-
.optional()
|
|
340
|
-
.describe(
|
|
341
|
-
"If true, follow related_to links from result entries and include linked entries (forward links) and backlinks (entries that reference the results). Enables bidirectional graph traversal.",
|
|
342
|
-
),
|
|
343
|
-
};
|
|
344
|
-
|
|
345
|
-
/**
|
|
346
|
-
* @param {object} args
|
|
347
|
-
* @param {import('../types.js').BaseCtx & Partial<import('../types.js').HostedCtxExtensions>} ctx
|
|
348
|
-
* @param {import('../types.js').ToolShared} shared
|
|
349
|
-
*/
|
|
350
|
-
export async function handler(
|
|
351
|
-
{
|
|
352
|
-
query,
|
|
353
|
-
kind,
|
|
354
|
-
category,
|
|
355
|
-
identity_key,
|
|
356
|
-
tags,
|
|
357
|
-
buckets,
|
|
358
|
-
since,
|
|
359
|
-
until,
|
|
360
|
-
limit,
|
|
361
|
-
include_superseded,
|
|
362
|
-
detect_conflicts,
|
|
363
|
-
max_tokens,
|
|
364
|
-
pivot_count,
|
|
365
|
-
include_ephemeral,
|
|
366
|
-
include_events,
|
|
367
|
-
scope,
|
|
368
|
-
follow_links,
|
|
369
|
-
},
|
|
370
|
-
ctx,
|
|
371
|
-
{ ensureIndexed, reindexFailed },
|
|
372
|
-
) {
|
|
373
|
-
const { config } = ctx;
|
|
374
|
-
const userId = ctx.userId !== undefined ? ctx.userId : undefined;
|
|
375
|
-
|
|
376
|
-
// Resolve natural-language temporal shortcuts → ISO date strings
|
|
377
|
-
const resolved = resolveTemporalParams({ since, until });
|
|
378
|
-
since = resolved.since;
|
|
379
|
-
until = resolved.until;
|
|
380
|
-
|
|
381
|
-
const hasQuery = query?.trim();
|
|
382
|
-
|
|
383
|
-
// Resolve effective scope — explicit scope param wins over include_events legacy flag.
|
|
384
|
-
// scope "hot" (default): knowledge + entity only — events excluded from search
|
|
385
|
-
// scope "events": force category filter to "event" (cold index query)
|
|
386
|
-
// scope "all": no category restriction — full vault
|
|
387
|
-
let effectiveScope = scope;
|
|
388
|
-
if (!effectiveScope) {
|
|
389
|
-
effectiveScope = include_events ? "all" : "hot";
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
// Scope "events" forces category to "event" unless caller already set a narrower category
|
|
393
|
-
const scopedCategory =
|
|
394
|
-
!category && effectiveScope === "events" ? "event" : category;
|
|
395
|
-
const shouldExcludeEvents =
|
|
396
|
-
hasQuery && effectiveScope === "hot" && !scopedCategory;
|
|
397
|
-
// Expand buckets to bucket: prefixed tags and merge with explicit tags
|
|
398
|
-
const bucketTags = buckets?.length ? buckets.map((b) => `bucket:${b}`) : [];
|
|
399
|
-
const effectiveTags = [...(tags ?? []), ...bucketTags];
|
|
400
|
-
const hasFilters =
|
|
401
|
-
kind ||
|
|
402
|
-
scopedCategory ||
|
|
403
|
-
effectiveTags.length ||
|
|
404
|
-
since ||
|
|
405
|
-
until ||
|
|
406
|
-
identity_key;
|
|
407
|
-
if (!hasQuery && !hasFilters)
|
|
408
|
-
return err(
|
|
409
|
-
"Required: query or at least one filter (kind, category, tags, since, until, identity_key)",
|
|
410
|
-
"INVALID_INPUT",
|
|
411
|
-
);
|
|
412
|
-
await ensureIndexed();
|
|
413
|
-
|
|
414
|
-
const kindFilter = kind ? normalizeKind(kind) : null;
|
|
415
|
-
|
|
416
|
-
// Gap 1: Entity exact-match by identity_key
|
|
417
|
-
if (identity_key) {
|
|
418
|
-
if (!kindFilter)
|
|
419
|
-
return err("identity_key requires kind to be specified", "INVALID_INPUT");
|
|
420
|
-
// Local mode: getByIdentityKey takes 2 params (no user_id).
|
|
421
|
-
// Hosted mode: 3 params — (kind, identity_key, userId).
|
|
422
|
-
const match =
|
|
423
|
-
ctx.stmts._mode === "local"
|
|
424
|
-
? ctx.stmts.getByIdentityKey.get(kindFilter, identity_key)
|
|
425
|
-
: ctx.stmts.getByIdentityKey.get(
|
|
426
|
-
kindFilter,
|
|
427
|
-
identity_key,
|
|
428
|
-
userId !== undefined ? userId : null,
|
|
429
|
-
);
|
|
430
|
-
if (match) {
|
|
431
|
-
const entryTags = match.tags ? JSON.parse(match.tags) : [];
|
|
432
|
-
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
433
|
-
const relPath =
|
|
434
|
-
match.file_path && config.vaultDir
|
|
435
|
-
? match.file_path.replace(config.vaultDir + "/", "")
|
|
436
|
-
: match.file_path || "n/a";
|
|
437
|
-
const lines = [
|
|
438
|
-
`## Entity Match (exact)\n`,
|
|
439
|
-
`### ${match.title || "(untitled)"} [${match.kind}/${match.category}]`,
|
|
440
|
-
`1.000 · ${tagStr} · ${relPath} · id: \`${match.id}\``,
|
|
441
|
-
match.body?.slice(0, 300) + (match.body?.length > 300 ? "..." : ""),
|
|
442
|
-
];
|
|
443
|
-
return ok(lines.join("\n"));
|
|
444
|
-
}
|
|
445
|
-
// Fall through to semantic search as fallback
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
// Gap 2: Event default time-window
|
|
449
|
-
const effectiveCategory =
|
|
450
|
-
scopedCategory || (kindFilter ? categoryFor(kindFilter) : null);
|
|
451
|
-
let effectiveSince = since || null;
|
|
452
|
-
let effectiveUntil = until || null;
|
|
453
|
-
let autoWindowed = false;
|
|
454
|
-
if (effectiveCategory === "event" && !since && !until) {
|
|
455
|
-
const decayMs = (config.eventDecayDays || 30) * 86400000;
|
|
456
|
-
effectiveSince = new Date(Date.now() - decayMs).toISOString();
|
|
457
|
-
autoWindowed = true;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
const effectiveLimit = limit || 10;
|
|
461
|
-
// When tag-filtering, over-fetch to compensate for post-filter reduction
|
|
462
|
-
const MAX_FETCH_LIMIT = 500;
|
|
463
|
-
const fetchLimit = effectiveTags.length
|
|
464
|
-
? Math.min(effectiveLimit * 10, MAX_FETCH_LIMIT)
|
|
465
|
-
: effectiveLimit;
|
|
466
|
-
|
|
467
|
-
let filtered;
|
|
468
|
-
if (hasQuery) {
|
|
469
|
-
// Hybrid search mode
|
|
470
|
-
const sorted = await hybridSearch(ctx, query, {
|
|
471
|
-
kindFilter,
|
|
472
|
-
categoryFilter: scopedCategory || null,
|
|
473
|
-
excludeEvents: shouldExcludeEvents,
|
|
474
|
-
since: effectiveSince,
|
|
475
|
-
until: effectiveUntil,
|
|
476
|
-
limit: fetchLimit,
|
|
477
|
-
decayDays: config.eventDecayDays || 30,
|
|
478
|
-
userIdFilter: userId,
|
|
479
|
-
includeSuperseeded: include_superseded ?? false,
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
// Post-filter by tags if provided, then apply requested limit
|
|
483
|
-
filtered = effectiveTags.length
|
|
484
|
-
? sorted
|
|
485
|
-
.filter((r) => {
|
|
486
|
-
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
487
|
-
return effectiveTags.some((t) => entryTags.includes(t));
|
|
488
|
-
})
|
|
489
|
-
.slice(0, effectiveLimit)
|
|
490
|
-
: sorted;
|
|
491
|
-
} else {
|
|
492
|
-
// Filter-only mode (no query, use SQL directly)
|
|
493
|
-
const clauses = [];
|
|
494
|
-
const params = [];
|
|
495
|
-
if (userId !== undefined) {
|
|
496
|
-
clauses.push("user_id = ?");
|
|
497
|
-
params.push(userId);
|
|
498
|
-
}
|
|
499
|
-
if (kindFilter) {
|
|
500
|
-
clauses.push("kind = ?");
|
|
501
|
-
params.push(kindFilter);
|
|
502
|
-
}
|
|
503
|
-
if (scopedCategory) {
|
|
504
|
-
clauses.push("category = ?");
|
|
505
|
-
params.push(scopedCategory);
|
|
506
|
-
}
|
|
507
|
-
if (effectiveSince) {
|
|
508
|
-
clauses.push("created_at >= ?");
|
|
509
|
-
params.push(effectiveSince);
|
|
510
|
-
}
|
|
511
|
-
if (effectiveUntil) {
|
|
512
|
-
clauses.push("created_at <= ?");
|
|
513
|
-
params.push(effectiveUntil);
|
|
514
|
-
}
|
|
515
|
-
clauses.push("(expires_at IS NULL OR expires_at > datetime('now'))");
|
|
516
|
-
if (!include_superseded) {
|
|
517
|
-
clauses.push("superseded_by IS NULL");
|
|
518
|
-
}
|
|
519
|
-
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
520
|
-
params.push(fetchLimit);
|
|
521
|
-
const rows = ctx.db
|
|
522
|
-
.prepare(`SELECT * FROM vault ${where} ORDER BY created_at DESC LIMIT ?`)
|
|
523
|
-
.all(...params);
|
|
524
|
-
|
|
525
|
-
// Post-filter by tags if provided, then apply requested limit
|
|
526
|
-
filtered = effectiveTags.length
|
|
527
|
-
? rows
|
|
528
|
-
.filter((r) => {
|
|
529
|
-
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
530
|
-
return effectiveTags.some((t) => entryTags.includes(t));
|
|
531
|
-
})
|
|
532
|
-
.slice(0, effectiveLimit)
|
|
533
|
-
: rows;
|
|
534
|
-
|
|
535
|
-
// Add score field for consistent output
|
|
536
|
-
for (const r of filtered) r.score = 0;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
// Brief score boost: briefs rank slightly higher so consolidated snapshots
|
|
540
|
-
// surface above the individual entries they summarize.
|
|
541
|
-
for (const r of filtered) {
|
|
542
|
-
if (r.kind === "brief") r.score = (r.score || 0) + BRIEF_SCORE_BOOST;
|
|
543
|
-
}
|
|
544
|
-
filtered.sort((a, b) => b.score - a.score);
|
|
545
|
-
|
|
546
|
-
// Tier filter: exclude ephemeral entries by default (NULL tier treated as working)
|
|
547
|
-
if (!include_ephemeral) {
|
|
548
|
-
filtered = filtered.filter((r) => r.tier !== "ephemeral");
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
// Event category filter: exclude events from semantic search by default
|
|
552
|
-
if (shouldExcludeEvents) {
|
|
553
|
-
filtered = filtered.filter((r) => r.category !== "event");
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
if (!filtered.length) {
|
|
557
|
-
if (autoWindowed) {
|
|
558
|
-
const days = config.eventDecayDays || 30;
|
|
559
|
-
return ok(
|
|
560
|
-
hasQuery
|
|
561
|
-
? `No results found for "${query}" in events (last ${days} days).\nTry with \`since: "YYYY-MM-DD"\` to search older events.`
|
|
562
|
-
: `No entries found matching the given filters in events (last ${days} days).\nTry with \`since: "YYYY-MM-DD"\` to search older events.`,
|
|
563
|
-
);
|
|
564
|
-
}
|
|
565
|
-
return ok(
|
|
566
|
-
hasQuery
|
|
567
|
-
? "No results found for: " + query
|
|
568
|
-
: "No entries found matching the given filters.",
|
|
569
|
-
);
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
// Decrypt encrypted entries if ctx.decrypt is available
|
|
573
|
-
if (ctx.decrypt) {
|
|
574
|
-
for (const r of filtered) {
|
|
575
|
-
if (r.body_encrypted) {
|
|
576
|
-
const decrypted = await ctx.decrypt(r);
|
|
577
|
-
r.body = decrypted.body;
|
|
578
|
-
if (decrypted.title) r.title = decrypted.title;
|
|
579
|
-
if (decrypted.meta) r.meta = JSON.stringify(decrypted.meta);
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
// Token-budgeted packing
|
|
585
|
-
let tokensBudget = null;
|
|
586
|
-
let tokensUsed = null;
|
|
587
|
-
if (max_tokens != null && max_tokens > 0) {
|
|
588
|
-
tokensBudget = max_tokens;
|
|
589
|
-
const packed = [];
|
|
590
|
-
let used = 0;
|
|
591
|
-
for (const entry of filtered) {
|
|
592
|
-
const entryTokens = Math.ceil((entry.body?.length || 0) / 4);
|
|
593
|
-
if (packed.length === 0 || used + entryTokens <= tokensBudget) {
|
|
594
|
-
packed.push(entry);
|
|
595
|
-
used += entryTokens;
|
|
596
|
-
}
|
|
597
|
-
if (used >= tokensBudget) break;
|
|
598
|
-
}
|
|
599
|
-
tokensUsed = used;
|
|
600
|
-
filtered = packed;
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// Skeleton mode: determine pivot threshold
|
|
604
|
-
const effectivePivot =
|
|
605
|
-
pivot_count != null ? pivot_count : DEFAULT_PIVOT_COUNT;
|
|
606
|
-
|
|
607
|
-
// Conflict detection
|
|
608
|
-
const conflicts = detect_conflicts ? detectConflicts(filtered, ctx) : [];
|
|
609
|
-
|
|
610
|
-
const lines = [];
|
|
611
|
-
if (reindexFailed)
|
|
612
|
-
lines.push(
|
|
613
|
-
`> **Warning:** Auto-reindex failed. Results may be stale. Run \`context-vault reindex\` to fix.\n`,
|
|
614
|
-
);
|
|
615
|
-
if (hasQuery && isEmbedAvailable() === false)
|
|
616
|
-
lines.push(
|
|
617
|
-
`> **Note:** Semantic search unavailable — results ranked by keyword match only. Run \`context-vault setup\` to download the embedding model.\n`,
|
|
618
|
-
);
|
|
619
|
-
const heading = hasQuery ? `Results for "${query}"` : "Filtered entries";
|
|
620
|
-
lines.push(`## ${heading} (${filtered.length} matches)\n`);
|
|
621
|
-
if (tokensBudget != null) {
|
|
622
|
-
lines.push(
|
|
623
|
-
`> Token budget: ${tokensUsed} / ${tokensBudget} tokens used.\n`,
|
|
624
|
-
);
|
|
625
|
-
}
|
|
626
|
-
if (autoWindowed) {
|
|
627
|
-
const days = config.eventDecayDays || 30;
|
|
628
|
-
lines.push(
|
|
629
|
-
`> ℹ Event search limited to last ${days} days. Use \`since\` parameter for older results.\n`,
|
|
630
|
-
);
|
|
631
|
-
}
|
|
632
|
-
for (let i = 0; i < filtered.length; i++) {
|
|
633
|
-
const r = filtered[i];
|
|
634
|
-
const isSkeleton = i >= effectivePivot;
|
|
635
|
-
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
636
|
-
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
637
|
-
const relPath =
|
|
638
|
-
r.file_path && config.vaultDir
|
|
639
|
-
? r.file_path.replace(config.vaultDir + "/", "")
|
|
640
|
-
: r.file_path || "n/a";
|
|
641
|
-
const skeletonLabel = isSkeleton ? " ⊘ skeleton" : "";
|
|
642
|
-
lines.push(
|
|
643
|
-
`### [${i + 1}/${filtered.length}] ${r.title || "(untitled)"} [${r.kind}/${r.category}]${skeletonLabel}`,
|
|
644
|
-
);
|
|
645
|
-
const dateStr =
|
|
646
|
-
r.updated_at && r.updated_at !== r.created_at
|
|
647
|
-
? `${r.created_at} (updated ${r.updated_at})`
|
|
648
|
-
: r.created_at || "";
|
|
649
|
-
const tierStr = r.tier ? ` · tier: ${r.tier}` : "";
|
|
650
|
-
lines.push(
|
|
651
|
-
`${r.score.toFixed(3)} · ${tagStr} · ${relPath} · ${dateStr} · skeleton: ${isSkeleton}${tierStr} · id: \`${r.id}\``,
|
|
652
|
-
);
|
|
653
|
-
const stalenessResult = checkStaleness(r);
|
|
654
|
-
if (stalenessResult) {
|
|
655
|
-
r.stale = true;
|
|
656
|
-
r.stale_reason = stalenessResult.stale_reason;
|
|
657
|
-
lines.push(`> ⚠ **Stale**: ${stalenessResult.stale_reason}`);
|
|
658
|
-
}
|
|
659
|
-
if (isSkeleton) {
|
|
660
|
-
lines.push(skeletonBody(r.body));
|
|
661
|
-
} else {
|
|
662
|
-
lines.push(r.body?.slice(0, 300) + (r.body?.length > 300 ? "..." : ""));
|
|
663
|
-
}
|
|
664
|
-
lines.push("");
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
if (detect_conflicts) {
|
|
668
|
-
if (conflicts.length === 0) {
|
|
669
|
-
lines.push(
|
|
670
|
-
`## Conflict Detection\n\nNo conflicts detected among results.\n`,
|
|
671
|
-
);
|
|
672
|
-
} else {
|
|
673
|
-
lines.push(`## Conflict Detection (${conflicts.length} flagged)\n`);
|
|
674
|
-
for (const c of conflicts) {
|
|
675
|
-
lines.push(
|
|
676
|
-
`- **${c.reason}**: \`${c.entry_a_id}\` vs \`${c.entry_b_id}\``,
|
|
677
|
-
);
|
|
678
|
-
lines.push(` Recommendation: ${c.recommendation}`);
|
|
679
|
-
}
|
|
680
|
-
lines.push("");
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
// Graph traversal: follow related_to links bidirectionally
|
|
685
|
-
if (follow_links) {
|
|
686
|
-
const { forward, backward } = collectLinkedEntries(
|
|
687
|
-
ctx.db,
|
|
688
|
-
filtered,
|
|
689
|
-
userId,
|
|
690
|
-
);
|
|
691
|
-
const allLinked = [...forward, ...backward];
|
|
692
|
-
const seen = new Set();
|
|
693
|
-
const uniqueLinked = allLinked.filter((e) => {
|
|
694
|
-
if (seen.has(e.id)) return false;
|
|
695
|
-
seen.add(e.id);
|
|
696
|
-
return true;
|
|
697
|
-
});
|
|
698
|
-
|
|
699
|
-
if (uniqueLinked.length > 0) {
|
|
700
|
-
lines.push(`## Linked Entries (${uniqueLinked.length} via related_to)\n`);
|
|
701
|
-
for (const r of uniqueLinked) {
|
|
702
|
-
const direction = forward.some((f) => f.id === r.id)
|
|
703
|
-
? "→ forward"
|
|
704
|
-
: "← backlink";
|
|
705
|
-
const entryTags = r.tags ? JSON.parse(r.tags) : [];
|
|
706
|
-
const tagStr = entryTags.length ? entryTags.join(", ") : "none";
|
|
707
|
-
const relPath =
|
|
708
|
-
r.file_path && config.vaultDir
|
|
709
|
-
? r.file_path.replace(config.vaultDir + "/", "")
|
|
710
|
-
: r.file_path || "n/a";
|
|
711
|
-
lines.push(
|
|
712
|
-
`### ${r.title || "(untitled)"} [${r.kind}/${r.category}] ${direction}`,
|
|
713
|
-
);
|
|
714
|
-
lines.push(`${tagStr} · ${relPath} · id: \`${r.id}\``);
|
|
715
|
-
lines.push(r.body?.slice(0, 200) + (r.body?.length > 200 ? "..." : ""));
|
|
716
|
-
lines.push("");
|
|
717
|
-
}
|
|
718
|
-
} else {
|
|
719
|
-
lines.push(`## Linked Entries\n\nNo related entries found.\n`);
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
// Consolidation suggestion detection — lazy, opportunistic, vault-wide
|
|
724
|
-
const consolidationOpts = {
|
|
725
|
-
tagThreshold:
|
|
726
|
-
config.consolidation?.tagThreshold ?? CONSOLIDATION_TAG_THRESHOLD,
|
|
727
|
-
maxAgeDays:
|
|
728
|
-
config.consolidation?.maxAgeDays ?? CONSOLIDATION_SNAPSHOT_MAX_AGE_DAYS,
|
|
729
|
-
};
|
|
730
|
-
const consolidationSuggestions = detectConsolidationHints(
|
|
731
|
-
filtered,
|
|
732
|
-
ctx.db,
|
|
733
|
-
userId,
|
|
734
|
-
consolidationOpts,
|
|
735
|
-
);
|
|
736
|
-
|
|
737
|
-
// Auto-consolidate: fire-and-forget create_snapshot for eligible tags
|
|
738
|
-
if (
|
|
739
|
-
config.consolidation?.autoConsolidate &&
|
|
740
|
-
consolidationSuggestions.length > 0
|
|
741
|
-
) {
|
|
742
|
-
const { handler: snapshotHandler } = await import("./create-snapshot.js");
|
|
743
|
-
for (const suggestion of consolidationSuggestions) {
|
|
744
|
-
snapshotHandler({ topic: suggestion.tag, tags: [suggestion.tag] }, ctx, {
|
|
745
|
-
ensureIndexed: async () => {},
|
|
746
|
-
}).catch(() => {});
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
const result = ok(lines.join("\n"));
|
|
751
|
-
const meta = {};
|
|
752
|
-
meta.scope = effectiveScope;
|
|
753
|
-
if (tokensBudget != null) {
|
|
754
|
-
meta.tokens_used = tokensUsed;
|
|
755
|
-
meta.tokens_budget = tokensBudget;
|
|
756
|
-
}
|
|
757
|
-
if (buckets?.length) {
|
|
758
|
-
meta.buckets = buckets;
|
|
759
|
-
}
|
|
760
|
-
if (consolidationSuggestions.length > 0) {
|
|
761
|
-
meta.consolidation_suggestions = consolidationSuggestions;
|
|
762
|
-
}
|
|
763
|
-
result._meta = meta;
|
|
764
|
-
return result;
|
|
765
|
-
}
|