@cortexkit/pi-magic-context 0.46.1 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +112 -0
- package/dist/embedding-activity-ew6dshfd.js +10 -0
- package/dist/embedding-backfill-bnnytehy.js +20 -0
- package/dist/embedding-cache-7rp4gqzr.js +16 -0
- package/dist/embedding-worker.js +11 -8
- package/dist/harness-9872e0ym.js +10 -0
- package/dist/historian-calibration-extension.js +3 -2
- package/dist/index-0am8a7k1.js +152 -0
- package/dist/index-0h2mkab2.js +17 -0
- package/dist/index-135e8hvg.js +43 -0
- package/dist/{index-wv28nczk.js → index-3cgbv47m.js} +354 -38
- package/dist/index-419bndtm.js +2309 -0
- package/dist/index-544c0e0x.js +3026 -0
- package/dist/index-5hd7h4rr.js +3098 -0
- package/dist/index-5jp1h0bz.js +750 -0
- package/dist/index-603mt6nm.js +171 -0
- package/dist/index-674f6qs2.js +16 -0
- package/dist/{index-r01b1b78.js → index-6pgq5ns2.js} +157 -2454
- package/dist/index-72bdc1bd.js +3591 -0
- package/dist/index-8976mezv.js +209 -0
- package/dist/{index-td885aah.js → index-8y5knskr.js} +98 -231
- package/dist/{index-ykb8kkm3.js → index-98z1c1r0.js} +14599 -22287
- package/dist/index-9p40dgkm.js +22 -0
- package/dist/index-9x1f96r8.js +11 -0
- package/dist/{index-266qrgss.js → index-9xb0axhc.js} +301 -27
- package/dist/{index-phh7e055.js → index-bgmv2kaf.js} +28 -28
- package/dist/index-dkx0b1kt.js +514 -0
- package/dist/{index-7n8gb6j3.js → index-f956htf3.js} +28 -31
- package/dist/index-fapenpch.js +13 -0
- package/dist/index-fm3ksh5e.js +1060 -0
- package/dist/{index-2p1e8706.js → index-fp37he9a.js} +3 -3
- package/dist/index-hrf13mn3.js +2388 -0
- package/dist/index-pvm4pd5d.js +3069 -0
- package/dist/index-r7xg4hvw.js +3480 -0
- package/dist/index-rt5jgzp1.js +1127 -0
- package/dist/{index-n3g5mwa8.js → index-wrm4nbj7.js} +3 -3
- package/dist/index-xjqheg7a.js +931 -0
- package/dist/index-yhzhdp3m.js +143 -0
- package/dist/{index-w76x3af7.js → index-zrr5t5a7.js} +1 -3
- package/dist/index.js +6345 -2585
- package/dist/logger-v3dsxdnm.js +19 -0
- package/dist/message-index-bounded-t7vvty76.js +120 -0
- package/dist/{message-time-backfill-4688z1k2.js → message-time-backfill-4ndf7hrq.js} +7 -6
- package/dist/migration-worker.js +24 -16
- package/dist/project-embedding-registry-jy2hmq28.js +85 -0
- package/dist/raw-ordinal-warmup-worker.js +60 -0
- package/dist/{read-session-chunk-tc6yn2pe.js → read-session-chunk-47pjn7fz.js} +12 -6
- package/dist/read-session-raw-yn8nawav.js +55 -0
- package/dist/{safe-notification-target-6cn3z99t.js → safe-notification-target-3c1ns931.js} +3 -2
- package/dist/search-42szfhwp.js +40 -0
- package/dist/sqlite-3yye76ht.js +44 -0
- package/dist/sqlite-context-pragmas-gcgch6qz.js +6 -0
- package/dist/storage-memory-1z20y55x.js +71 -0
- package/dist/storage-tags-jym5z6yk.js +127 -0
- package/dist/subagent-entry.js +36 -16
- package/dist/transformers-node-wasm.js +18415 -18415
- package/package.json +3 -3
- package/dist/index-4vfrjwf6.js +0 -7737
- package/dist/index-xvw8zptx.js +0 -5123
- /package/dist/{index-901ekbga.js → index-5prbx026.js} +0 -0
|
@@ -0,0 +1,3069 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getHarness2
|
|
3
|
+
} from "./index-0h2mkab2.js";
|
|
4
|
+
import {
|
|
5
|
+
isTransientSqliteError2
|
|
6
|
+
} from "./index-3cgbv47m.js";
|
|
7
|
+
import {
|
|
8
|
+
PROMOTABLE_CATEGORIES,
|
|
9
|
+
CATEGORY_DEFAULT_TTL
|
|
10
|
+
} from "./index-135e8hvg.js";
|
|
11
|
+
import {
|
|
12
|
+
log2,
|
|
13
|
+
sessionLog2
|
|
14
|
+
} from "./index-f956htf3.js";
|
|
15
|
+
import {
|
|
16
|
+
resolveWorkspaceShareCategories,
|
|
17
|
+
resolveWorkspaceIdentitySet,
|
|
18
|
+
expandWorkspaceIdentitySetWithAliases,
|
|
19
|
+
resolveStoredPathWorkspaceIdentity,
|
|
20
|
+
sourceNameForMemory,
|
|
21
|
+
managedAuthorityNoteRow
|
|
22
|
+
} from "./index-72bdc1bd.js";
|
|
23
|
+
import {
|
|
24
|
+
isKnownAutocommit
|
|
25
|
+
} from "./index-8y5knskr.js";
|
|
26
|
+
import {
|
|
27
|
+
loadCompartmentChunkEmbeddingsForSearch
|
|
28
|
+
} from "./index-fm3ksh5e.js";
|
|
29
|
+
import {
|
|
30
|
+
recordEmbeddingMeasurement,
|
|
31
|
+
saveCommitEmbedding,
|
|
32
|
+
loadProjectCommitEmbeddings,
|
|
33
|
+
loadUnembeddedCommits,
|
|
34
|
+
countEmbeddedCommits,
|
|
35
|
+
upsertCommits,
|
|
36
|
+
getLatestIndexedCommitSha,
|
|
37
|
+
enforceProjectCap,
|
|
38
|
+
contentSha2562,
|
|
39
|
+
getShadowEmbeddingMeasurementCohort2,
|
|
40
|
+
getPrimaryEmbeddingMeasurementCohort2,
|
|
41
|
+
embedShadowTextForProject2,
|
|
42
|
+
enqueueShadowEmbeddingItems2,
|
|
43
|
+
getProjectEmbeddingSnapshot2,
|
|
44
|
+
embedTextForProject2,
|
|
45
|
+
embedItemsForProject2
|
|
46
|
+
} from "./index-544c0e0x.js";
|
|
47
|
+
import {
|
|
48
|
+
saveEmbeddingIfHashMatches,
|
|
49
|
+
getProjectEmbeddings2
|
|
50
|
+
} from "./index-yhzhdp3m.js";
|
|
51
|
+
import {
|
|
52
|
+
cosineSimilarity,
|
|
53
|
+
isEmbeddingEnabled,
|
|
54
|
+
embedText,
|
|
55
|
+
ensureMemoryEmbeddings2
|
|
56
|
+
} from "./index-0am8a7k1.js";
|
|
57
|
+
import {
|
|
58
|
+
getMemorySelectColumns2,
|
|
59
|
+
isMemoryRow2,
|
|
60
|
+
toMemory2,
|
|
61
|
+
ModuleMemoryAuthorityError2,
|
|
62
|
+
insertMemory2,
|
|
63
|
+
getMemoryByHash2,
|
|
64
|
+
getMemoriesByProject2,
|
|
65
|
+
buildWorkspaceMemorySqlFilter2,
|
|
66
|
+
getMemoriesByProjects2,
|
|
67
|
+
getMemoryById2,
|
|
68
|
+
updateMemorySeenCount2,
|
|
69
|
+
updateMemoryRetrievalCount2
|
|
70
|
+
} from "./index-xjqheg7a.js";
|
|
71
|
+
import {
|
|
72
|
+
getIndexedMessageCorpusSize
|
|
73
|
+
} from "./index-5jp1h0bz.js";
|
|
74
|
+
import {
|
|
75
|
+
computeNormalizedHash
|
|
76
|
+
} from "./index-9x1f96r8.js";
|
|
77
|
+
|
|
78
|
+
// ../plugin/src/features/magic-context/storage-notes.ts
|
|
79
|
+
var NOTE_TYPES = new Set(["session", "smart"]);
|
|
80
|
+
var NOTE_STATUSES = new Set(["active", "pending", "ready", "dismissed"]);
|
|
81
|
+
var NOTE_CHECK_STATUSES = new Set([
|
|
82
|
+
"uncompiled",
|
|
83
|
+
"compiled",
|
|
84
|
+
"failing",
|
|
85
|
+
"fallback",
|
|
86
|
+
"parked"
|
|
87
|
+
]);
|
|
88
|
+
var CONDITION_COMPILE_STATUSES = new Set([
|
|
89
|
+
"compiled",
|
|
90
|
+
"plain",
|
|
91
|
+
"refused"
|
|
92
|
+
]);
|
|
93
|
+
var DEFAULT_SMART_STATUSES = ["pending", "ready"];
|
|
94
|
+
function toNullableString(value) {
|
|
95
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
96
|
+
}
|
|
97
|
+
function toNullableNumber(value) {
|
|
98
|
+
return typeof value === "number" ? value : null;
|
|
99
|
+
}
|
|
100
|
+
function isNoteRow(row) {
|
|
101
|
+
if (row === null || typeof row !== "object")
|
|
102
|
+
return false;
|
|
103
|
+
const candidate = row;
|
|
104
|
+
return typeof candidate.id === "number" && typeof candidate.type === "string" && NOTE_TYPES.has(candidate.type) && typeof candidate.status === "string" && NOTE_STATUSES.has(candidate.status) && typeof candidate.content === "string" && (candidate.session_id === null || typeof candidate.session_id === "string") && (candidate.project_path === null || typeof candidate.project_path === "string") && (candidate.surface_condition === null || typeof candidate.surface_condition === "string") && typeof candidate.created_at === "number" && typeof candidate.updated_at === "number" && (candidate.last_checked_at === null || typeof candidate.last_checked_at === "number") && (candidate.ready_at === null || typeof candidate.ready_at === "number") && (candidate.ready_reason === null || typeof candidate.ready_reason === "string");
|
|
105
|
+
}
|
|
106
|
+
function toNote(row) {
|
|
107
|
+
return {
|
|
108
|
+
id: row.id,
|
|
109
|
+
type: row.type,
|
|
110
|
+
status: row.status,
|
|
111
|
+
content: row.content,
|
|
112
|
+
sessionId: toNullableString(row.session_id),
|
|
113
|
+
projectPath: toNullableString(row.project_path),
|
|
114
|
+
surfaceCondition: toNullableString(row.surface_condition),
|
|
115
|
+
compiledProvider: toNullableString(row.compiled_provider),
|
|
116
|
+
compiledConfig: toNullableString(row.compiled_config),
|
|
117
|
+
compiledAt: toNullableNumber(row.compiled_at),
|
|
118
|
+
compileStatus: typeof row.compile_status === "string" && CONDITION_COMPILE_STATUSES.has(row.compile_status) ? row.compile_status : null,
|
|
119
|
+
createdAt: row.created_at,
|
|
120
|
+
updatedAt: row.updated_at,
|
|
121
|
+
lastCheckedAt: toNullableNumber(row.last_checked_at),
|
|
122
|
+
readyAt: toNullableNumber(row.ready_at),
|
|
123
|
+
readyReason: toNullableString(row.ready_reason),
|
|
124
|
+
anchorOrdinal: toNullableNumber(row.anchor_ordinal),
|
|
125
|
+
compiledCheck: toNullableString(row.compiled_check),
|
|
126
|
+
manifestJson: toNullableString(row.manifest_json),
|
|
127
|
+
checkHash: toNullableString(row.check_hash),
|
|
128
|
+
checkCron: toNullableString(row.check_cron),
|
|
129
|
+
checkVersion: toNullableNumber(row.check_version),
|
|
130
|
+
checkStatus: typeof row.check_status === "string" && NOTE_CHECK_STATUSES.has(row.check_status) ? row.check_status : null,
|
|
131
|
+
checkFailureCount: toNullableNumber(row.check_failure_count) ?? 0,
|
|
132
|
+
checkNetworkFailureCount: toNullableNumber(row.check_network_failure_count) ?? 0,
|
|
133
|
+
checkQuarantinedUntil: toNullableNumber(row.check_quarantined_until),
|
|
134
|
+
checkNextDueAt: toNullableNumber(row.check_next_due_at),
|
|
135
|
+
checkCompiledAt: toNullableNumber(row.check_compiled_at),
|
|
136
|
+
checkFalseSinceAt: toNullableNumber(row.check_false_since_at),
|
|
137
|
+
checkLastLivenessAt: toNullableNumber(row.check_last_liveness_at),
|
|
138
|
+
policyVersion: toNullableNumber(row.policy_version)
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function noteCheckColumnsExist(db) {
|
|
142
|
+
try {
|
|
143
|
+
const rows = db.prepare("PRAGMA table_info(notes)").all();
|
|
144
|
+
return rows.some((row) => row.name === "compiled_check");
|
|
145
|
+
} catch {
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
var PENDING_SESSION_NOTE_PREDICATE = "type = 'session' AND status = 'pending' AND surface_condition IS NOT NULL";
|
|
150
|
+
function healPendingSessionNotes(db) {
|
|
151
|
+
let predicate = `${PENDING_SESSION_NOTE_PREDICATE} AND NOT ${managedAuthorityNoteRow("notes")}`;
|
|
152
|
+
let healable;
|
|
153
|
+
try {
|
|
154
|
+
healable = db.prepare(`SELECT 1 FROM notes WHERE ${predicate} LIMIT 1`).get();
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (!(error instanceof Error) || !error.message.includes("no such table"))
|
|
157
|
+
throw error;
|
|
158
|
+
predicate = PENDING_SESSION_NOTE_PREDICATE;
|
|
159
|
+
healable = db.prepare(`SELECT 1 FROM notes WHERE ${predicate} LIMIT 1`).get();
|
|
160
|
+
}
|
|
161
|
+
if (healable == null)
|
|
162
|
+
return;
|
|
163
|
+
try {
|
|
164
|
+
db.prepare(`UPDATE notes SET status = 'active', surface_condition = NULL WHERE ${predicate}`).run();
|
|
165
|
+
} catch (error) {
|
|
166
|
+
if (!isTransientSqliteError2(error))
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function getNoteById(db, noteId) {
|
|
171
|
+
healPendingSessionNotes(db);
|
|
172
|
+
const row = db.prepare("SELECT * FROM notes WHERE id = ?").get(noteId);
|
|
173
|
+
return isNoteRow(row) ? toNote(row) : null;
|
|
174
|
+
}
|
|
175
|
+
function getNoteByIdInScope(db, noteId, scope) {
|
|
176
|
+
const note = getNoteById(db, noteId);
|
|
177
|
+
return note && noteBelongsToScope(note, scope) ? note : null;
|
|
178
|
+
}
|
|
179
|
+
function noteBelongsToScope(note, scope) {
|
|
180
|
+
if (note.type === "session") {
|
|
181
|
+
return note.sessionId === scope.sessionId;
|
|
182
|
+
}
|
|
183
|
+
return note.projectPath === scope.projectPath;
|
|
184
|
+
}
|
|
185
|
+
function buildStatusClause(status) {
|
|
186
|
+
if (status === undefined) {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
const statuses = Array.isArray(status) ? status : [status];
|
|
190
|
+
if (statuses.length === 0) {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
const placeholders = statuses.map(() => "?").join(", ");
|
|
194
|
+
return {
|
|
195
|
+
sql: `status IN (${placeholders})`,
|
|
196
|
+
params: statuses
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function getNotes(db, options = {}) {
|
|
200
|
+
healPendingSessionNotes(db);
|
|
201
|
+
const clauses = [];
|
|
202
|
+
const params = [];
|
|
203
|
+
if (options.sessionId !== undefined) {
|
|
204
|
+
clauses.push("session_id = ?");
|
|
205
|
+
params.push(options.sessionId);
|
|
206
|
+
}
|
|
207
|
+
if (options.projectPath !== undefined) {
|
|
208
|
+
clauses.push("project_path = ?");
|
|
209
|
+
params.push(options.projectPath);
|
|
210
|
+
}
|
|
211
|
+
if (options.type !== undefined) {
|
|
212
|
+
clauses.push("type = ?");
|
|
213
|
+
params.push(options.type);
|
|
214
|
+
}
|
|
215
|
+
const statusClause = buildStatusClause(options.status);
|
|
216
|
+
if (statusClause) {
|
|
217
|
+
clauses.push(statusClause.sql);
|
|
218
|
+
params.push(...statusClause.params);
|
|
219
|
+
}
|
|
220
|
+
const where = clauses.length > 0 ? ` WHERE ${clauses.join(" AND ")}` : "";
|
|
221
|
+
return db.prepare(`SELECT * FROM notes${where} ORDER BY created_at ASC, id ASC`).all(...params).filter(isNoteRow).map(toNote);
|
|
222
|
+
}
|
|
223
|
+
function addNote(db, type, options) {
|
|
224
|
+
const now = Date.now();
|
|
225
|
+
const result = type === "session" ? db.prepare("INSERT INTO notes (type, status, content, session_id, created_at, updated_at, harness, anchor_ordinal) VALUES ('session', 'active', ?, ?, ?, ?, ?, ?) RETURNING *").get(options.content, options.sessionId, now, now, getHarness2(), options.anchorOrdinal ?? null) : db.prepare("INSERT INTO notes (type, status, content, session_id, project_path, surface_condition, compiled_provider, compiled_config, compiled_at, compile_status, created_at, updated_at, harness, anchor_ordinal) VALUES ('smart', 'pending', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *").get(options.content, options.sessionId ?? null, options.projectPath, options.surfaceCondition, options.compiledProvider ?? null, options.compiledConfig ?? null, options.compiledAt ?? null, options.compileStatus ?? null, now, now, getHarness2(), options.anchorOrdinal ?? null);
|
|
226
|
+
if (!isNoteRow(result)) {
|
|
227
|
+
throw new Error("[notes] failed to insert note");
|
|
228
|
+
}
|
|
229
|
+
return toNote(result);
|
|
230
|
+
}
|
|
231
|
+
function getSessionNotes(db, sessionId) {
|
|
232
|
+
return getNotes(db, { sessionId, type: "session", status: "active" });
|
|
233
|
+
}
|
|
234
|
+
function getSmartNotes(db, projectPath, status) {
|
|
235
|
+
return getNotes(db, {
|
|
236
|
+
projectPath,
|
|
237
|
+
type: "smart",
|
|
238
|
+
status: status ?? DEFAULT_SMART_STATUSES
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
function getPendingSmartNotes(db, projectPath) {
|
|
242
|
+
return getSmartNotes(db, projectPath, "pending");
|
|
243
|
+
}
|
|
244
|
+
function getReadySmartNotes(db, projectPath) {
|
|
245
|
+
return getSmartNotes(db, projectPath, "ready");
|
|
246
|
+
}
|
|
247
|
+
function updateNote(db, noteId, updates, scope) {
|
|
248
|
+
const existing = getNoteByIdInScope(db, noteId, scope);
|
|
249
|
+
if (!existing) {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
if (updates.surfaceCondition !== undefined && existing.type !== "smart") {
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
const now = Date.now();
|
|
256
|
+
const sets = ["updated_at = ?"];
|
|
257
|
+
const params = [now];
|
|
258
|
+
if (updates.content !== undefined) {
|
|
259
|
+
sets.push("content = ?");
|
|
260
|
+
params.push(updates.content);
|
|
261
|
+
}
|
|
262
|
+
if (updates.sessionId !== undefined) {
|
|
263
|
+
sets.push("session_id = ?");
|
|
264
|
+
params.push(updates.sessionId);
|
|
265
|
+
}
|
|
266
|
+
const smartConditionChanged = existing.type === "smart" && updates.surfaceCondition !== undefined && updates.surfaceCondition !== existing.surfaceCondition;
|
|
267
|
+
if (updates.status !== undefined && !smartConditionChanged) {
|
|
268
|
+
sets.push("status = ?");
|
|
269
|
+
params.push(updates.status);
|
|
270
|
+
}
|
|
271
|
+
if (existing.type === "smart") {
|
|
272
|
+
if (updates.projectPath !== undefined) {
|
|
273
|
+
sets.push("project_path = ?");
|
|
274
|
+
params.push(updates.projectPath);
|
|
275
|
+
}
|
|
276
|
+
if (updates.surfaceCondition !== undefined) {
|
|
277
|
+
sets.push("surface_condition = ?");
|
|
278
|
+
params.push(updates.surfaceCondition);
|
|
279
|
+
}
|
|
280
|
+
if (smartConditionChanged) {
|
|
281
|
+
sets.push("status = 'pending'", "last_checked_at = NULL", "ready_at = NULL", "ready_reason = NULL");
|
|
282
|
+
sets.push("compiled_provider = ?", "compiled_config = ?", "compiled_at = ?", "compile_status = ?");
|
|
283
|
+
params.push(updates.compiledProvider ?? null, updates.compiledConfig ?? null, updates.compiledAt ?? null, updates.compileStatus ?? null);
|
|
284
|
+
if (noteCheckColumnsExist(db)) {
|
|
285
|
+
sets.push("compiled_check = NULL", "manifest_json = NULL", "check_hash = NULL", "check_cron = NULL", "check_version = 0", "check_status = 'uncompiled'", "check_failure_count = 0", "check_network_failure_count = 0", "check_quarantined_until = NULL", "check_next_due_at = NULL", "check_compiled_at = NULL", "check_false_since_at = NULL", "check_last_liveness_at = NULL");
|
|
286
|
+
}
|
|
287
|
+
} else {
|
|
288
|
+
if (updates.lastCheckedAt !== undefined) {
|
|
289
|
+
sets.push("last_checked_at = ?");
|
|
290
|
+
params.push(updates.lastCheckedAt);
|
|
291
|
+
}
|
|
292
|
+
if (updates.readyAt !== undefined) {
|
|
293
|
+
sets.push("ready_at = ?");
|
|
294
|
+
params.push(updates.readyAt);
|
|
295
|
+
}
|
|
296
|
+
if (updates.readyReason !== undefined) {
|
|
297
|
+
sets.push("ready_reason = ?");
|
|
298
|
+
params.push(updates.readyReason);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (sets.length === 1) {
|
|
303
|
+
return null;
|
|
304
|
+
}
|
|
305
|
+
params.push(noteId);
|
|
306
|
+
const result = db.prepare(`UPDATE notes SET ${sets.join(", ")} WHERE id = ? RETURNING *`).get(...params);
|
|
307
|
+
return isNoteRow(result) ? toNote(result) : null;
|
|
308
|
+
}
|
|
309
|
+
function dismissNotes(db, noteIds, scope) {
|
|
310
|
+
const now = Date.now();
|
|
311
|
+
return db.transaction(() => noteIds.map((noteId) => {
|
|
312
|
+
const existing = getNoteById(db, noteId);
|
|
313
|
+
if (!existing)
|
|
314
|
+
return { noteId, outcome: "not_found" };
|
|
315
|
+
if (!noteBelongsToScope(existing, scope)) {
|
|
316
|
+
return { noteId, outcome: "not_owned" };
|
|
317
|
+
}
|
|
318
|
+
if (existing.status === "dismissed") {
|
|
319
|
+
return { noteId, outcome: "already_dismissed" };
|
|
320
|
+
}
|
|
321
|
+
const result = db.prepare("UPDATE notes SET status = 'dismissed', updated_at = ? WHERE id = ? AND status != 'dismissed'").run(now, noteId);
|
|
322
|
+
return {
|
|
323
|
+
noteId,
|
|
324
|
+
outcome: result.changes > 0 ? "dismissed" : "already_dismissed"
|
|
325
|
+
};
|
|
326
|
+
})).immediate();
|
|
327
|
+
}
|
|
328
|
+
function dismissNote(db, noteId, scope) {
|
|
329
|
+
return dismissNotes(db, [noteId], scope)[0]?.outcome === "dismissed";
|
|
330
|
+
}
|
|
331
|
+
function markNoteReady(db, noteId, reason) {
|
|
332
|
+
const now = Date.now();
|
|
333
|
+
db.prepare("UPDATE notes SET status = 'ready', ready_at = ?, ready_reason = ?, updated_at = ?, last_checked_at = ? WHERE id = ? AND type = 'smart'").run(now, reason ?? null, now, now, noteId);
|
|
334
|
+
}
|
|
335
|
+
function markNoteChecked(db, noteId) {
|
|
336
|
+
const now = Date.now();
|
|
337
|
+
db.prepare("UPDATE notes SET last_checked_at = ?, updated_at = ? WHERE id = ? AND type = 'smart'").run(now, now, noteId);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// ../plugin/src/features/magic-context/storage-primers.ts
|
|
341
|
+
var PRIMER_CANDIDATE_TTL_MS = 90 * 24 * 60 * 60 * 1000;
|
|
342
|
+
var PRIMER_CANDIDATE_MAX_AGE_MS = 180 * 24 * 60 * 60 * 1000;
|
|
343
|
+
function normalizePrimerQuestion(question) {
|
|
344
|
+
return question.trim().toLowerCase().replace(/[“”]/g, '"').replace(/[’]/g, "'").replace(/\s+/g, " ").replace(/[?.!]+$/g, "").trim();
|
|
345
|
+
}
|
|
346
|
+
function primerOccurrenceKey(candidate) {
|
|
347
|
+
return [
|
|
348
|
+
candidate.projectPath,
|
|
349
|
+
candidate.harness,
|
|
350
|
+
candidate.sessionId,
|
|
351
|
+
candidate.sourceStartMessageId,
|
|
352
|
+
candidate.sourceEndMessageId
|
|
353
|
+
].join("\x1F");
|
|
354
|
+
}
|
|
355
|
+
function primerOccurrenceUtcDay(sourceMessageTime) {
|
|
356
|
+
return new Date(sourceMessageTime).toISOString().slice(0, 10);
|
|
357
|
+
}
|
|
358
|
+
function vectorBlob(vector) {
|
|
359
|
+
if (!vector)
|
|
360
|
+
return null;
|
|
361
|
+
return new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
362
|
+
}
|
|
363
|
+
function blobToFloat32Array(value) {
|
|
364
|
+
if (!value)
|
|
365
|
+
return null;
|
|
366
|
+
const bytes = value instanceof ArrayBuffer ? new Uint8Array(value) : value;
|
|
367
|
+
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
|
|
368
|
+
}
|
|
369
|
+
function parseCandidateIds(raw) {
|
|
370
|
+
if (!raw)
|
|
371
|
+
return [];
|
|
372
|
+
try {
|
|
373
|
+
const parsed = JSON.parse(raw);
|
|
374
|
+
return Array.isArray(parsed) ? parsed.filter((id) => typeof id === "number" && Number.isFinite(id)) : [];
|
|
375
|
+
} catch {
|
|
376
|
+
return [];
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
function parsePrimerSourceProvenance(raw) {
|
|
380
|
+
if (raw === null)
|
|
381
|
+
return null;
|
|
382
|
+
try {
|
|
383
|
+
const parsed = JSON.parse(raw);
|
|
384
|
+
if (!Array.isArray(parsed))
|
|
385
|
+
return null;
|
|
386
|
+
const provenance = [];
|
|
387
|
+
for (const value of parsed) {
|
|
388
|
+
if (!value || typeof value !== "object")
|
|
389
|
+
return null;
|
|
390
|
+
const source = value;
|
|
391
|
+
if (typeof source.candidate_id !== "number" || !Number.isFinite(source.candidate_id) || typeof source.project_path !== "string" || typeof source.harness !== "string" || typeof source.session_id !== "string" || source.source_compartment_start !== null && typeof source.source_compartment_start !== "number" || source.source_compartment_end !== null && typeof source.source_compartment_end !== "number" || typeof source.source_start_message_id !== "string" || typeof source.source_end_message_id !== "string") {
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
provenance.push({
|
|
395
|
+
candidateId: source.candidate_id,
|
|
396
|
+
projectPath: source.project_path,
|
|
397
|
+
harness: source.harness,
|
|
398
|
+
sessionId: source.session_id,
|
|
399
|
+
sourceCompartmentStart: source.source_compartment_start,
|
|
400
|
+
sourceCompartmentEnd: source.source_compartment_end,
|
|
401
|
+
sourceStartMessageId: source.source_start_message_id,
|
|
402
|
+
sourceEndMessageId: source.source_end_message_id
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
return provenance;
|
|
406
|
+
} catch {
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function loadPrimerSourceProvenance(db, candidateIds) {
|
|
411
|
+
const ids = [...new Set(candidateIds)].sort((a, b) => a - b);
|
|
412
|
+
if (ids.length === 0)
|
|
413
|
+
return [];
|
|
414
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
415
|
+
const rows = db.prepare(`SELECT id, project_path, harness, session_id,
|
|
416
|
+
source_compartment_start, source_compartment_end,
|
|
417
|
+
source_start_message_id, source_end_message_id
|
|
418
|
+
FROM primer_candidates
|
|
419
|
+
WHERE id IN (${placeholders})
|
|
420
|
+
ORDER BY id ASC`).all(...ids);
|
|
421
|
+
return rows.map((row) => ({
|
|
422
|
+
candidateId: row.id,
|
|
423
|
+
projectPath: row.project_path,
|
|
424
|
+
harness: row.harness,
|
|
425
|
+
sessionId: row.session_id,
|
|
426
|
+
sourceCompartmentStart: row.source_compartment_start,
|
|
427
|
+
sourceCompartmentEnd: row.source_compartment_end,
|
|
428
|
+
sourceStartMessageId: row.source_start_message_id,
|
|
429
|
+
sourceEndMessageId: row.source_end_message_id
|
|
430
|
+
}));
|
|
431
|
+
}
|
|
432
|
+
function serializePrimerSourceProvenance(provenance, sourceCandidateIds) {
|
|
433
|
+
if (provenance.length === 0 && sourceCandidateIds.length > 0)
|
|
434
|
+
return null;
|
|
435
|
+
return JSON.stringify(provenance.map((source) => ({
|
|
436
|
+
candidate_id: source.candidateId,
|
|
437
|
+
project_path: source.projectPath,
|
|
438
|
+
harness: source.harness,
|
|
439
|
+
session_id: source.sessionId,
|
|
440
|
+
source_compartment_start: source.sourceCompartmentStart,
|
|
441
|
+
source_compartment_end: source.sourceCompartmentEnd,
|
|
442
|
+
source_start_message_id: source.sourceStartMessageId,
|
|
443
|
+
source_end_message_id: source.sourceEndMessageId
|
|
444
|
+
})));
|
|
445
|
+
}
|
|
446
|
+
function normalizedCandidateIds(candidateIds) {
|
|
447
|
+
return [...new Set(candidateIds)].sort((a, b) => a - b);
|
|
448
|
+
}
|
|
449
|
+
function toCandidate(row) {
|
|
450
|
+
return {
|
|
451
|
+
id: row.id,
|
|
452
|
+
projectPath: row.project_path,
|
|
453
|
+
harness: row.harness,
|
|
454
|
+
sessionId: row.session_id,
|
|
455
|
+
question: row.question,
|
|
456
|
+
normalizedQuestion: row.normalized_question,
|
|
457
|
+
sourceCompartmentStart: row.source_compartment_start,
|
|
458
|
+
sourceCompartmentEnd: row.source_compartment_end,
|
|
459
|
+
sourceStartMessageId: row.source_start_message_id,
|
|
460
|
+
sourceEndMessageId: row.source_end_message_id,
|
|
461
|
+
sourceMessageTime: row.source_message_time,
|
|
462
|
+
questionEmbedding: blobToFloat32Array(row.question_embedding),
|
|
463
|
+
questionEmbeddingModelId: row.question_embedding_model_id,
|
|
464
|
+
createdAt: row.created_at
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
function toPrimer(row) {
|
|
468
|
+
const status = row.status === "archived" ? "archived" : "active";
|
|
469
|
+
return {
|
|
470
|
+
id: row.id,
|
|
471
|
+
projectPath: row.project_path,
|
|
472
|
+
question: row.question,
|
|
473
|
+
questionEmbedding: blobToFloat32Array(row.question_embedding),
|
|
474
|
+
questionEmbeddingModelId: row.question_embedding_model_id,
|
|
475
|
+
answer: row.answer,
|
|
476
|
+
status,
|
|
477
|
+
totalSupport: row.total_support,
|
|
478
|
+
lastObservedAt: row.last_observed_at,
|
|
479
|
+
answerRefreshedAt: row.answer_refreshed_at,
|
|
480
|
+
sourceCandidateIds: parseCandidateIds(row.source_candidate_ids),
|
|
481
|
+
sourceProvenance: parsePrimerSourceProvenance(row.source_candidate_provenance),
|
|
482
|
+
createdAt: row.created_at,
|
|
483
|
+
updatedAt: row.updated_at
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
function insertPrimerCandidates(db, candidates) {
|
|
487
|
+
const ids = [];
|
|
488
|
+
const stmt = db.prepare(`
|
|
489
|
+
INSERT INTO primer_candidates (
|
|
490
|
+
project_path, harness, session_id, question, normalized_question,
|
|
491
|
+
source_compartment_start, source_compartment_end,
|
|
492
|
+
source_start_message_id, source_end_message_id, source_message_time,
|
|
493
|
+
question_embedding, question_embedding_model_id, created_at
|
|
494
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
495
|
+
ON CONFLICT(project_path, harness, session_id, source_start_message_id, source_end_message_id)
|
|
496
|
+
DO UPDATE SET
|
|
497
|
+
question = excluded.question,
|
|
498
|
+
normalized_question = excluded.normalized_question,
|
|
499
|
+
source_compartment_start = excluded.source_compartment_start,
|
|
500
|
+
source_compartment_end = excluded.source_compartment_end,
|
|
501
|
+
source_message_time = excluded.source_message_time,
|
|
502
|
+
question_embedding = COALESCE(excluded.question_embedding, primer_candidates.question_embedding),
|
|
503
|
+
question_embedding_model_id = COALESCE(excluded.question_embedding_model_id, primer_candidates.question_embedding_model_id),
|
|
504
|
+
created_at = MIN(primer_candidates.created_at, excluded.created_at)
|
|
505
|
+
`);
|
|
506
|
+
const select = db.prepare(`
|
|
507
|
+
SELECT id FROM primer_candidates
|
|
508
|
+
WHERE project_path = ? AND harness = ? AND session_id = ?
|
|
509
|
+
AND source_start_message_id = ? AND source_end_message_id = ?
|
|
510
|
+
`);
|
|
511
|
+
db.transaction(() => {
|
|
512
|
+
for (const candidate of candidates) {
|
|
513
|
+
const question = candidate.question.trim();
|
|
514
|
+
if (!question)
|
|
515
|
+
continue;
|
|
516
|
+
const normalized = candidate.normalizedQuestion ?? normalizePrimerQuestion(question);
|
|
517
|
+
stmt.run(candidate.projectPath, candidate.harness || "opencode", candidate.sessionId, question, normalized, candidate.sourceCompartmentStart ?? null, candidate.sourceCompartmentEnd ?? null, candidate.sourceStartMessageId, candidate.sourceEndMessageId, candidate.sourceMessageTime, vectorBlob(candidate.questionEmbedding), candidate.questionEmbeddingModelId ?? null, candidate.createdAt ?? Date.now());
|
|
518
|
+
const row = select.get(candidate.projectPath, candidate.harness || "opencode", candidate.sessionId, candidate.sourceStartMessageId, candidate.sourceEndMessageId);
|
|
519
|
+
if (typeof row?.id === "number")
|
|
520
|
+
ids.push(row.id);
|
|
521
|
+
}
|
|
522
|
+
}).immediate();
|
|
523
|
+
return ids;
|
|
524
|
+
}
|
|
525
|
+
function updatePrimerCandidateEmbedding(db, candidateId, vector, modelId) {
|
|
526
|
+
db.prepare("UPDATE primer_candidates SET question_embedding = ?, question_embedding_model_id = ? WHERE id = ?").run(vectorBlob(vector), modelId, candidateId);
|
|
527
|
+
}
|
|
528
|
+
function getPrimerCandidatesByIds(db, ids) {
|
|
529
|
+
if (ids.length === 0)
|
|
530
|
+
return [];
|
|
531
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
532
|
+
const rows = db.prepare(`SELECT * FROM primer_candidates WHERE id IN (${placeholders})`).all(...ids);
|
|
533
|
+
return rows.map(toCandidate);
|
|
534
|
+
}
|
|
535
|
+
function getPrimerCandidatesForPromotion(db, projectPath, now = Date.now(), ttlMs = PRIMER_CANDIDATE_TTL_MS) {
|
|
536
|
+
const cutoff = now - ttlMs;
|
|
537
|
+
const rows = db.prepare(`SELECT * FROM primer_candidates
|
|
538
|
+
WHERE project_path = ? AND source_message_time >= ?
|
|
539
|
+
ORDER BY project_path ASC, harness ASC, session_id ASC, source_start_message_id ASC, source_end_message_id ASC, id ASC`).all(projectPath, cutoff);
|
|
540
|
+
return rows.map(toCandidate);
|
|
541
|
+
}
|
|
542
|
+
function countPrimerCandidatesForProject(db, projectPath) {
|
|
543
|
+
const row = db.prepare("SELECT COUNT(*) AS count FROM primer_candidates WHERE project_path = ?").get(projectPath);
|
|
544
|
+
return row?.count ?? 0;
|
|
545
|
+
}
|
|
546
|
+
function getActivePrimers(db, projectPath) {
|
|
547
|
+
const rows = db.prepare(`SELECT * FROM primers
|
|
548
|
+
WHERE project_path = ? AND status = 'active'
|
|
549
|
+
ORDER BY COALESCE(last_observed_at, created_at) DESC, id ASC`).all(projectPath);
|
|
550
|
+
return rows.map(toPrimer);
|
|
551
|
+
}
|
|
552
|
+
function createPrimer(db, input) {
|
|
553
|
+
return db.transaction(() => {
|
|
554
|
+
const now = input.now ?? Date.now();
|
|
555
|
+
const sourceCandidateIds = normalizedCandidateIds(input.sourceCandidateIds);
|
|
556
|
+
const sourceProvenance = loadPrimerSourceProvenance(db, sourceCandidateIds);
|
|
557
|
+
const info = db.prepare(`INSERT INTO primers (
|
|
558
|
+
project_path, question, question_embedding, question_embedding_model_id, answer,
|
|
559
|
+
status, total_support, last_observed_at, answer_refreshed_at,
|
|
560
|
+
source_candidate_ids, source_candidate_provenance, created_at, updated_at
|
|
561
|
+
) VALUES (?, ?, ?, ?, ?, 'active', ?, ?, NULL, ?, ?, ?, ?)`).run(input.projectPath, input.question, vectorBlob(input.questionEmbedding), input.questionEmbeddingModelId ?? null, input.answer ?? "", input.totalSupport, input.lastObservedAt, JSON.stringify(sourceCandidateIds), serializePrimerSourceProvenance(sourceProvenance, sourceCandidateIds), now, now);
|
|
562
|
+
return Number(info.lastInsertRowid);
|
|
563
|
+
}).immediate();
|
|
564
|
+
}
|
|
565
|
+
function updatePrimerSupport(db, input) {
|
|
566
|
+
db.transaction(() => {
|
|
567
|
+
const sourceCandidateIds = normalizedCandidateIds(input.sourceCandidateIds);
|
|
568
|
+
const allowedIds = new Set(sourceCandidateIds);
|
|
569
|
+
const existingRow = db.prepare("SELECT source_candidate_provenance FROM primers WHERE id = ?").get(input.primerId);
|
|
570
|
+
const provenanceByCandidateId = new Map;
|
|
571
|
+
for (const source of parsePrimerSourceProvenance(existingRow?.source_candidate_provenance ?? null) ?? []) {
|
|
572
|
+
if (allowedIds.has(source.candidateId)) {
|
|
573
|
+
provenanceByCandidateId.set(source.candidateId, source);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
for (const source of loadPrimerSourceProvenance(db, sourceCandidateIds)) {
|
|
577
|
+
provenanceByCandidateId.set(source.candidateId, source);
|
|
578
|
+
}
|
|
579
|
+
const sourceProvenance = [...provenanceByCandidateId.values()].sort((a, b) => a.candidateId - b.candidateId);
|
|
580
|
+
db.prepare(`UPDATE primers
|
|
581
|
+
SET question_embedding = COALESCE(?, question_embedding),
|
|
582
|
+
question_embedding_model_id = COALESCE(?, question_embedding_model_id),
|
|
583
|
+
total_support = ?,
|
|
584
|
+
last_observed_at = ?,
|
|
585
|
+
source_candidate_ids = ?,
|
|
586
|
+
source_candidate_provenance = ?,
|
|
587
|
+
updated_at = ?
|
|
588
|
+
WHERE id = ?`).run(vectorBlob(input.questionEmbedding), input.questionEmbeddingModelId ?? null, input.totalSupport, input.lastObservedAt, JSON.stringify(sourceCandidateIds), serializePrimerSourceProvenance(sourceProvenance, sourceCandidateIds), input.now ?? Date.now(), input.primerId);
|
|
589
|
+
}).immediate();
|
|
590
|
+
}
|
|
591
|
+
function updatePrimerAnswer(db, primerId, answer, refreshedAt = Date.now()) {
|
|
592
|
+
db.prepare("UPDATE primers SET answer = ?, answer_refreshed_at = ?, updated_at = ? WHERE id = ?").run(answer, refreshedAt, refreshedAt, primerId);
|
|
593
|
+
}
|
|
594
|
+
// ../plugin/src/features/magic-context/memory/promotion.ts
|
|
595
|
+
function isPromotableCategory(category) {
|
|
596
|
+
return PROMOTABLE_CATEGORIES.some((promotableCategory) => promotableCategory === category);
|
|
597
|
+
}
|
|
598
|
+
function resolveExpiresAt(category) {
|
|
599
|
+
const ttl = CATEGORY_DEFAULT_TTL[category];
|
|
600
|
+
return ttl === undefined ? null : Date.now() + ttl;
|
|
601
|
+
}
|
|
602
|
+
function promoteSessionFactsDurable(db, sessionId, projectPath, facts) {
|
|
603
|
+
const newMemoryRefs = [];
|
|
604
|
+
let factsPromoted = 0;
|
|
605
|
+
for (const fact of facts) {
|
|
606
|
+
if (!fact || typeof fact.category !== "string" || typeof fact.content !== "string" || fact.content.trim().length === 0) {
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
if (!isPromotableCategory(fact.category)) {
|
|
610
|
+
continue;
|
|
611
|
+
}
|
|
612
|
+
const normalizedHash = computeNormalizedHash(fact.content);
|
|
613
|
+
const existingMemory = getMemoryByHash2(db, projectPath, fact.category, normalizedHash);
|
|
614
|
+
if (existingMemory) {
|
|
615
|
+
updateMemorySeenCount2(db, existingMemory.id);
|
|
616
|
+
factsPromoted += 1;
|
|
617
|
+
continue;
|
|
618
|
+
}
|
|
619
|
+
const memoryInput = {
|
|
620
|
+
projectPath,
|
|
621
|
+
category: fact.category,
|
|
622
|
+
content: fact.content,
|
|
623
|
+
sourceSessionId: sessionId,
|
|
624
|
+
sourceType: "historian",
|
|
625
|
+
expiresAt: resolveExpiresAt(fact.category)
|
|
626
|
+
};
|
|
627
|
+
const memory = insertMemory2(db, memoryInput);
|
|
628
|
+
newMemoryRefs.push({ memoryId: memory.id, content: memory.content });
|
|
629
|
+
factsPromoted += 1;
|
|
630
|
+
}
|
|
631
|
+
return { newMemoryRefs, factsPromoted };
|
|
632
|
+
}
|
|
633
|
+
async function embedPromotedFacts(db, sessionId, projectPath, refs) {
|
|
634
|
+
for (const ref of refs) {
|
|
635
|
+
await embedAndStoreMemory(db, sessionId, projectPath, ref.memoryId, ref.content);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
async function embedAndStoreMemory(db, sessionId, projectPath, memoryId, content) {
|
|
639
|
+
try {
|
|
640
|
+
const hashBeforeEmbed = getMemoryById2(db, memoryId)?.normalizedHash;
|
|
641
|
+
if (!hashBeforeEmbed) {
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
const result = await embedTextForProject2(projectPath, content);
|
|
645
|
+
if (result) {
|
|
646
|
+
db.transaction(() => {
|
|
647
|
+
saveEmbeddingIfHashMatches(db, memoryId, result.vector, result.modelId, hashBeforeEmbed);
|
|
648
|
+
}).immediate();
|
|
649
|
+
}
|
|
650
|
+
} catch (error) {
|
|
651
|
+
sessionLog2(sessionId, `memory embedding failed for memory ${memoryId}:`, error);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
// ../plugin/src/features/magic-context/memory/storage-memory-fts.ts
|
|
655
|
+
var DEFAULT_SEARCH_LIMIT = 10;
|
|
656
|
+
var searchStatements = new WeakMap;
|
|
657
|
+
var datedSearchStatements = new WeakMap;
|
|
658
|
+
var unionSearchStatements = new Map;
|
|
659
|
+
var datedUnionSearchStatements = new Map;
|
|
660
|
+
function getSearchStatement(db, dated = false) {
|
|
661
|
+
const statements = dated ? datedSearchStatements : searchStatements;
|
|
662
|
+
let stmt = statements.get(db);
|
|
663
|
+
if (!stmt) {
|
|
664
|
+
stmt = db.prepare(`SELECT ${getMemorySelectColumns2(db)} FROM memories_fts INNER JOIN memories ON memories.id = memories_fts.rowid WHERE memories.project_path = ? AND memories.status IN ('active', 'permanent') AND (memories.expires_at IS NULL OR memories.expires_at > ?)${dated ? " AND memories.created_at BETWEEN ? AND ?" : ""} AND memories_fts MATCH ? ORDER BY bm25(memories_fts), memories.updated_at DESC, memories.id ASC LIMIT ?`);
|
|
665
|
+
statements.set(db, stmt);
|
|
666
|
+
}
|
|
667
|
+
return stmt;
|
|
668
|
+
}
|
|
669
|
+
function getUnionSearchStatement(db, arity, dated = false) {
|
|
670
|
+
const statementRegistry = dated ? datedUnionSearchStatements : unionSearchStatements;
|
|
671
|
+
let statements = statementRegistry.get(arity);
|
|
672
|
+
if (!statements) {
|
|
673
|
+
statements = new WeakMap;
|
|
674
|
+
statementRegistry.set(arity, statements);
|
|
675
|
+
}
|
|
676
|
+
let stmt = statements.get(db);
|
|
677
|
+
if (!stmt) {
|
|
678
|
+
const placeholders = Array.from({ length: arity }, () => "?").join(", ");
|
|
679
|
+
stmt = db.prepare(`SELECT ${getMemorySelectColumns2(db)} FROM memories_fts INNER JOIN memories ON memories.id = memories_fts.rowid WHERE memories.project_path IN (${placeholders}) AND memories.status IN ('active', 'permanent') AND (memories.expires_at IS NULL OR memories.expires_at > ?)${dated ? " AND memories.created_at BETWEEN ? AND ?" : ""} AND memories_fts MATCH ? ORDER BY bm25(memories_fts), memories.updated_at DESC, memories.id ASC LIMIT ?`);
|
|
680
|
+
statements.set(db, stmt);
|
|
681
|
+
}
|
|
682
|
+
return stmt;
|
|
683
|
+
}
|
|
684
|
+
function uniqueProjectPaths(projectPaths) {
|
|
685
|
+
return [...new Set(projectPaths.filter((path) => path.length > 0))];
|
|
686
|
+
}
|
|
687
|
+
function relaxedFtsQuery(query) {
|
|
688
|
+
const tokens = [...new Set(query.match(/[\p{L}\p{N}_]+/gu) ?? [])].filter((token) => token.length > 2 || /\d/.test(token)).slice(0, 16);
|
|
689
|
+
return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(" OR ");
|
|
690
|
+
}
|
|
691
|
+
function sanitizeFtsQuery(query) {
|
|
692
|
+
const tokens = query.replaceAll("\x00", " ").split(/\s+/).filter((token) => token.length > 0);
|
|
693
|
+
if (tokens.length === 0)
|
|
694
|
+
return "";
|
|
695
|
+
return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(" ");
|
|
696
|
+
}
|
|
697
|
+
function searchMemoriesFTS(db, projectPath, query, limit = DEFAULT_SEARCH_LIMIT, dateRange = null) {
|
|
698
|
+
const trimmedQuery = query.trim();
|
|
699
|
+
if (trimmedQuery.length === 0 || limit <= 0) {
|
|
700
|
+
return [];
|
|
701
|
+
}
|
|
702
|
+
const sanitized = sanitizeFtsQuery(trimmedQuery);
|
|
703
|
+
if (sanitized.length === 0) {
|
|
704
|
+
return [];
|
|
705
|
+
}
|
|
706
|
+
let rows = getSearchStatement(db, dateRange !== null).all(projectPath, Date.now(), ...dateRange === null ? [] : [dateRange.from, dateRange.to], sanitized, limit).filter(isMemoryRow2);
|
|
707
|
+
if (rows.length === 0) {
|
|
708
|
+
const relaxed = relaxedFtsQuery(trimmedQuery);
|
|
709
|
+
if (relaxed)
|
|
710
|
+
rows = getSearchStatement(db, dateRange !== null).all(projectPath, Date.now(), ...dateRange === null ? [] : [dateRange.from, dateRange.to], relaxed, limit).filter(isMemoryRow2);
|
|
711
|
+
}
|
|
712
|
+
return rows.map(toMemory2);
|
|
713
|
+
}
|
|
714
|
+
function searchMemoriesFTSUnion(db, projectPaths, query, limit = DEFAULT_SEARCH_LIMIT, ownIdentities, shareCategories, dateRange = null) {
|
|
715
|
+
const identities = uniqueProjectPaths(projectPaths);
|
|
716
|
+
if (identities.length === 0)
|
|
717
|
+
return [];
|
|
718
|
+
const sharingFilter = buildWorkspaceMemorySqlFilter2({
|
|
719
|
+
identities,
|
|
720
|
+
ownIdentities,
|
|
721
|
+
shareCategories,
|
|
722
|
+
tableName: "memories",
|
|
723
|
+
includeClassificationFields: (() => {
|
|
724
|
+
const columns = db.prepare("PRAGMA table_info(memories)").all();
|
|
725
|
+
return columns.some((row) => row.name === "shareable") && columns.some((row) => row.name === "scope");
|
|
726
|
+
})()
|
|
727
|
+
});
|
|
728
|
+
if (identities.length === 1 && !sharingFilter.active) {
|
|
729
|
+
return searchMemoriesFTS(db, identities[0], query, limit, dateRange);
|
|
730
|
+
}
|
|
731
|
+
const trimmedQuery = query.trim();
|
|
732
|
+
if (trimmedQuery.length === 0 || limit <= 0)
|
|
733
|
+
return [];
|
|
734
|
+
const sanitized = sanitizeFtsQuery(trimmedQuery);
|
|
735
|
+
if (sanitized.length === 0)
|
|
736
|
+
return [];
|
|
737
|
+
let rows = sharingFilter.active ? db.prepare(`SELECT ${getMemorySelectColumns2(db)} FROM memories_fts INNER JOIN memories ON memories.id = memories_fts.rowid WHERE memories.project_path IN (${identities.map(() => "?").join(", ")}) AND memories.status IN ('active', 'permanent') AND (memories.expires_at IS NULL OR memories.expires_at > ?)${dateRange === null ? "" : " AND memories.created_at BETWEEN ? AND ?"} AND memories_fts MATCH ?${sharingFilter.clause} ORDER BY bm25(memories_fts), memories.updated_at DESC, memories.id ASC LIMIT ?`).all(...identities, Date.now(), ...dateRange === null ? [] : [dateRange.from, dateRange.to], sanitized, ...sharingFilter.params, limit).filter(isMemoryRow2) : getUnionSearchStatement(db, identities.length, dateRange !== null).all(...identities, Date.now(), ...dateRange === null ? [] : [dateRange.from, dateRange.to], sanitized, limit).filter(isMemoryRow2);
|
|
738
|
+
if (rows.length === 0) {
|
|
739
|
+
const relaxed = relaxedFtsQuery(trimmedQuery);
|
|
740
|
+
if (relaxed) {
|
|
741
|
+
rows = (sharingFilter.active ? db.prepare(`SELECT ${getMemorySelectColumns2(db)} FROM memories_fts INNER JOIN memories ON memories.id = memories_fts.rowid WHERE memories.project_path IN (${identities.map(() => "?").join(", ")}) AND memories.status IN ('active', 'permanent') AND (memories.expires_at IS NULL OR memories.expires_at > ?)${dateRange === null ? "" : " AND memories.created_at BETWEEN ? AND ?"} AND memories_fts MATCH ?${sharingFilter.clause} ORDER BY bm25(memories_fts), memories.updated_at DESC, memories.id ASC LIMIT ?`) : getUnionSearchStatement(db, identities.length, dateRange !== null)).all(...identities, Date.now(), ...dateRange === null ? [] : [dateRange.from, dateRange.to], relaxed, ...sharingFilter.params, limit).filter(isMemoryRow2);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
return rows.map(toMemory2);
|
|
745
|
+
}
|
|
746
|
+
// ../plugin/src/features/magic-context/memory/storage-memory-verifications.ts
|
|
747
|
+
var MEMORY_VERIFICATION_SENTINEL = "";
|
|
748
|
+
function uniqueSortedFiles(files) {
|
|
749
|
+
return Array.from(new Set(files.filter((file) => file !== MEMORY_VERIFICATION_SENTINEL))).sort();
|
|
750
|
+
}
|
|
751
|
+
function recordMemoryMapping(db, memoryId, normalizedFiles, now, mappingOrigin = "mapper") {
|
|
752
|
+
const realFiles = uniqueSortedFiles(normalizedFiles);
|
|
753
|
+
const filesToWrite = realFiles.length > 0 ? realFiles : [MEMORY_VERIFICATION_SENTINEL];
|
|
754
|
+
db.prepare("DELETE FROM memory_verifications WHERE memory_id = ?").run(memoryId);
|
|
755
|
+
const insert = db.prepare("INSERT INTO memory_verifications (memory_id, file_path, verified_at, mapped_at, mapping_origin) VALUES (?, ?, 0, ?, ?)");
|
|
756
|
+
for (const file of filesToWrite) {
|
|
757
|
+
insert.run(memoryId, file, now, mappingOrigin);
|
|
758
|
+
}
|
|
759
|
+
return filesToWrite.length;
|
|
760
|
+
}
|
|
761
|
+
function recordMemoryVerifications(db, memoryId, normalizedFiles, now) {
|
|
762
|
+
const realFiles = uniqueSortedFiles(normalizedFiles);
|
|
763
|
+
const filesToWrite = realFiles.length > 0 ? realFiles : [MEMORY_VERIFICATION_SENTINEL];
|
|
764
|
+
db.prepare("DELETE FROM memory_verifications WHERE memory_id = ?").run(memoryId);
|
|
765
|
+
const insert = db.prepare("INSERT INTO memory_verifications (memory_id, file_path, verified_at, mapped_at, mapping_origin) VALUES (?, ?, ?, ?, 'mapper')");
|
|
766
|
+
for (const file of filesToWrite) {
|
|
767
|
+
insert.run(memoryId, file, now, now);
|
|
768
|
+
}
|
|
769
|
+
return filesToWrite.length;
|
|
770
|
+
}
|
|
771
|
+
function getUnmappedMemoryIds(db, memoryIds) {
|
|
772
|
+
const ids = Array.from(new Set(memoryIds.filter(Number.isInteger)));
|
|
773
|
+
if (ids.length === 0)
|
|
774
|
+
return [];
|
|
775
|
+
const rows = db.prepare("SELECT DISTINCT memory_id FROM memory_verifications WHERE memory_id IN (SELECT value FROM json_each(?))").all(JSON.stringify(ids));
|
|
776
|
+
const mapped = new Set(rows.map((r) => r.memory_id));
|
|
777
|
+
return ids.filter((id) => !mapped.has(id));
|
|
778
|
+
}
|
|
779
|
+
function clearMemoryVerifications(db, memoryId) {
|
|
780
|
+
db.prepare("DELETE FROM memory_verifications WHERE memory_id = ?").run(memoryId);
|
|
781
|
+
}
|
|
782
|
+
function getMemoryVerifications(db, memoryIds) {
|
|
783
|
+
const ids = Array.from(new Set(memoryIds.filter(Number.isInteger)));
|
|
784
|
+
const result = new Map;
|
|
785
|
+
if (ids.length === 0)
|
|
786
|
+
return result;
|
|
787
|
+
const rows = db.prepare(`SELECT memory_id, file_path, verified_at, mapped_at, mapping_origin
|
|
788
|
+
FROM memory_verifications
|
|
789
|
+
WHERE memory_id IN (SELECT value FROM json_each(?))
|
|
790
|
+
ORDER BY memory_id, file_path`).all(JSON.stringify(ids));
|
|
791
|
+
for (const row of rows) {
|
|
792
|
+
const existing = result.get(row.memory_id) ?? {
|
|
793
|
+
files: [],
|
|
794
|
+
hasSentinel: false,
|
|
795
|
+
mappingOrigin: "mapper",
|
|
796
|
+
verifiedAt: 0,
|
|
797
|
+
mappedAt: 0
|
|
798
|
+
};
|
|
799
|
+
if (row.file_path === MEMORY_VERIFICATION_SENTINEL) {
|
|
800
|
+
existing.hasSentinel = true;
|
|
801
|
+
} else if (!existing.files.includes(row.file_path)) {
|
|
802
|
+
existing.files.push(row.file_path);
|
|
803
|
+
}
|
|
804
|
+
if (row.mapping_origin === "host_rejected_fallback") {
|
|
805
|
+
existing.mappingOrigin = "host_rejected_fallback";
|
|
806
|
+
}
|
|
807
|
+
existing.verifiedAt = Math.max(existing.verifiedAt, row.verified_at);
|
|
808
|
+
existing.mappedAt = Math.max(existing.mappedAt, row.mapped_at ?? 0);
|
|
809
|
+
result.set(row.memory_id, existing);
|
|
810
|
+
}
|
|
811
|
+
for (const state of result.values()) {
|
|
812
|
+
state.files.sort();
|
|
813
|
+
}
|
|
814
|
+
return result;
|
|
815
|
+
}
|
|
816
|
+
// ../plugin/src/features/magic-context/memory/verification-paths.ts
|
|
817
|
+
import { execFile } from "node:child_process";
|
|
818
|
+
import { existsSync, realpathSync, statSync } from "node:fs";
|
|
819
|
+
import path from "node:path";
|
|
820
|
+
import { promisify } from "node:util";
|
|
821
|
+
var execFileAsync = promisify(execFile);
|
|
822
|
+
var GIT_TIMEOUT_MS = 1e4;
|
|
823
|
+
var defaultExecFileForVerificationPaths = async (file, args, options) => await execFileAsync(file, [...args], {
|
|
824
|
+
...options,
|
|
825
|
+
windowsHide: true
|
|
826
|
+
});
|
|
827
|
+
var execFileForVerificationPaths = defaultExecFileForVerificationPaths;
|
|
828
|
+
async function runGit(cwd, args) {
|
|
829
|
+
try {
|
|
830
|
+
const result = await execFileForVerificationPaths("git", args, {
|
|
831
|
+
cwd,
|
|
832
|
+
timeout: GIT_TIMEOUT_MS,
|
|
833
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
834
|
+
encoding: "utf8",
|
|
835
|
+
windowsHide: true
|
|
836
|
+
});
|
|
837
|
+
return String(result.stdout);
|
|
838
|
+
} catch (error) {
|
|
839
|
+
if (error.killed || error.code === "ETIMEDOUT") {
|
|
840
|
+
throw new Error(`Git verification command git ${args.join(" ")} timed out after ${GIT_TIMEOUT_MS}ms`, {
|
|
841
|
+
cause: error
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
return null;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
function toPosixPath(value) {
|
|
848
|
+
return value.split(path.sep).join("/");
|
|
849
|
+
}
|
|
850
|
+
function isWithin(root, candidate) {
|
|
851
|
+
const rel = path.relative(root, candidate);
|
|
852
|
+
return rel === "" || !rel.startsWith("..") && !path.isAbsolute(rel);
|
|
853
|
+
}
|
|
854
|
+
function safeRealpath(value) {
|
|
855
|
+
try {
|
|
856
|
+
return realpathSync.native(value);
|
|
857
|
+
} catch {
|
|
858
|
+
return null;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
async function resolveGitTopLevel(cwd) {
|
|
862
|
+
const stdout = await runGit(cwd, ["rev-parse", "--show-toplevel"]);
|
|
863
|
+
const root = stdout?.trim();
|
|
864
|
+
return root ? safeRealpath(root) ?? path.resolve(root) : null;
|
|
865
|
+
}
|
|
866
|
+
async function readGitHead(cwd) {
|
|
867
|
+
const stdout = await runGit(cwd, ["rev-parse", "HEAD"]);
|
|
868
|
+
const head = stdout?.trim();
|
|
869
|
+
return head && /^[0-9a-f]{40}$/i.test(head) ? head : null;
|
|
870
|
+
}
|
|
871
|
+
async function readGitChangedFilesSince(cwd, revision, knownGitRoot) {
|
|
872
|
+
if (!/^[0-9a-f]{7,40}$/i.test(revision))
|
|
873
|
+
return null;
|
|
874
|
+
const gitRoot = knownGitRoot ?? await resolveGitTopLevel(cwd);
|
|
875
|
+
if (!gitRoot)
|
|
876
|
+
return null;
|
|
877
|
+
const stdout = await runGit(gitRoot, ["diff", "--name-only", "-z", revision]);
|
|
878
|
+
if (stdout === null)
|
|
879
|
+
return null;
|
|
880
|
+
return new Set(stdout.split("\x00").filter(Boolean));
|
|
881
|
+
}
|
|
882
|
+
async function readGitFileChangeTimesSince(cwd, sinceMs, knownGitRoot) {
|
|
883
|
+
const gitRoot = knownGitRoot ?? await resolveGitTopLevel(cwd);
|
|
884
|
+
if (!gitRoot)
|
|
885
|
+
return null;
|
|
886
|
+
const sinceSec = Math.max(0, Math.floor(sinceMs / 1000));
|
|
887
|
+
const stdout = await runGit(gitRoot, [
|
|
888
|
+
"log",
|
|
889
|
+
`--since=@${sinceSec}`,
|
|
890
|
+
"--name-only",
|
|
891
|
+
"--format=%ct"
|
|
892
|
+
]);
|
|
893
|
+
if (stdout === null)
|
|
894
|
+
return null;
|
|
895
|
+
const times = new Map;
|
|
896
|
+
let currentMs = 0;
|
|
897
|
+
for (const rawLine of stdout.split(`
|
|
898
|
+
`)) {
|
|
899
|
+
const line = rawLine.trimEnd();
|
|
900
|
+
if (line === "")
|
|
901
|
+
continue;
|
|
902
|
+
if (/^\d+$/.test(line)) {
|
|
903
|
+
currentMs = Number.parseInt(line, 10) * 1000;
|
|
904
|
+
continue;
|
|
905
|
+
}
|
|
906
|
+
if (currentMs > 0 && !times.has(line)) {
|
|
907
|
+
times.set(line, currentMs);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return times;
|
|
911
|
+
}
|
|
912
|
+
async function gitTrackedPath(gitRoot, repoRelativePath) {
|
|
913
|
+
const stdout = await runGit(gitRoot, [
|
|
914
|
+
"ls-files",
|
|
915
|
+
"-z",
|
|
916
|
+
"--full-name",
|
|
917
|
+
"--error-unmatch",
|
|
918
|
+
"--",
|
|
919
|
+
repoRelativePath
|
|
920
|
+
]);
|
|
921
|
+
const fallbackStdout = stdout === null ? await runGit(gitRoot, ["ls-files", "-z", "--full-name"]) : null;
|
|
922
|
+
if (stdout === null && fallbackStdout === null)
|
|
923
|
+
return null;
|
|
924
|
+
const matches = (stdout ?? fallbackStdout ?? "").split("\x00").filter(Boolean);
|
|
925
|
+
if (matches.length === 0)
|
|
926
|
+
return null;
|
|
927
|
+
return matches.find((match) => match === repoRelativePath) ?? matches.find((match) => match.toLowerCase() === repoRelativePath.toLowerCase()) ?? (matches.length === 1 ? matches[0] : null);
|
|
928
|
+
}
|
|
929
|
+
function verificationFileExists(baseRoot, filePath) {
|
|
930
|
+
if (!filePath || filePath === ".")
|
|
931
|
+
return false;
|
|
932
|
+
const root = path.resolve(baseRoot);
|
|
933
|
+
const candidate = path.resolve(root, filePath);
|
|
934
|
+
return isWithin(root, candidate) && existsSync(candidate);
|
|
935
|
+
}
|
|
936
|
+
async function normalizeVerificationFiles(args) {
|
|
937
|
+
const cwd = path.resolve(args.cwd);
|
|
938
|
+
const gitRoot = await resolveGitTopLevel(cwd);
|
|
939
|
+
const root = gitRoot ?? cwd;
|
|
940
|
+
const rootReal = safeRealpath(root) ?? root;
|
|
941
|
+
const warnings = [];
|
|
942
|
+
const normalized = [];
|
|
943
|
+
let trackedPaths;
|
|
944
|
+
for (const raw of args.files) {
|
|
945
|
+
const value = typeof raw === "string" ? raw.trim() : "";
|
|
946
|
+
if (!value) {
|
|
947
|
+
warnings.push("Skipped blank verification path.");
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
if (value === ".") {
|
|
951
|
+
warnings.push('Skipped verification path "." (repo/project root is not a file).');
|
|
952
|
+
continue;
|
|
953
|
+
}
|
|
954
|
+
const candidate = path.resolve(cwd, value);
|
|
955
|
+
const candidateReal = safeRealpath(candidate);
|
|
956
|
+
if (candidateReal && !isWithin(rootReal, candidateReal)) {
|
|
957
|
+
warnings.push(`Skipped verification path "${value}" because it resolves outside the project.`);
|
|
958
|
+
continue;
|
|
959
|
+
}
|
|
960
|
+
if (!candidateReal && !isWithin(path.resolve(root), candidate)) {
|
|
961
|
+
warnings.push(`Skipped verification path "${value}" because it escapes the project.`);
|
|
962
|
+
continue;
|
|
963
|
+
}
|
|
964
|
+
if (path.resolve(root) === candidate) {
|
|
965
|
+
warnings.push(`Skipped verification path "${value}" because it is the repo/project root.`);
|
|
966
|
+
continue;
|
|
967
|
+
}
|
|
968
|
+
if (existsSync(candidate)) {
|
|
969
|
+
try {
|
|
970
|
+
if (statSync(candidate).isDirectory()) {
|
|
971
|
+
warnings.push(`Skipped verification path "${value}" because it is a directory.`);
|
|
972
|
+
continue;
|
|
973
|
+
}
|
|
974
|
+
} catch {
|
|
975
|
+
warnings.push(`Skipped verification path "${value}" because it could not be inspected.`);
|
|
976
|
+
continue;
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
if (gitRoot) {
|
|
980
|
+
const repoRelative = toPosixPath(path.relative(gitRoot, candidateReal ?? candidate));
|
|
981
|
+
if (!repoRelative || repoRelative === "." || repoRelative.startsWith("../")) {
|
|
982
|
+
warnings.push(`Skipped verification path "${value}" because it is not inside the git repo.`);
|
|
983
|
+
continue;
|
|
984
|
+
}
|
|
985
|
+
if (!trackedPaths) {
|
|
986
|
+
let trackedOutput = null;
|
|
987
|
+
try {
|
|
988
|
+
trackedOutput = await runGit(gitRoot, ["ls-files", "-z", "--full-name"]);
|
|
989
|
+
} catch {}
|
|
990
|
+
trackedPaths = new Set(trackedOutput?.split("\x00").filter(Boolean) ?? []);
|
|
991
|
+
}
|
|
992
|
+
const tracked = trackedPaths.has(repoRelative) ? repoRelative : await gitTrackedPath(gitRoot, repoRelative);
|
|
993
|
+
if (!tracked) {
|
|
994
|
+
warnings.push(`Skipped verification path "${value}" because it is not a tracked git file.`);
|
|
995
|
+
continue;
|
|
996
|
+
}
|
|
997
|
+
if (candidateReal && tracked !== repoRelative) {
|
|
998
|
+
const realRelative = toPosixPath(path.relative(gitRoot, candidateReal));
|
|
999
|
+
if (realRelative !== tracked) {
|
|
1000
|
+
warnings.push(`Skipped verification path "${value}" because it is not a tracked git file.`);
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
normalized.push(tracked);
|
|
1005
|
+
} else {
|
|
1006
|
+
if (!existsSync(candidate)) {
|
|
1007
|
+
warnings.push(`Skipped verification path "${value}" because it does not exist.`);
|
|
1008
|
+
continue;
|
|
1009
|
+
}
|
|
1010
|
+
const projectRelative = toPosixPath(path.relative(cwd, candidate));
|
|
1011
|
+
if (!projectRelative || projectRelative === "." || projectRelative.startsWith("../")) {
|
|
1012
|
+
warnings.push(`Skipped verification path "${value}" because it is not inside the project.`);
|
|
1013
|
+
continue;
|
|
1014
|
+
}
|
|
1015
|
+
normalized.push(projectRelative);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
return {
|
|
1019
|
+
files: Array.from(new Set(normalized)).sort(),
|
|
1020
|
+
warnings,
|
|
1021
|
+
gitRoot
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
// ../plugin/src/features/magic-context/git-commits/git-log-reader.ts
|
|
1025
|
+
import { execFile as execFile2 } from "node:child_process";
|
|
1026
|
+
import { promisify as promisify2 } from "node:util";
|
|
1027
|
+
var execFileAsync2 = promisify2(execFile2);
|
|
1028
|
+
var GIT_TIMEOUT_MS2 = 1e4;
|
|
1029
|
+
var DEFAULT_MAX_COMMITS = 5000;
|
|
1030
|
+
var RECORD_SEPARATOR = "\x1E";
|
|
1031
|
+
var FIELD_SEPARATOR = "\x1F";
|
|
1032
|
+
var FULL_OBJECT_NAME = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
|
|
1033
|
+
function classifyGitLogFailure(message) {
|
|
1034
|
+
if (message.includes("not a git repository"))
|
|
1035
|
+
return "not_a_repo";
|
|
1036
|
+
if (message.includes("unknown revision or path not in the working tree") || message.includes("does not have any commits yet") || message.includes("bad revision")) {
|
|
1037
|
+
return "no_head";
|
|
1038
|
+
}
|
|
1039
|
+
return "transient";
|
|
1040
|
+
}
|
|
1041
|
+
async function readGitCommitsResult(directory, options = {}) {
|
|
1042
|
+
const revision = options.branch ?? "HEAD";
|
|
1043
|
+
if (revision.startsWith("-")) {
|
|
1044
|
+
throw new Error(`readGitCommits: refusing revision that looks like an option: "${revision}"`);
|
|
1045
|
+
}
|
|
1046
|
+
const projectLabel = options.projectIdentity ?? "<project>";
|
|
1047
|
+
const exclusion = options.excludeReachableFrom && FULL_OBJECT_NAME.test(options.excludeReachableFrom) ? options.excludeReachableFrom : null;
|
|
1048
|
+
const args = [
|
|
1049
|
+
"log",
|
|
1050
|
+
revision,
|
|
1051
|
+
...exclusion ? [`^${exclusion}`] : [],
|
|
1052
|
+
"--no-merges",
|
|
1053
|
+
`--max-count=${options.maxCommits ?? DEFAULT_MAX_COMMITS}`,
|
|
1054
|
+
`--format=%H${FIELD_SEPARATOR}%s${FIELD_SEPARATOR}%ae${FIELD_SEPARATOR}%ct${FIELD_SEPARATOR}%b${RECORD_SEPARATOR}`
|
|
1055
|
+
];
|
|
1056
|
+
if (options.sinceMs !== undefined && options.sinceMs > 0) {
|
|
1057
|
+
const iso = new Date(options.sinceMs).toISOString();
|
|
1058
|
+
args.push(`--since=${iso}`);
|
|
1059
|
+
}
|
|
1060
|
+
args.push("--");
|
|
1061
|
+
let stdout;
|
|
1062
|
+
try {
|
|
1063
|
+
const result = await execFileAsync2("git", args, {
|
|
1064
|
+
windowsHide: true,
|
|
1065
|
+
cwd: directory,
|
|
1066
|
+
timeout: GIT_TIMEOUT_MS2,
|
|
1067
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
1068
|
+
encoding: "utf8"
|
|
1069
|
+
});
|
|
1070
|
+
stdout = result.stdout;
|
|
1071
|
+
} catch (error) {
|
|
1072
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1073
|
+
if (exclusion && message.includes("bad object")) {
|
|
1074
|
+
log2(`[git-commits] indexed tip ${exclusion.slice(0, 7)} is missing for ${projectLabel}; reading the full window`);
|
|
1075
|
+
return readGitCommitsResult(directory, { ...options, excludeReachableFrom: undefined });
|
|
1076
|
+
}
|
|
1077
|
+
const failure = classifyGitLogFailure(message);
|
|
1078
|
+
if (failure === "transient") {
|
|
1079
|
+
log2(`[git-commits] readGitCommits failed for ${projectLabel}: ${message.slice(0, 500)}`);
|
|
1080
|
+
} else {
|
|
1081
|
+
log2(`[git-commits] ${projectLabel} is not indexable (${failure})`);
|
|
1082
|
+
}
|
|
1083
|
+
return { commits: [], failure };
|
|
1084
|
+
}
|
|
1085
|
+
if (stdout.trim().length === 0) {
|
|
1086
|
+
log2(`[git-commits] readGitCommits returned empty stdout for ${projectLabel} (sinceMs=${options.sinceMs ?? "none"} args=${args.slice(0, 4).join(" ")})`);
|
|
1087
|
+
}
|
|
1088
|
+
return { commits: parseGitLogOutput(stdout), failure: null };
|
|
1089
|
+
}
|
|
1090
|
+
function parseGitLogOutput(stdout) {
|
|
1091
|
+
const commits = [];
|
|
1092
|
+
const records = stdout.split(RECORD_SEPARATOR);
|
|
1093
|
+
for (const rawRecord of records) {
|
|
1094
|
+
const record = rawRecord.replace(/^\s+/, "");
|
|
1095
|
+
if (!record)
|
|
1096
|
+
continue;
|
|
1097
|
+
const fields = [];
|
|
1098
|
+
let remaining = record;
|
|
1099
|
+
for (let i = 0;i < 4; i++) {
|
|
1100
|
+
const idx = remaining.indexOf(FIELD_SEPARATOR);
|
|
1101
|
+
if (idx < 0)
|
|
1102
|
+
break;
|
|
1103
|
+
fields.push(remaining.slice(0, idx));
|
|
1104
|
+
remaining = remaining.slice(idx + FIELD_SEPARATOR.length);
|
|
1105
|
+
}
|
|
1106
|
+
fields.push(remaining);
|
|
1107
|
+
if (fields.length < 5)
|
|
1108
|
+
continue;
|
|
1109
|
+
const sha = fields[0].trim();
|
|
1110
|
+
const subject = fields[1].trim();
|
|
1111
|
+
const author = fields[2].trim();
|
|
1112
|
+
const timeSec = Number.parseInt(fields[3].trim(), 10);
|
|
1113
|
+
const body = fields[4].trim();
|
|
1114
|
+
if (!FULL_OBJECT_NAME.test(sha) || !Number.isFinite(timeSec) || timeSec <= 0) {
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1117
|
+
const message = body.length > 0 ? `${subject}
|
|
1118
|
+
|
|
1119
|
+
${body}` : subject;
|
|
1120
|
+
commits.push({
|
|
1121
|
+
sha,
|
|
1122
|
+
shortSha: sha.slice(0, 7),
|
|
1123
|
+
message,
|
|
1124
|
+
author: author.length > 0 ? author : null,
|
|
1125
|
+
committedAtMs: timeSec * 1000
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
1128
|
+
return commits;
|
|
1129
|
+
}
|
|
1130
|
+
// ../plugin/src/features/magic-context/git-commits/indexer.ts
|
|
1131
|
+
var MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
1132
|
+
var EMBED_BATCH_SIZE = 16;
|
|
1133
|
+
var EMBED_MAX_PER_SWEEP = 500;
|
|
1134
|
+
var EMBED_SWEEP_MAX_WALL_CLOCK_MS = 5 * 60 * 1000;
|
|
1135
|
+
var indexInProgress = new Set;
|
|
1136
|
+
var embedInProgress = new Set;
|
|
1137
|
+
async function indexCommitsForProject(db, projectPath, directory, options) {
|
|
1138
|
+
const result = {
|
|
1139
|
+
scanned: 0,
|
|
1140
|
+
inserted: 0,
|
|
1141
|
+
updated: 0,
|
|
1142
|
+
evicted: 0,
|
|
1143
|
+
embedded: 0,
|
|
1144
|
+
nonIndexable: false
|
|
1145
|
+
};
|
|
1146
|
+
if (indexInProgress.has(projectPath)) {
|
|
1147
|
+
log2(`[git-commits] index already in progress for ${projectPath}, skipping`);
|
|
1148
|
+
return result;
|
|
1149
|
+
}
|
|
1150
|
+
indexInProgress.add(projectPath);
|
|
1151
|
+
try {
|
|
1152
|
+
const latestIndexed = getLatestIndexedCommitSha(db, projectPath);
|
|
1153
|
+
const sinceMs = Date.now() - options.sinceDays * MS_PER_DAY;
|
|
1154
|
+
const read = await readGitCommitsResult(directory, {
|
|
1155
|
+
sinceMs,
|
|
1156
|
+
maxCommits: options.maxCommits,
|
|
1157
|
+
projectIdentity: projectPath,
|
|
1158
|
+
excludeReachableFrom: latestIndexed ?? undefined
|
|
1159
|
+
});
|
|
1160
|
+
const commits = read.commits;
|
|
1161
|
+
result.scanned = commits.length;
|
|
1162
|
+
if (read.failure === "not_a_repo" || read.failure === "no_head") {
|
|
1163
|
+
result.nonIndexable = true;
|
|
1164
|
+
return result;
|
|
1165
|
+
}
|
|
1166
|
+
if (commits.length === 0) {
|
|
1167
|
+
result.evicted = enforceProjectCap(db, projectPath, options.maxCommits);
|
|
1168
|
+
log2(`[git-commits] no new commits for ${projectPath} (sinceMs=${sinceMs} latestIndexed=${latestIndexed?.slice(0, 7) ?? "none"} evicted=${result.evicted})`);
|
|
1169
|
+
return result;
|
|
1170
|
+
}
|
|
1171
|
+
log2(`[git-commits] read ${commits.length} commits for ${projectPath} (sinceMs=${sinceMs} latestIndexed=${latestIndexed?.slice(0, 7) ?? "none"})`);
|
|
1172
|
+
const upsert = upsertCommits(db, projectPath, commits);
|
|
1173
|
+
result.inserted = upsert.inserted;
|
|
1174
|
+
result.updated = upsert.updated;
|
|
1175
|
+
result.evicted = enforceProjectCap(db, projectPath, options.maxCommits);
|
|
1176
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectPath);
|
|
1177
|
+
if (options.skipEmbed || !snapshot?.gitCommitEnabled) {
|
|
1178
|
+
log2(`[git-commits] indexed ${projectPath}: scanned=${result.scanned} inserted=${result.inserted} updated=${result.updated} evicted=${result.evicted} embedded=0 (embedding skipped: skipEmbed=${options.skipEmbed === true} gitCommitEnabled=${snapshot?.gitCommitEnabled === true})`);
|
|
1179
|
+
return result;
|
|
1180
|
+
}
|
|
1181
|
+
result.embedded = await embedUnembeddedCommits(db, projectPath);
|
|
1182
|
+
log2(`[git-commits] indexed ${projectPath}: scanned=${result.scanned} inserted=${result.inserted} updated=${result.updated} evicted=${result.evicted} embedded=${result.embedded}`);
|
|
1183
|
+
return result;
|
|
1184
|
+
} finally {
|
|
1185
|
+
indexInProgress.delete(projectPath);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
async function embedUnembeddedCommits(db, projectPath) {
|
|
1189
|
+
if (embedInProgress.has(projectPath)) {
|
|
1190
|
+
return 0;
|
|
1191
|
+
}
|
|
1192
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectPath);
|
|
1193
|
+
if (!snapshot?.gitCommitEnabled) {
|
|
1194
|
+
return 0;
|
|
1195
|
+
}
|
|
1196
|
+
embedInProgress.add(projectPath);
|
|
1197
|
+
const startedAt = Date.now();
|
|
1198
|
+
const deadline = startedAt + EMBED_SWEEP_MAX_WALL_CLOCK_MS;
|
|
1199
|
+
let total = 0;
|
|
1200
|
+
try {
|
|
1201
|
+
while (Date.now() < deadline && total < EMBED_MAX_PER_SWEEP) {
|
|
1202
|
+
const rows = loadUnembeddedCommits(db, projectPath, snapshot.modelId, EMBED_BATCH_SIZE);
|
|
1203
|
+
if (rows.length === 0)
|
|
1204
|
+
break;
|
|
1205
|
+
let embeddedThisBatch = 0;
|
|
1206
|
+
try {
|
|
1207
|
+
const result = await embedItemsForProject2(projectPath, rows.map((row) => ({
|
|
1208
|
+
id: `commit:${row.sha}`,
|
|
1209
|
+
text: row.message,
|
|
1210
|
+
contentSha256: contentSha2562(row.message)
|
|
1211
|
+
})), undefined, db, projectPath);
|
|
1212
|
+
if (!result)
|
|
1213
|
+
break;
|
|
1214
|
+
db.transaction(() => {
|
|
1215
|
+
for (const row of rows) {
|
|
1216
|
+
const embedding = result.vectors.get(`commit:${row.sha}`);
|
|
1217
|
+
if (!embedding)
|
|
1218
|
+
continue;
|
|
1219
|
+
saveCommitEmbedding(db, row.sha, embedding, result.modelId);
|
|
1220
|
+
embeddedThisBatch += 1;
|
|
1221
|
+
}
|
|
1222
|
+
}).immediate();
|
|
1223
|
+
enqueueShadowEmbeddingItems2(projectPath, "commit", rows.filter((row) => result.vectors.has(`commit:${row.sha}`)).map((row) => row.sha));
|
|
1224
|
+
} catch (error) {
|
|
1225
|
+
log2(`[git-commits] embed batch failed for ${projectPath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1226
|
+
break;
|
|
1227
|
+
}
|
|
1228
|
+
if (embeddedThisBatch === 0)
|
|
1229
|
+
break;
|
|
1230
|
+
total += embeddedThisBatch;
|
|
1231
|
+
if (embeddedThisBatch < rows.length)
|
|
1232
|
+
break;
|
|
1233
|
+
}
|
|
1234
|
+
if (total > 0) {
|
|
1235
|
+
const totalEmbedded = countEmbeddedCommits(db, projectPath, snapshot.modelId);
|
|
1236
|
+
log2(`[git-commits] embedded ${total} commits for ${projectPath} (total embedded: ${totalEmbedded})`);
|
|
1237
|
+
}
|
|
1238
|
+
return total;
|
|
1239
|
+
} finally {
|
|
1240
|
+
embedInProgress.delete(projectPath);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
// ../plugin/src/features/magic-context/git-commits/search-git-commits.ts
|
|
1244
|
+
var ftsStatements = new WeakMap;
|
|
1245
|
+
var datedFtsStatements = new WeakMap;
|
|
1246
|
+
var ftsPlainStatements = new WeakMap;
|
|
1247
|
+
var datedFtsPlainStatements = new WeakMap;
|
|
1248
|
+
var getBySHAsStatements = new WeakMap;
|
|
1249
|
+
var datedGetBySHAsStatements = new WeakMap;
|
|
1250
|
+
var embeddingCache = new WeakMap;
|
|
1251
|
+
function getSearchEmbeddings(db, projectPath, modelId) {
|
|
1252
|
+
if (!isKnownAutocommit(db)) {
|
|
1253
|
+
embeddingCache.delete(db);
|
|
1254
|
+
return loadProjectCommitEmbeddings(db, projectPath, modelId);
|
|
1255
|
+
}
|
|
1256
|
+
let cache = embeddingCache.get(db);
|
|
1257
|
+
if (!cache) {
|
|
1258
|
+
cache = {
|
|
1259
|
+
revisionStatement: db.prepare(`SELECT total_changes() AS writes,
|
|
1260
|
+
(SELECT data_version FROM pragma_data_version) AS dataVersion,
|
|
1261
|
+
(SELECT schema_version FROM pragma_schema_version) AS schemaVersion`),
|
|
1262
|
+
revision: "",
|
|
1263
|
+
entries: new Map
|
|
1264
|
+
};
|
|
1265
|
+
embeddingCache.set(db, cache);
|
|
1266
|
+
}
|
|
1267
|
+
const revision = JSON.stringify(cache.revisionStatement.get());
|
|
1268
|
+
if (revision !== cache.revision) {
|
|
1269
|
+
cache.entries.clear();
|
|
1270
|
+
cache.revision = revision;
|
|
1271
|
+
}
|
|
1272
|
+
const key = JSON.stringify([projectPath, modelId]);
|
|
1273
|
+
const cached = cache.entries.get(key);
|
|
1274
|
+
if (cached) {
|
|
1275
|
+
cache.entries.delete(key);
|
|
1276
|
+
cache.entries.set(key, cached);
|
|
1277
|
+
return cached;
|
|
1278
|
+
}
|
|
1279
|
+
const vectors = loadProjectCommitEmbeddings(db, projectPath, modelId);
|
|
1280
|
+
const size = (entries) => [...entries.values()].reduce((sum, vector) => sum + vector.byteLength, 0);
|
|
1281
|
+
if (size(vectors) <= 32 * 1024 * 1024)
|
|
1282
|
+
cache.entries.set(key, vectors);
|
|
1283
|
+
while (cache.entries.size > 16 || [...cache.entries.values()].reduce((sum, entry) => sum + size(entry), 0) > 32 * 1024 * 1024) {
|
|
1284
|
+
const oldest = cache.entries.keys().next().value;
|
|
1285
|
+
if (oldest === undefined)
|
|
1286
|
+
break;
|
|
1287
|
+
cache.entries.delete(oldest);
|
|
1288
|
+
}
|
|
1289
|
+
return vectors;
|
|
1290
|
+
}
|
|
1291
|
+
function rowToCommit(row) {
|
|
1292
|
+
return {
|
|
1293
|
+
sha: row.sha,
|
|
1294
|
+
shortSha: row.short_sha,
|
|
1295
|
+
projectPath: row.project_path,
|
|
1296
|
+
message: row.message,
|
|
1297
|
+
author: row.author,
|
|
1298
|
+
committedAtMs: row.committed_at,
|
|
1299
|
+
indexedAtMs: row.indexed_at
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
function getFtsStatement(db, dated = false) {
|
|
1303
|
+
const statements = dated ? datedFtsStatements : ftsStatements;
|
|
1304
|
+
let stmt = statements.get(db);
|
|
1305
|
+
if (!stmt) {
|
|
1306
|
+
stmt = db.prepare(`SELECT c.sha AS sha, c.project_path AS project_path, c.short_sha AS short_sha,
|
|
1307
|
+
c.message AS message, c.author AS author,
|
|
1308
|
+
c.committed_at AS committed_at, c.indexed_at AS indexed_at
|
|
1309
|
+
FROM git_commits_fts
|
|
1310
|
+
INNER JOIN git_commits c ON c.sha = git_commits_fts.sha
|
|
1311
|
+
WHERE c.project_path = ?
|
|
1312
|
+
${dated ? "AND c.committed_at BETWEEN ? AND ?" : ""}
|
|
1313
|
+
AND git_commits_fts MATCH ?
|
|
1314
|
+
ORDER BY bm25(git_commits_fts) LIMIT ?`);
|
|
1315
|
+
statements.set(db, stmt);
|
|
1316
|
+
}
|
|
1317
|
+
return stmt;
|
|
1318
|
+
}
|
|
1319
|
+
function escapeLikePattern(text) {
|
|
1320
|
+
return text.replace(/[\\%_]/g, (char) => `\\${char}`);
|
|
1321
|
+
}
|
|
1322
|
+
function getLikeFallbackStatement(db, dated = false) {
|
|
1323
|
+
const statements = dated ? datedFtsPlainStatements : ftsPlainStatements;
|
|
1324
|
+
let stmt = statements.get(db);
|
|
1325
|
+
if (!stmt) {
|
|
1326
|
+
stmt = db.prepare(`SELECT sha, project_path, short_sha, message, author, committed_at, indexed_at
|
|
1327
|
+
FROM git_commits
|
|
1328
|
+
WHERE project_path = ?
|
|
1329
|
+
${dated ? "AND committed_at BETWEEN ? AND ?" : ""}
|
|
1330
|
+
AND lower(message) LIKE '%' || lower(?) || '%' ESCAPE '\\'
|
|
1331
|
+
ORDER BY committed_at DESC LIMIT ?`);
|
|
1332
|
+
statements.set(db, stmt);
|
|
1333
|
+
}
|
|
1334
|
+
return stmt;
|
|
1335
|
+
}
|
|
1336
|
+
function getBySHAsStatement(db, dated = false) {
|
|
1337
|
+
const statements = dated ? datedGetBySHAsStatements : getBySHAsStatements;
|
|
1338
|
+
let stmt = statements.get(db);
|
|
1339
|
+
if (!stmt) {
|
|
1340
|
+
stmt = db.prepare(`SELECT sha, project_path, short_sha, message, author, committed_at, indexed_at
|
|
1341
|
+
FROM git_commits
|
|
1342
|
+
WHERE project_path = ?
|
|
1343
|
+
${dated ? "AND committed_at BETWEEN ? AND ?" : ""}
|
|
1344
|
+
AND sha IN (SELECT value FROM json_each(?))`);
|
|
1345
|
+
statements.set(db, stmt);
|
|
1346
|
+
}
|
|
1347
|
+
return stmt;
|
|
1348
|
+
}
|
|
1349
|
+
function clamp01(value) {
|
|
1350
|
+
if (!Number.isFinite(value))
|
|
1351
|
+
return 0;
|
|
1352
|
+
return Math.min(1, Math.max(0, value));
|
|
1353
|
+
}
|
|
1354
|
+
function searchGitCommitsSync(db, projectPath, query, options) {
|
|
1355
|
+
const trimmed = query.trim();
|
|
1356
|
+
if (trimmed.length === 0 || options.limit <= 0)
|
|
1357
|
+
return [];
|
|
1358
|
+
const semanticWeight = options.semanticWeight ?? 0.7;
|
|
1359
|
+
const ftsWeight = options.ftsWeight ?? 0.3;
|
|
1360
|
+
const singleSourcePenalty = options.singleSourcePenalty ?? 0.8;
|
|
1361
|
+
const fetchLimit = Math.max(options.limit * 3, 30);
|
|
1362
|
+
const dated = options.from !== undefined || options.to !== undefined;
|
|
1363
|
+
const from = options.from ?? Number.MIN_SAFE_INTEGER;
|
|
1364
|
+
const to = options.to ?? Number.MAX_SAFE_INTEGER;
|
|
1365
|
+
const ftsCandidates = [];
|
|
1366
|
+
const sanitized = sanitizeFtsQuery(trimmed);
|
|
1367
|
+
if (sanitized.length > 0) {
|
|
1368
|
+
try {
|
|
1369
|
+
for (const row of getFtsStatement(db, dated).all(projectPath, ...dated ? [from, to] : [], sanitized, fetchLimit)) {
|
|
1370
|
+
ftsCandidates.push(rowToCommit(row));
|
|
1371
|
+
}
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
log2(`[git-commits] FTS query failed for "${trimmed}": ${error instanceof Error ? error.message : String(error)}`);
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
if (ftsCandidates.length === 0) {
|
|
1377
|
+
for (const row of getLikeFallbackStatement(db, dated).all(projectPath, ...dated ? [from, to] : [], escapeLikePattern(trimmed), fetchLimit)) {
|
|
1378
|
+
ftsCandidates.push(rowToCommit(row));
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
const ftsScores = new Map;
|
|
1382
|
+
ftsCandidates.forEach((commit, rank) => {
|
|
1383
|
+
ftsScores.set(commit.sha, 1 / (rank + 1));
|
|
1384
|
+
});
|
|
1385
|
+
const semanticScores = new Map;
|
|
1386
|
+
if (options.queryEmbedding && options.queryModelId && options.queryModelId !== "off") {
|
|
1387
|
+
const embeddings = getSearchEmbeddings(db, projectPath, options.queryModelId);
|
|
1388
|
+
for (const [sha, embedding] of embeddings.entries()) {
|
|
1389
|
+
const similarity = clamp01(cosineSimilarity(options.queryEmbedding, embedding));
|
|
1390
|
+
if (similarity > 0) {
|
|
1391
|
+
semanticScores.set(sha, similarity);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
const bySha = new Map;
|
|
1396
|
+
for (const commit of ftsCandidates)
|
|
1397
|
+
bySha.set(commit.sha, commit);
|
|
1398
|
+
const semanticOnlyShas = [...semanticScores.keys()].filter((sha) => !bySha.has(sha));
|
|
1399
|
+
if (semanticOnlyShas.length > 0) {
|
|
1400
|
+
const rows = getBySHAsStatement(db, dated).all(projectPath, ...dated ? [from, to] : [], JSON.stringify(semanticOnlyShas));
|
|
1401
|
+
for (const row of rows)
|
|
1402
|
+
bySha.set(row.sha, rowToCommit(row));
|
|
1403
|
+
}
|
|
1404
|
+
const results = [];
|
|
1405
|
+
for (const [sha, commit] of bySha.entries()) {
|
|
1406
|
+
const sem = semanticScores.get(sha);
|
|
1407
|
+
const fts = ftsScores.get(sha);
|
|
1408
|
+
let score = 0;
|
|
1409
|
+
let matchType = "fts";
|
|
1410
|
+
if (sem !== undefined && fts !== undefined) {
|
|
1411
|
+
score = semanticWeight * sem + ftsWeight * fts;
|
|
1412
|
+
matchType = "hybrid";
|
|
1413
|
+
} else if (sem !== undefined) {
|
|
1414
|
+
score = sem * singleSourcePenalty;
|
|
1415
|
+
matchType = "semantic";
|
|
1416
|
+
} else if (fts !== undefined) {
|
|
1417
|
+
score = fts * singleSourcePenalty;
|
|
1418
|
+
matchType = "fts";
|
|
1419
|
+
}
|
|
1420
|
+
if (score <= 0)
|
|
1421
|
+
continue;
|
|
1422
|
+
results.push({ commit, score, matchType });
|
|
1423
|
+
}
|
|
1424
|
+
results.sort((left, right) => {
|
|
1425
|
+
if (right.score !== left.score)
|
|
1426
|
+
return right.score - left.score;
|
|
1427
|
+
return right.commit.committedAtMs - left.commit.committedAtMs;
|
|
1428
|
+
});
|
|
1429
|
+
return results.slice(0, options.limit);
|
|
1430
|
+
}
|
|
1431
|
+
// ../plugin/src/features/magic-context/literal-probes.ts
|
|
1432
|
+
var MAX_PROBES = 5;
|
|
1433
|
+
var MIN_PROBE_LENGTH = 3;
|
|
1434
|
+
var SLASH_COMMAND_RE = /\/[a-z][a-z0-9]*(?:-[a-z0-9]+)+/gi;
|
|
1435
|
+
var KEBAB_SNAKE_RE = /[a-z][a-z0-9]*(?:[-_][a-z0-9]+)+/gi;
|
|
1436
|
+
var DOTTED_RE = /[a-z0-9][a-z0-9_-]*(?:\.[a-z0-9_-]+)+/gi;
|
|
1437
|
+
var CAMEL_RE = /\b[a-zA-Z][a-z0-9]*(?:[A-Z][a-z0-9]*)+\b/g;
|
|
1438
|
+
var SHA_RE = /\b[0-9a-f]{7,40}\b/gi;
|
|
1439
|
+
var ERROR_CODE_RE = /\b(?:TS\d{4,}|ERR_[A-Z][A-Z0-9_]*)\b/g;
|
|
1440
|
+
var QUOTED_RE = /["`]([^"`]{3,80})["`]/g;
|
|
1441
|
+
function looksLikeSha(token) {
|
|
1442
|
+
return /[0-9]/.test(token) && /^[0-9a-f]{7,40}$/i.test(token);
|
|
1443
|
+
}
|
|
1444
|
+
function extractLiteralProbes(query) {
|
|
1445
|
+
const trimmed = query.trim();
|
|
1446
|
+
if (trimmed.length === 0)
|
|
1447
|
+
return [];
|
|
1448
|
+
const ordered = [];
|
|
1449
|
+
const seen = new Set;
|
|
1450
|
+
const add = (raw) => {
|
|
1451
|
+
if (!raw)
|
|
1452
|
+
return;
|
|
1453
|
+
const probe = raw.trim();
|
|
1454
|
+
if (probe.length < MIN_PROBE_LENGTH)
|
|
1455
|
+
return;
|
|
1456
|
+
const key = probe.toLowerCase();
|
|
1457
|
+
if (seen.has(key))
|
|
1458
|
+
return;
|
|
1459
|
+
seen.add(key);
|
|
1460
|
+
ordered.push(probe);
|
|
1461
|
+
};
|
|
1462
|
+
for (const m of trimmed.matchAll(QUOTED_RE))
|
|
1463
|
+
add(m[1]);
|
|
1464
|
+
for (const m of trimmed.matchAll(SLASH_COMMAND_RE))
|
|
1465
|
+
add(m[0]);
|
|
1466
|
+
for (const m of trimmed.matchAll(ERROR_CODE_RE))
|
|
1467
|
+
add(m[0]);
|
|
1468
|
+
for (const m of trimmed.matchAll(DOTTED_RE))
|
|
1469
|
+
add(m[0]);
|
|
1470
|
+
for (const m of trimmed.matchAll(KEBAB_SNAKE_RE))
|
|
1471
|
+
add(m[0]);
|
|
1472
|
+
for (const m of trimmed.matchAll(CAMEL_RE))
|
|
1473
|
+
add(m[0]);
|
|
1474
|
+
for (const m of trimmed.matchAll(SHA_RE)) {
|
|
1475
|
+
if (looksLikeSha(m[0]))
|
|
1476
|
+
add(m[0]);
|
|
1477
|
+
}
|
|
1478
|
+
return ordered.slice(0, MAX_PROBES);
|
|
1479
|
+
}
|
|
1480
|
+
function containsProbeVerbatim(text, probes) {
|
|
1481
|
+
if (probes.length === 0)
|
|
1482
|
+
return false;
|
|
1483
|
+
const haystack = text.toLowerCase();
|
|
1484
|
+
return probes.some((probe) => haystack.includes(probe.toLowerCase()));
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
// ../plugin/src/features/magic-context/message-fts-session-filter.ts
|
|
1488
|
+
var MESSAGE_FTS_SESSION_FILTER_SQL = `+message_history_fts.rowid IN (
|
|
1489
|
+
SELECT fts_rowid FROM message_fts_rowid_map WHERE session_id = ?1
|
|
1490
|
+
) AND `;
|
|
1491
|
+
var coverageCaches = new WeakMap;
|
|
1492
|
+
function getCoverageCache(db) {
|
|
1493
|
+
let cache = coverageCaches.get(db);
|
|
1494
|
+
if (!cache) {
|
|
1495
|
+
cache = {
|
|
1496
|
+
pin: db.prepare("SELECT fts_rowid FROM message_fts_rowid_map LIMIT 1"),
|
|
1497
|
+
stamp: db.prepare("SELECT total_changes() AS changes, data_version AS version FROM pragma_data_version"),
|
|
1498
|
+
missing: db.prepare(`SELECT DISTINCT session_id AS sessionId
|
|
1499
|
+
FROM message_history_fts WHERE rowid IN (
|
|
1500
|
+
SELECT id FROM message_history_fts_docsize
|
|
1501
|
+
EXCEPT SELECT fts_rowid FROM message_fts_rowid_map
|
|
1502
|
+
)`)
|
|
1503
|
+
};
|
|
1504
|
+
coverageCaches.set(db, cache);
|
|
1505
|
+
}
|
|
1506
|
+
return cache;
|
|
1507
|
+
}
|
|
1508
|
+
function withMessageFtsSessionFilter(db, sessionId, read) {
|
|
1509
|
+
let cacheable = false;
|
|
1510
|
+
try {
|
|
1511
|
+
const transaction = db;
|
|
1512
|
+
cacheable = transaction.inTransaction === false || transaction.isTransaction === false;
|
|
1513
|
+
if (!cacheable)
|
|
1514
|
+
coverageCaches.delete(db);
|
|
1515
|
+
} catch {
|
|
1516
|
+
coverageCaches.delete(db);
|
|
1517
|
+
}
|
|
1518
|
+
return db.transaction(() => {
|
|
1519
|
+
let sessionFirst = false;
|
|
1520
|
+
try {
|
|
1521
|
+
const cache = getCoverageCache(db);
|
|
1522
|
+
cache.pin.get();
|
|
1523
|
+
const stamp = cache.stamp.get();
|
|
1524
|
+
let incomplete = cache.incompleteSessions;
|
|
1525
|
+
if (!cacheable || !incomplete || cache.version !== stamp.version || cache.changes !== stamp.changes) {
|
|
1526
|
+
const rows = cache.missing.all();
|
|
1527
|
+
incomplete = new Set(rows.map((row) => row.sessionId));
|
|
1528
|
+
if (cacheable) {
|
|
1529
|
+
cache.version = stamp.version;
|
|
1530
|
+
cache.changes = stamp.changes;
|
|
1531
|
+
cache.incompleteSessions = incomplete;
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
sessionFirst = !incomplete.has(sessionId);
|
|
1535
|
+
} catch {
|
|
1536
|
+
coverageCaches.delete(db);
|
|
1537
|
+
}
|
|
1538
|
+
return read(sessionFirst);
|
|
1539
|
+
}).deferred();
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
// ../plugin/src/features/magic-context/search-measurement.ts
|
|
1543
|
+
import { createHash } from "node:crypto";
|
|
1544
|
+
function resultId(result) {
|
|
1545
|
+
switch (result.source) {
|
|
1546
|
+
case "memory":
|
|
1547
|
+
return `memory:${result.memoryId}`;
|
|
1548
|
+
case "message":
|
|
1549
|
+
return `message:${result.messageId}`;
|
|
1550
|
+
case "compartment":
|
|
1551
|
+
return `chunk:${result.compartmentId}`;
|
|
1552
|
+
case "git_commit":
|
|
1553
|
+
return `commit:${result.sha}`;
|
|
1554
|
+
case "primer":
|
|
1555
|
+
return `primer:${result.primerId}`;
|
|
1556
|
+
case "note":
|
|
1557
|
+
return `note:${result.noteId}`;
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
async function recordShadowMeasurement(args) {
|
|
1561
|
+
try {
|
|
1562
|
+
const shadowCohort = getShadowEmbeddingMeasurementCohort2(args.projectPath);
|
|
1563
|
+
if (!shadowCohort)
|
|
1564
|
+
return;
|
|
1565
|
+
const primaryCohort = getPrimaryEmbeddingMeasurementCohort2(args.projectPath);
|
|
1566
|
+
const shadowStartedAt = Date.now();
|
|
1567
|
+
let shadowFailed = false;
|
|
1568
|
+
let shadowResults = [];
|
|
1569
|
+
try {
|
|
1570
|
+
const vector = await embedShadowTextForProject2(args.projectPath, args.query);
|
|
1571
|
+
if (!vector) {
|
|
1572
|
+
shadowFailed = true;
|
|
1573
|
+
} else {
|
|
1574
|
+
shadowResults = await args.search(args.db, args.sessionId, args.projectPath, args.query, {
|
|
1575
|
+
...args.options,
|
|
1576
|
+
embedQuery: async () => vector,
|
|
1577
|
+
isEmbeddingRuntimeEnabled: () => true,
|
|
1578
|
+
embeddingEnabled: true,
|
|
1579
|
+
embeddingModelIdOverride: shadowCohort.modelId,
|
|
1580
|
+
chunkModelIdOverride: shadowCohort.chunkModelId,
|
|
1581
|
+
measurementDisabled: true,
|
|
1582
|
+
countRetrievals: false
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
} catch {
|
|
1586
|
+
shadowFailed = true;
|
|
1587
|
+
}
|
|
1588
|
+
const primaryModelId = args.primaryQuery?.modelId ?? primaryCohort?.modelId ?? "";
|
|
1589
|
+
const primaryFingerprint = primaryCohort?.fingerprint ?? "";
|
|
1590
|
+
const primaryEpoch = primaryCohort?.epoch ?? 0;
|
|
1591
|
+
const cohortKey = JSON.stringify({
|
|
1592
|
+
primaryFingerprint,
|
|
1593
|
+
primaryEpoch,
|
|
1594
|
+
shadowFingerprint: shadowCohort.fingerprint,
|
|
1595
|
+
shadowEpoch: shadowCohort.epoch
|
|
1596
|
+
});
|
|
1597
|
+
const primaryIds = args.primaryResults.map(resultId);
|
|
1598
|
+
const shadowIds = shadowResults.map(resultId);
|
|
1599
|
+
const corpusHash = sha256(JSON.stringify({ query: args.query, primaryIds, shadowIds }));
|
|
1600
|
+
recordEmbeddingMeasurement(args.db, {
|
|
1601
|
+
sessionId: args.sessionId,
|
|
1602
|
+
projectPath: args.projectPath,
|
|
1603
|
+
queryText: args.query,
|
|
1604
|
+
cohortKey,
|
|
1605
|
+
primaryResultIds: primaryIds,
|
|
1606
|
+
shadowResultIds: shadowIds,
|
|
1607
|
+
primaryLatencyMs: args.primaryLatencyMs,
|
|
1608
|
+
shadowLatencyMs: Date.now() - shadowStartedAt,
|
|
1609
|
+
primaryFailed: args.primaryQuery === null,
|
|
1610
|
+
shadowFailed,
|
|
1611
|
+
primaryModelId,
|
|
1612
|
+
shadowModelId: shadowCohort.modelId,
|
|
1613
|
+
primaryFingerprint,
|
|
1614
|
+
shadowFingerprint: shadowCohort.fingerprint,
|
|
1615
|
+
primaryEpoch,
|
|
1616
|
+
shadowEpoch: shadowCohort.epoch,
|
|
1617
|
+
corpusHash,
|
|
1618
|
+
coverage: {
|
|
1619
|
+
primaryResultCount: primaryIds.length,
|
|
1620
|
+
shadowResultCount: shadowIds.length,
|
|
1621
|
+
primaryAnswered: args.primaryQuery !== null,
|
|
1622
|
+
shadowAnswered: !shadowFailed
|
|
1623
|
+
}
|
|
1624
|
+
});
|
|
1625
|
+
} catch (error) {
|
|
1626
|
+
log2("[magic-context] shadow embedding measurement failed:", error);
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
function sha256(value) {
|
|
1630
|
+
return createHash("sha256").update(value).digest("hex");
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// ../plugin/src/features/magic-context/search.ts
|
|
1634
|
+
var DEFAULT_UNIFIED_SEARCH_LIMIT = 10;
|
|
1635
|
+
var MAX_UNIFIED_SEARCH_LIMIT = 25;
|
|
1636
|
+
var FTS_SEMANTIC_CANDIDATE_LIMIT = 50;
|
|
1637
|
+
var SEMANTIC_WEIGHT = 0.7;
|
|
1638
|
+
var FTS_WEIGHT = 0.3;
|
|
1639
|
+
var SINGLE_SOURCE_PENALTY = 0.8;
|
|
1640
|
+
var RESULT_PREVIEW_LIMIT = 220;
|
|
1641
|
+
var MEMORY_SOURCE_BOOST = 1.3;
|
|
1642
|
+
var MESSAGE_SOURCE_BOOST = 1.275;
|
|
1643
|
+
var GIT_COMMIT_SOURCE_BOOST = 1.2;
|
|
1644
|
+
var PRIMER_SOURCE_BOOST = 1.25;
|
|
1645
|
+
var messageSearchStatements = new WeakMap;
|
|
1646
|
+
var sessionFirstMessageSearchStatements = new WeakMap;
|
|
1647
|
+
var messageSearchStatementsWithCutoff = new WeakMap;
|
|
1648
|
+
var sessionFirstMessageSearchStatementsWithCutoff = new WeakMap;
|
|
1649
|
+
var messageSearchStatementsWithDateRange = new WeakMap;
|
|
1650
|
+
var messageSearchDiagnosticStatements = new WeakMap;
|
|
1651
|
+
var sessionFirstMessageSearchDiagnosticStatements = new WeakMap;
|
|
1652
|
+
var messageSearchDiagnosticStatementsWithDateRange = new WeakMap;
|
|
1653
|
+
var batchedMessageSearchStatements = new WeakMap;
|
|
1654
|
+
var batchedFtsCountStatements = new WeakMap;
|
|
1655
|
+
function createUnifiedSearchDiagnostics2() {
|
|
1656
|
+
return {
|
|
1657
|
+
suppressedVisibleMemoryIds: [],
|
|
1658
|
+
suppressedLiveMessageMatches: 0,
|
|
1659
|
+
gitCommitUnavailable: null
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1662
|
+
function searchMessageHistory2(db, request) {
|
|
1663
|
+
const diagnostics = request.collectDiagnostics ? createUnifiedSearchDiagnostics2() : undefined;
|
|
1664
|
+
const results = searchMessages({
|
|
1665
|
+
db,
|
|
1666
|
+
sessionId: request.sessionId,
|
|
1667
|
+
query: request.query,
|
|
1668
|
+
limit: request.limit,
|
|
1669
|
+
maxOrdinal: request.maxOrdinal,
|
|
1670
|
+
probes: request.probes,
|
|
1671
|
+
relaxedRecall: request.relaxedRecall,
|
|
1672
|
+
diagnostics,
|
|
1673
|
+
dateRange: request.dateRange
|
|
1674
|
+
});
|
|
1675
|
+
return {
|
|
1676
|
+
results,
|
|
1677
|
+
suppressedLiveMessageMatches: diagnostics?.suppressedLiveMessageMatches ?? 0
|
|
1678
|
+
};
|
|
1679
|
+
}
|
|
1680
|
+
function normalizeLimit(limit) {
|
|
1681
|
+
if (typeof limit !== "number" || !Number.isFinite(limit)) {
|
|
1682
|
+
return DEFAULT_UNIFIED_SEARCH_LIMIT;
|
|
1683
|
+
}
|
|
1684
|
+
return Math.min(MAX_UNIFIED_SEARCH_LIMIT, Math.max(1, Math.floor(limit)));
|
|
1685
|
+
}
|
|
1686
|
+
function normalizeDateRange(from, to) {
|
|
1687
|
+
if (from === undefined && to === undefined)
|
|
1688
|
+
return null;
|
|
1689
|
+
return {
|
|
1690
|
+
from: typeof from === "number" && Number.isSafeInteger(from) ? from : Number.MIN_SAFE_INTEGER,
|
|
1691
|
+
to: typeof to === "number" && Number.isSafeInteger(to) ? to : Number.MAX_SAFE_INTEGER
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
function timestampIsInRange(timestamp, range) {
|
|
1695
|
+
return range === null || timestamp >= range.from && timestamp <= range.to;
|
|
1696
|
+
}
|
|
1697
|
+
var ID_SHAPED_QUERY_MAX_TOKENS = 5;
|
|
1698
|
+
var ID_SHAPED_TOKEN = /^#?\d+$/;
|
|
1699
|
+
function parseIdShapedQuery2(query) {
|
|
1700
|
+
const trimmed = query.trim();
|
|
1701
|
+
if (trimmed.length === 0) {
|
|
1702
|
+
return null;
|
|
1703
|
+
}
|
|
1704
|
+
const tokens = trimmed.split(/[\s,]+/).filter((token) => token.length > 0);
|
|
1705
|
+
if (tokens.length === 0 || tokens.length > ID_SHAPED_QUERY_MAX_TOKENS) {
|
|
1706
|
+
return null;
|
|
1707
|
+
}
|
|
1708
|
+
const ids = [];
|
|
1709
|
+
for (const token of tokens) {
|
|
1710
|
+
if (!ID_SHAPED_TOKEN.test(token)) {
|
|
1711
|
+
return null;
|
|
1712
|
+
}
|
|
1713
|
+
const parsed = Number.parseInt(token.replace(/^#/, ""), 10);
|
|
1714
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
1715
|
+
return null;
|
|
1716
|
+
}
|
|
1717
|
+
ids.push(parsed);
|
|
1718
|
+
}
|
|
1719
|
+
return ids;
|
|
1720
|
+
}
|
|
1721
|
+
function normalizeCosineScore(score) {
|
|
1722
|
+
if (!Number.isFinite(score)) {
|
|
1723
|
+
return 0;
|
|
1724
|
+
}
|
|
1725
|
+
return Math.min(1, Math.max(0, score));
|
|
1726
|
+
}
|
|
1727
|
+
function previewText(text) {
|
|
1728
|
+
const normalized = text.replace(/\s+/g, " ").trim();
|
|
1729
|
+
if (normalized.length <= RESULT_PREVIEW_LIMIT) {
|
|
1730
|
+
return normalized;
|
|
1731
|
+
}
|
|
1732
|
+
return `${normalized.slice(0, RESULT_PREVIEW_LIMIT - 1).trimEnd()}…`;
|
|
1733
|
+
}
|
|
1734
|
+
function resolveSearchWorkspaceContext(db, projectPath, identitySet) {
|
|
1735
|
+
const resolved = identitySet ?? resolveWorkspaceIdentitySet(db, projectPath);
|
|
1736
|
+
const isWorkspaced = resolved.identities.length > 1;
|
|
1737
|
+
const expanded = expandWorkspaceIdentitySetWithAliases(db, resolved.identities);
|
|
1738
|
+
const expandedIdentities = isWorkspaced ? expanded.expandedIdentities : resolved.identities;
|
|
1739
|
+
const canonicalIdentityByStoredPath = isWorkspaced ? expanded.canonicalIdentityByStoredPath : new Map(resolved.identities.map((identity) => [identity, identity]));
|
|
1740
|
+
const ownIdentities = expandedIdentities.filter((identity) => canonicalIdentityByStoredPath.get(identity) === projectPath);
|
|
1741
|
+
return {
|
|
1742
|
+
identities: resolved.identities,
|
|
1743
|
+
expandedIdentities,
|
|
1744
|
+
ownIdentities,
|
|
1745
|
+
shareCategories: isWorkspaced ? resolveWorkspaceShareCategories(db, projectPath) : null,
|
|
1746
|
+
namesByIdentity: resolved.namesByIdentity,
|
|
1747
|
+
canonicalIdentityByStoredPath,
|
|
1748
|
+
isWorkspaced
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1751
|
+
function memoryWorkspaceIdentity(memory, workspace) {
|
|
1752
|
+
return resolveStoredPathWorkspaceIdentity(memory.projectPath, workspace.identities, workspace.canonicalIdentityByStoredPath);
|
|
1753
|
+
}
|
|
1754
|
+
function sourceNamesForSearchMemories(args) {
|
|
1755
|
+
if (!args.workspace.isWorkspaced)
|
|
1756
|
+
return;
|
|
1757
|
+
const sourceNames = new Map;
|
|
1758
|
+
for (const memory of args.memories) {
|
|
1759
|
+
const source = sourceNameForMemory(memory.projectPath, args.projectPath, args.workspace.identities, args.workspace.namesByIdentity, args.workspace.canonicalIdentityByStoredPath);
|
|
1760
|
+
if (source)
|
|
1761
|
+
sourceNames.set(memory.id, source);
|
|
1762
|
+
}
|
|
1763
|
+
return sourceNames.size > 0 ? sourceNames : undefined;
|
|
1764
|
+
}
|
|
1765
|
+
function getMessageSearchStatement(db, sessionFirst) {
|
|
1766
|
+
const statements = sessionFirst ? sessionFirstMessageSearchStatements : messageSearchStatements;
|
|
1767
|
+
let stmt = statements.get(db);
|
|
1768
|
+
if (!stmt) {
|
|
1769
|
+
stmt = db.prepare(`SELECT message_ordinal AS messageOrdinal, message_id AS messageId, role, content FROM message_history_fts WHERE ${sessionFirst ? MESSAGE_FTS_SESSION_FILTER_SQL : ""}session_id = ?1 AND message_history_fts MATCH ? ORDER BY bm25(message_history_fts), CAST(message_ordinal AS INTEGER) ASC LIMIT ?`);
|
|
1770
|
+
statements.set(db, stmt);
|
|
1771
|
+
}
|
|
1772
|
+
return stmt;
|
|
1773
|
+
}
|
|
1774
|
+
function getMessageSearchStatementWithCutoff(db, sessionFirst) {
|
|
1775
|
+
const statements = sessionFirst ? sessionFirstMessageSearchStatementsWithCutoff : messageSearchStatementsWithCutoff;
|
|
1776
|
+
let stmt = statements.get(db);
|
|
1777
|
+
if (!stmt) {
|
|
1778
|
+
stmt = db.prepare(`SELECT message_ordinal AS messageOrdinal, message_id AS messageId, role, content FROM message_history_fts WHERE ${sessionFirst ? MESSAGE_FTS_SESSION_FILTER_SQL : ""}session_id = ?1 AND message_history_fts MATCH ? AND CAST(message_ordinal AS INTEGER) <= ? ORDER BY bm25(message_history_fts), CAST(message_ordinal AS INTEGER) ASC LIMIT ?`);
|
|
1779
|
+
statements.set(db, stmt);
|
|
1780
|
+
}
|
|
1781
|
+
return stmt;
|
|
1782
|
+
}
|
|
1783
|
+
function getMessageSearchStatementWithDateRange(db, withCutoff) {
|
|
1784
|
+
let statements = messageSearchStatementsWithDateRange.get(db);
|
|
1785
|
+
if (!statements) {
|
|
1786
|
+
statements = new Map;
|
|
1787
|
+
messageSearchStatementsWithDateRange.set(db, statements);
|
|
1788
|
+
}
|
|
1789
|
+
const key = withCutoff ? "cutoff" : "all";
|
|
1790
|
+
let statement = statements.get(key);
|
|
1791
|
+
if (!statement) {
|
|
1792
|
+
statement = db.prepare(`SELECT message_history_fts.message_ordinal AS messageOrdinal,
|
|
1793
|
+
message_history_fts.message_id AS messageId,
|
|
1794
|
+
message_history_fts.role AS role,
|
|
1795
|
+
message_history_fts.content AS content
|
|
1796
|
+
FROM message_history_fts
|
|
1797
|
+
JOIN message_fts_rowid_map AS map
|
|
1798
|
+
ON map.session_id = message_history_fts.session_id
|
|
1799
|
+
AND map.fts_rowid = message_history_fts.rowid
|
|
1800
|
+
WHERE ${MESSAGE_FTS_SESSION_FILTER_SQL}message_history_fts.session_id = ?1
|
|
1801
|
+
AND message_history_fts MATCH ?
|
|
1802
|
+
AND map.message_time_ms BETWEEN ? AND ?
|
|
1803
|
+
${withCutoff ? "AND CAST(message_history_fts.message_ordinal AS INTEGER) <= ?" : ""}
|
|
1804
|
+
ORDER BY bm25(message_history_fts),
|
|
1805
|
+
CAST(message_history_fts.message_ordinal AS INTEGER) ASC
|
|
1806
|
+
LIMIT ?`);
|
|
1807
|
+
statements.set(key, statement);
|
|
1808
|
+
}
|
|
1809
|
+
return statement;
|
|
1810
|
+
}
|
|
1811
|
+
function getMessageSearchDiagnosticStatement(db, sessionFirst) {
|
|
1812
|
+
const statements = sessionFirst ? sessionFirstMessageSearchDiagnosticStatements : messageSearchDiagnosticStatements;
|
|
1813
|
+
let stmt = statements.get(db);
|
|
1814
|
+
if (!stmt) {
|
|
1815
|
+
stmt = db.prepare(`
|
|
1816
|
+
WITH matches AS MATERIALIZED (
|
|
1817
|
+
SELECT
|
|
1818
|
+
message_ordinal AS messageOrdinal,
|
|
1819
|
+
message_id AS messageId,
|
|
1820
|
+
role,
|
|
1821
|
+
content,
|
|
1822
|
+
CAST(message_ordinal AS INTEGER) AS ordinalValue,
|
|
1823
|
+
bm25(message_history_fts) AS ftsRank
|
|
1824
|
+
FROM message_history_fts
|
|
1825
|
+
WHERE ${sessionFirst ? MESSAGE_FTS_SESSION_FILTER_SQL : ""}session_id = ?1 AND message_history_fts MATCH ?
|
|
1826
|
+
),
|
|
1827
|
+
eligible AS (
|
|
1828
|
+
SELECT * FROM matches
|
|
1829
|
+
WHERE ordinalValue <= ?
|
|
1830
|
+
ORDER BY ftsRank, ordinalValue ASC
|
|
1831
|
+
LIMIT ?
|
|
1832
|
+
),
|
|
1833
|
+
summary AS (
|
|
1834
|
+
SELECT COUNT(*) AS suppressedCount
|
|
1835
|
+
FROM matches
|
|
1836
|
+
WHERE ordinalValue > ?
|
|
1837
|
+
)
|
|
1838
|
+
SELECT
|
|
1839
|
+
eligible.messageOrdinal,
|
|
1840
|
+
eligible.messageId,
|
|
1841
|
+
eligible.role,
|
|
1842
|
+
eligible.content,
|
|
1843
|
+
eligible.ftsRank,
|
|
1844
|
+
summary.suppressedCount,
|
|
1845
|
+
0 AS summaryOnly
|
|
1846
|
+
FROM eligible CROSS JOIN summary
|
|
1847
|
+
UNION ALL
|
|
1848
|
+
SELECT NULL, NULL, NULL, NULL, NULL, summary.suppressedCount, 1
|
|
1849
|
+
FROM summary
|
|
1850
|
+
WHERE NOT EXISTS (SELECT 1 FROM eligible)
|
|
1851
|
+
ORDER BY summaryOnly ASC, ftsRank ASC, messageOrdinal ASC
|
|
1852
|
+
`);
|
|
1853
|
+
statements.set(db, stmt);
|
|
1854
|
+
}
|
|
1855
|
+
return stmt;
|
|
1856
|
+
}
|
|
1857
|
+
function getMessageSearchDiagnosticStatementWithDateRange(db) {
|
|
1858
|
+
let statement = messageSearchDiagnosticStatementsWithDateRange.get(db);
|
|
1859
|
+
if (!statement) {
|
|
1860
|
+
statement = db.prepare(`
|
|
1861
|
+
WITH matches AS MATERIALIZED (
|
|
1862
|
+
SELECT
|
|
1863
|
+
message_history_fts.message_ordinal AS messageOrdinal,
|
|
1864
|
+
message_history_fts.message_id AS messageId,
|
|
1865
|
+
message_history_fts.role AS role,
|
|
1866
|
+
message_history_fts.content AS content,
|
|
1867
|
+
CAST(message_history_fts.message_ordinal AS INTEGER) AS ordinalValue,
|
|
1868
|
+
bm25(message_history_fts) AS ftsRank
|
|
1869
|
+
FROM message_history_fts
|
|
1870
|
+
JOIN message_fts_rowid_map AS map
|
|
1871
|
+
ON map.session_id = message_history_fts.session_id
|
|
1872
|
+
AND map.fts_rowid = message_history_fts.rowid
|
|
1873
|
+
WHERE ${MESSAGE_FTS_SESSION_FILTER_SQL}message_history_fts.session_id = ?1
|
|
1874
|
+
AND message_history_fts MATCH ?
|
|
1875
|
+
AND map.message_time_ms BETWEEN ? AND ?
|
|
1876
|
+
),
|
|
1877
|
+
eligible AS (
|
|
1878
|
+
SELECT * FROM matches
|
|
1879
|
+
WHERE ordinalValue <= ?
|
|
1880
|
+
ORDER BY ftsRank, ordinalValue ASC
|
|
1881
|
+
LIMIT ?
|
|
1882
|
+
),
|
|
1883
|
+
summary AS (
|
|
1884
|
+
SELECT COUNT(*) AS suppressedCount
|
|
1885
|
+
FROM matches
|
|
1886
|
+
WHERE ordinalValue > ?
|
|
1887
|
+
)
|
|
1888
|
+
SELECT
|
|
1889
|
+
eligible.messageOrdinal,
|
|
1890
|
+
eligible.messageId,
|
|
1891
|
+
eligible.role,
|
|
1892
|
+
eligible.content,
|
|
1893
|
+
eligible.ftsRank,
|
|
1894
|
+
summary.suppressedCount,
|
|
1895
|
+
0 AS summaryOnly
|
|
1896
|
+
FROM eligible CROSS JOIN summary
|
|
1897
|
+
UNION ALL
|
|
1898
|
+
SELECT NULL, NULL, NULL, NULL, NULL, summary.suppressedCount, 1
|
|
1899
|
+
FROM summary
|
|
1900
|
+
WHERE NOT EXISTS (SELECT 1 FROM eligible)
|
|
1901
|
+
ORDER BY summaryOnly ASC, ftsRank ASC, messageOrdinal ASC
|
|
1902
|
+
`);
|
|
1903
|
+
messageSearchDiagnosticStatementsWithDateRange.set(db, statement);
|
|
1904
|
+
}
|
|
1905
|
+
return statement;
|
|
1906
|
+
}
|
|
1907
|
+
function getBatchedFtsCountStatement(db, queryCount, cutoff, dateRange, sessionFirst) {
|
|
1908
|
+
let statements = batchedFtsCountStatements.get(db);
|
|
1909
|
+
if (!statements) {
|
|
1910
|
+
statements = new Map;
|
|
1911
|
+
batchedFtsCountStatements.set(db, statements);
|
|
1912
|
+
}
|
|
1913
|
+
const key = `${queryCount}:${cutoff === null ? "all" : "cutoff"}:${dateRange === null ? "all-dates" : "dated"}:${sessionFirst}`;
|
|
1914
|
+
let statement = statements.get(key);
|
|
1915
|
+
if (!statement) {
|
|
1916
|
+
const cutoffSql = cutoff === null ? "" : " AND CAST(message_history_fts.message_ordinal AS INTEGER) <= ?";
|
|
1917
|
+
const joinSql = dateRange === null ? "" : ` JOIN message_fts_rowid_map AS map
|
|
1918
|
+
ON map.session_id = message_history_fts.session_id
|
|
1919
|
+
AND map.fts_rowid = message_history_fts.rowid`;
|
|
1920
|
+
const dateSql = dateRange === null ? "" : " AND map.message_time_ms BETWEEN ? AND ?";
|
|
1921
|
+
statement = db.prepare(Array.from({ length: queryCount }, (_, index) => `SELECT ${index} AS queryIndex, COUNT(*) AS count
|
|
1922
|
+
FROM message_history_fts${joinSql}
|
|
1923
|
+
WHERE ${sessionFirst || dateRange !== null ? MESSAGE_FTS_SESSION_FILTER_SQL : ""}message_history_fts.session_id = ${index === 0 ? "?1" : "?"}
|
|
1924
|
+
AND message_history_fts MATCH ?${dateSql}${cutoffSql}`).join(`
|
|
1925
|
+
UNION ALL
|
|
1926
|
+
`));
|
|
1927
|
+
statements.set(key, statement);
|
|
1928
|
+
}
|
|
1929
|
+
return statement;
|
|
1930
|
+
}
|
|
1931
|
+
function contentOnlyMessageQuery(ftsQuery) {
|
|
1932
|
+
return ftsQuery.length === 0 ? "" : `content : (${ftsQuery})`;
|
|
1933
|
+
}
|
|
1934
|
+
function countSessionFtsMatchesBatch(db, sessionId, ftsQueries, cutoff, dateRange, sessionFirst) {
|
|
1935
|
+
if (ftsQueries.length === 0)
|
|
1936
|
+
return [];
|
|
1937
|
+
const bindings = [];
|
|
1938
|
+
for (const query of ftsQueries) {
|
|
1939
|
+
bindings.push(sessionId, contentOnlyMessageQuery(query));
|
|
1940
|
+
if (dateRange !== null)
|
|
1941
|
+
bindings.push(dateRange.from, dateRange.to);
|
|
1942
|
+
if (cutoff !== null)
|
|
1943
|
+
bindings.push(cutoff);
|
|
1944
|
+
}
|
|
1945
|
+
try {
|
|
1946
|
+
const rows = getBatchedFtsCountStatement(db, ftsQueries.length, cutoff, dateRange, sessionFirst).all(...bindings);
|
|
1947
|
+
const counts = Array.from({ length: ftsQueries.length }, () => 0);
|
|
1948
|
+
for (const row of rows) {
|
|
1949
|
+
if (typeof row.queryIndex === "number" && row.queryIndex >= 0 && row.queryIndex < counts.length && typeof row.count === "number") {
|
|
1950
|
+
counts[row.queryIndex] = row.count;
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
return counts;
|
|
1954
|
+
} catch {
|
|
1955
|
+
return Array.from({ length: ftsQueries.length }, () => 0);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
function getMessageOrdinal(value) {
|
|
1959
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
1960
|
+
return value;
|
|
1961
|
+
}
|
|
1962
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
1963
|
+
const parsed = Number.parseInt(value, 10);
|
|
1964
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
1965
|
+
}
|
|
1966
|
+
return null;
|
|
1967
|
+
}
|
|
1968
|
+
async function getSemanticScores(args) {
|
|
1969
|
+
const semanticScores = new Map;
|
|
1970
|
+
if (!args.queryEmbedding || args.memories.length === 0 || !args.queryModelId || args.queryModelId === "off") {
|
|
1971
|
+
return semanticScores;
|
|
1972
|
+
}
|
|
1973
|
+
if (!args.workspace?.isWorkspaced) {
|
|
1974
|
+
const cachedEmbeddings = getProjectEmbeddings2(args.db, args.projectPath, args.queryModelId);
|
|
1975
|
+
const embeddings = args.backfillMemoryEmbeddings === false ? cachedEmbeddings : await ensureMemoryEmbeddings2({
|
|
1976
|
+
db: args.db,
|
|
1977
|
+
projectIdentity: args.projectPath,
|
|
1978
|
+
memories: args.memories,
|
|
1979
|
+
existingEmbeddings: cachedEmbeddings
|
|
1980
|
+
});
|
|
1981
|
+
for (const memory of args.memories) {
|
|
1982
|
+
const memoryEmbedding = embeddings.get(memory.id);
|
|
1983
|
+
if (!memoryEmbedding) {
|
|
1984
|
+
continue;
|
|
1985
|
+
}
|
|
1986
|
+
semanticScores.set(memory.id, normalizeCosineScore(cosineSimilarity(args.queryEmbedding, memoryEmbedding.embedding)));
|
|
1987
|
+
}
|
|
1988
|
+
return semanticScores;
|
|
1989
|
+
}
|
|
1990
|
+
const workspace = args.workspace;
|
|
1991
|
+
const memoriesByIdentity = new Map;
|
|
1992
|
+
for (const memory of args.memories) {
|
|
1993
|
+
const identity = memoryWorkspaceIdentity(memory, workspace);
|
|
1994
|
+
if (!identity)
|
|
1995
|
+
continue;
|
|
1996
|
+
const list = memoriesByIdentity.get(identity) ?? [];
|
|
1997
|
+
list.push(memory);
|
|
1998
|
+
memoriesByIdentity.set(identity, list);
|
|
1999
|
+
}
|
|
2000
|
+
const ownMemories = memoriesByIdentity.get(args.projectPath) ?? [];
|
|
2001
|
+
if (ownMemories.length > 0 && args.backfillMemoryEmbeddings !== false) {
|
|
2002
|
+
const ownEmbeddings = getProjectEmbeddings2(args.db, args.projectPath, args.queryModelId);
|
|
2003
|
+
await ensureMemoryEmbeddings2({
|
|
2004
|
+
db: args.db,
|
|
2005
|
+
projectIdentity: args.projectPath,
|
|
2006
|
+
memories: ownMemories,
|
|
2007
|
+
existingEmbeddings: ownEmbeddings
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
for (const identity of workspace.identities) {
|
|
2011
|
+
const memberMemories = memoriesByIdentity.get(identity) ?? [];
|
|
2012
|
+
if (memberMemories.length === 0)
|
|
2013
|
+
continue;
|
|
2014
|
+
const cachedEmbeddings = getProjectEmbeddings2(args.db, identity, args.queryModelId);
|
|
2015
|
+
for (const memory of memberMemories) {
|
|
2016
|
+
const memoryEmbedding = cachedEmbeddings.get(memory.id);
|
|
2017
|
+
if (!memoryEmbedding || memoryEmbedding.modelId !== args.queryModelId)
|
|
2018
|
+
continue;
|
|
2019
|
+
semanticScores.set(memory.id, normalizeCosineScore(cosineSimilarity(args.queryEmbedding, memoryEmbedding.embedding)));
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
return semanticScores;
|
|
2023
|
+
}
|
|
2024
|
+
function getFtsMatches(args) {
|
|
2025
|
+
try {
|
|
2026
|
+
return args.workspace?.isWorkspaced ? searchMemoriesFTSUnion(args.db, args.workspace.expandedIdentities, args.query, args.limit, args.workspace.ownIdentities, args.workspace.shareCategories, args.dateRange) : searchMemoriesFTS(args.db, args.projectPath, args.query, args.limit, args.dateRange);
|
|
2027
|
+
} catch (error) {
|
|
2028
|
+
log2(`[search] FTS query failed for "${args.query}": ${error instanceof Error ? error.message : String(error)}`);
|
|
2029
|
+
return [];
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
function getFtsScores(matches) {
|
|
2033
|
+
return new Map(matches.map((memory, rank) => [memory.id, 1 / (rank + 1)]));
|
|
2034
|
+
}
|
|
2035
|
+
function selectSemanticCandidates(args) {
|
|
2036
|
+
if (args.ftsMatches.length === 0) {
|
|
2037
|
+
return args.memories;
|
|
2038
|
+
}
|
|
2039
|
+
const candidateIds = new Set(args.ftsMatches.map((memory) => memory.id));
|
|
2040
|
+
if (args.queryModelId && args.queryModelId !== "off") {
|
|
2041
|
+
const embeddingProjects = args.workspace?.isWorkspaced ? args.workspace.identities : [args.projectPath];
|
|
2042
|
+
for (const projectPath of embeddingProjects) {
|
|
2043
|
+
const storedEmbeddings = getProjectEmbeddings2(args.db, projectPath, args.queryModelId);
|
|
2044
|
+
for (const memoryId of storedEmbeddings.keys()) {
|
|
2045
|
+
candidateIds.add(memoryId);
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
return args.memories.filter((memory) => candidateIds.has(memory.id));
|
|
2050
|
+
}
|
|
2051
|
+
function mergeMemoryResults(args) {
|
|
2052
|
+
const memoryById = new Map(args.memories.map((memory) => [memory.id, memory]));
|
|
2053
|
+
const candidateIds = new Set([...args.semanticScores.keys(), ...args.ftsScores.keys()]);
|
|
2054
|
+
const results = [];
|
|
2055
|
+
const suppressedVisibleIds = [];
|
|
2056
|
+
for (const id of candidateIds) {
|
|
2057
|
+
const memory = memoryById.get(id);
|
|
2058
|
+
if (!memory) {
|
|
2059
|
+
continue;
|
|
2060
|
+
}
|
|
2061
|
+
const semanticScore = args.semanticScores.get(id);
|
|
2062
|
+
const ftsScore = args.ftsScores.get(id);
|
|
2063
|
+
let score = 0;
|
|
2064
|
+
let matchType = "fts";
|
|
2065
|
+
if (semanticScore !== undefined && ftsScore !== undefined) {
|
|
2066
|
+
score = SEMANTIC_WEIGHT * semanticScore + FTS_WEIGHT * ftsScore;
|
|
2067
|
+
matchType = "hybrid";
|
|
2068
|
+
} else if (semanticScore !== undefined) {
|
|
2069
|
+
score = semanticScore * SINGLE_SOURCE_PENALTY;
|
|
2070
|
+
matchType = "semantic";
|
|
2071
|
+
} else if (ftsScore !== undefined) {
|
|
2072
|
+
score = ftsScore * SINGLE_SOURCE_PENALTY;
|
|
2073
|
+
matchType = "fts";
|
|
2074
|
+
}
|
|
2075
|
+
if (score <= 0) {
|
|
2076
|
+
continue;
|
|
2077
|
+
}
|
|
2078
|
+
if (args.visibleMemoryIds?.has(id)) {
|
|
2079
|
+
suppressedVisibleIds.push(id);
|
|
2080
|
+
continue;
|
|
2081
|
+
}
|
|
2082
|
+
results.push({
|
|
2083
|
+
source: "memory",
|
|
2084
|
+
content: previewText(memory.content),
|
|
2085
|
+
score,
|
|
2086
|
+
memoryId: memory.id,
|
|
2087
|
+
category: memory.category,
|
|
2088
|
+
matchType,
|
|
2089
|
+
sourceName: args.sourceNameByMemoryId?.get(memory.id)
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
return {
|
|
2093
|
+
results: results.sort((left, right) => {
|
|
2094
|
+
if (right.score !== left.score) {
|
|
2095
|
+
return right.score - left.score;
|
|
2096
|
+
}
|
|
2097
|
+
return left.memoryId - right.memoryId;
|
|
2098
|
+
}).slice(0, args.limit),
|
|
2099
|
+
suppressedVisibleIds: suppressedVisibleIds.sort((left, right) => left - right)
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
async function searchMemories(args) {
|
|
2103
|
+
if (!args.memoryEnabled) {
|
|
2104
|
+
return { results: [], suppressedVisibleIds: [] };
|
|
2105
|
+
}
|
|
2106
|
+
const unfilteredMemories = args.workspace?.isWorkspaced ? getMemoriesByProjects2(args.db, args.workspace.expandedIdentities, ["active", "permanent"], Date.now(), args.workspace.ownIdentities, args.workspace.shareCategories) : getMemoriesByProject2(args.db, args.projectPath);
|
|
2107
|
+
const memories = unfilteredMemories.filter((memory) => timestampIsInRange(memory.createdAt, args.dateRange));
|
|
2108
|
+
if (memories.length === 0) {
|
|
2109
|
+
return { results: [], suppressedVisibleIds: [] };
|
|
2110
|
+
}
|
|
2111
|
+
const ftsMatches = getFtsMatches({
|
|
2112
|
+
db: args.db,
|
|
2113
|
+
projectPath: args.projectPath,
|
|
2114
|
+
query: args.query,
|
|
2115
|
+
limit: FTS_SEMANTIC_CANDIDATE_LIMIT,
|
|
2116
|
+
workspace: args.workspace,
|
|
2117
|
+
dateRange: args.dateRange
|
|
2118
|
+
});
|
|
2119
|
+
const ftsScores = getFtsScores(ftsMatches);
|
|
2120
|
+
const semanticCandidates = selectSemanticCandidates({
|
|
2121
|
+
db: args.db,
|
|
2122
|
+
memories,
|
|
2123
|
+
projectPath: args.projectPath,
|
|
2124
|
+
ftsMatches,
|
|
2125
|
+
queryModelId: args.queryModelId,
|
|
2126
|
+
workspace: args.workspace
|
|
2127
|
+
});
|
|
2128
|
+
const semanticScores = await getSemanticScores({
|
|
2129
|
+
db: args.db,
|
|
2130
|
+
projectPath: args.projectPath,
|
|
2131
|
+
memories: semanticCandidates,
|
|
2132
|
+
backfillMemoryEmbeddings: args.backfillMemoryEmbeddings,
|
|
2133
|
+
queryEmbedding: args.queryEmbedding,
|
|
2134
|
+
queryModelId: args.queryModelId,
|
|
2135
|
+
workspace: args.workspace
|
|
2136
|
+
});
|
|
2137
|
+
return mergeMemoryResults({
|
|
2138
|
+
memories,
|
|
2139
|
+
semanticScores,
|
|
2140
|
+
ftsScores,
|
|
2141
|
+
limit: args.limit,
|
|
2142
|
+
visibleMemoryIds: args.visibleMemoryIds,
|
|
2143
|
+
sourceNameByMemoryId: sourceNamesForSearchMemories({
|
|
2144
|
+
memories,
|
|
2145
|
+
projectPath: args.projectPath,
|
|
2146
|
+
workspace: args.workspace ?? {
|
|
2147
|
+
identities: [args.projectPath],
|
|
2148
|
+
expandedIdentities: [args.projectPath],
|
|
2149
|
+
namesByIdentity: new Map,
|
|
2150
|
+
canonicalIdentityByStoredPath: new Map([[args.projectPath, args.projectPath]]),
|
|
2151
|
+
ownIdentities: [args.projectPath],
|
|
2152
|
+
shareCategories: null,
|
|
2153
|
+
isWorkspaced: false
|
|
2154
|
+
}
|
|
2155
|
+
})
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2158
|
+
function linearDecayScore(rank, total) {
|
|
2159
|
+
if (total <= 0)
|
|
2160
|
+
return 0;
|
|
2161
|
+
return Math.max(0, 1 - rank / total);
|
|
2162
|
+
}
|
|
2163
|
+
function normalizeMessageSearchRow(row, cutoff) {
|
|
2164
|
+
const messageOrdinal = getMessageOrdinal(row.messageOrdinal);
|
|
2165
|
+
if (messageOrdinal === null || typeof row.messageId !== "string" || typeof row.role !== "string" || typeof row.content !== "string") {
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
2168
|
+
if (cutoff !== null && messageOrdinal > cutoff)
|
|
2169
|
+
return null;
|
|
2170
|
+
return {
|
|
2171
|
+
messageOrdinal,
|
|
2172
|
+
messageId: row.messageId,
|
|
2173
|
+
role: row.role,
|
|
2174
|
+
content: row.content
|
|
2175
|
+
};
|
|
2176
|
+
}
|
|
2177
|
+
function runMessageFtsQuery(db, sessionId, ftsQuery, fetchLimit, cutoff, dateRange, sessionFirst) {
|
|
2178
|
+
if (ftsQuery.length === 0)
|
|
2179
|
+
return [];
|
|
2180
|
+
let rawRows;
|
|
2181
|
+
const matchQuery = contentOnlyMessageQuery(ftsQuery);
|
|
2182
|
+
if (dateRange !== null) {
|
|
2183
|
+
const bindings = [sessionId, matchQuery, dateRange.from, dateRange.to];
|
|
2184
|
+
if (cutoff !== null)
|
|
2185
|
+
bindings.push(cutoff);
|
|
2186
|
+
bindings.push(fetchLimit);
|
|
2187
|
+
rawRows = getMessageSearchStatementWithDateRange(db, cutoff !== null).all(...bindings);
|
|
2188
|
+
} else {
|
|
2189
|
+
rawRows = cutoff !== null ? getMessageSearchStatementWithCutoff(db, sessionFirst).all(sessionId, matchQuery, cutoff, fetchLimit) : getMessageSearchStatement(db, sessionFirst).all(sessionId, matchQuery, fetchLimit);
|
|
2190
|
+
}
|
|
2191
|
+
const rows = rawRows.map((row) => row);
|
|
2192
|
+
const result = [];
|
|
2193
|
+
for (const row of rows) {
|
|
2194
|
+
const normalized = normalizeMessageSearchRow(row, cutoff);
|
|
2195
|
+
if (normalized)
|
|
2196
|
+
result.push(normalized);
|
|
2197
|
+
}
|
|
2198
|
+
return result;
|
|
2199
|
+
}
|
|
2200
|
+
function runMessageFtsQueryWithDiagnostics(args) {
|
|
2201
|
+
if (args.ftsQuery.length === 0)
|
|
2202
|
+
return { rows: [], suppressedCount: 0 };
|
|
2203
|
+
const matchQuery = contentOnlyMessageQuery(args.ftsQuery);
|
|
2204
|
+
const rawRows = (args.dateRange === null ? getMessageSearchDiagnosticStatement(args.db, args.sessionFirst).all(args.sessionId, matchQuery, args.cutoff, args.fetchLimit, args.cutoff) : getMessageSearchDiagnosticStatementWithDateRange(args.db).all(args.sessionId, matchQuery, args.dateRange.from, args.dateRange.to, args.cutoff, args.fetchLimit, args.cutoff)).map((row) => row);
|
|
2205
|
+
const suppressedCount = rawRows[0]?.suppressedCount ?? 0;
|
|
2206
|
+
const rows = [];
|
|
2207
|
+
for (const row of rawRows) {
|
|
2208
|
+
if (row.summaryOnly === 1)
|
|
2209
|
+
continue;
|
|
2210
|
+
const normalized = normalizeMessageSearchRow(row, args.cutoff);
|
|
2211
|
+
if (normalized)
|
|
2212
|
+
rows.push(normalized);
|
|
2213
|
+
}
|
|
2214
|
+
return { rows, suppressedCount };
|
|
2215
|
+
}
|
|
2216
|
+
function getBatchedMessageSearchStatement(db, queryCount, cutoff, dateRange, sessionFirst) {
|
|
2217
|
+
let statements = batchedMessageSearchStatements.get(db);
|
|
2218
|
+
if (!statements) {
|
|
2219
|
+
statements = new Map;
|
|
2220
|
+
batchedMessageSearchStatements.set(db, statements);
|
|
2221
|
+
}
|
|
2222
|
+
const key = `${queryCount}:${cutoff === null ? "all" : "cutoff"}:${dateRange === null ? "all-dates" : "dated"}:${sessionFirst}`;
|
|
2223
|
+
let statement = statements.get(key);
|
|
2224
|
+
if (!statement) {
|
|
2225
|
+
const cutoffSql = cutoff === null ? "" : " AND CAST(message_history_fts.message_ordinal AS INTEGER) <= ?";
|
|
2226
|
+
const joinSql = dateRange === null ? "" : ` JOIN message_fts_rowid_map AS map
|
|
2227
|
+
ON map.session_id = message_history_fts.session_id
|
|
2228
|
+
AND map.fts_rowid = message_history_fts.rowid`;
|
|
2229
|
+
const dateSql = dateRange === null ? "" : " AND map.message_time_ms BETWEEN ? AND ?";
|
|
2230
|
+
const branches = Array.from({ length: queryCount }, (_, index) => `SELECT * FROM (
|
|
2231
|
+
SELECT ${index} AS queryIndex,
|
|
2232
|
+
message_history_fts.message_ordinal AS messageOrdinal,
|
|
2233
|
+
message_history_fts.message_id AS messageId,
|
|
2234
|
+
message_history_fts.role AS role,
|
|
2235
|
+
message_history_fts.content AS content,
|
|
2236
|
+
bm25(message_history_fts) AS ftsRank
|
|
2237
|
+
FROM message_history_fts${joinSql}
|
|
2238
|
+
WHERE ${sessionFirst || dateRange !== null ? MESSAGE_FTS_SESSION_FILTER_SQL : ""}message_history_fts.session_id = ${index === 0 ? "?1" : "?"}
|
|
2239
|
+
AND message_history_fts MATCH ?${dateSql}${cutoffSql}
|
|
2240
|
+
ORDER BY ftsRank
|
|
2241
|
+
LIMIT ?
|
|
2242
|
+
)`);
|
|
2243
|
+
statement = db.prepare(`${branches.join(`
|
|
2244
|
+
UNION ALL
|
|
2245
|
+
`)}
|
|
2246
|
+
ORDER BY queryIndex ASC, ftsRank ASC`);
|
|
2247
|
+
statements.set(key, statement);
|
|
2248
|
+
}
|
|
2249
|
+
return statement;
|
|
2250
|
+
}
|
|
2251
|
+
function runMessageFtsQueriesBatch(db, sessionId, ftsQueries, fetchLimit, cutoff, dateRange, sessionFirst) {
|
|
2252
|
+
if (ftsQueries.length === 0)
|
|
2253
|
+
return [];
|
|
2254
|
+
const bindings = [];
|
|
2255
|
+
for (const query of ftsQueries) {
|
|
2256
|
+
bindings.push(sessionId, contentOnlyMessageQuery(query));
|
|
2257
|
+
if (dateRange !== null)
|
|
2258
|
+
bindings.push(dateRange.from, dateRange.to);
|
|
2259
|
+
if (cutoff !== null)
|
|
2260
|
+
bindings.push(cutoff);
|
|
2261
|
+
bindings.push(fetchLimit);
|
|
2262
|
+
}
|
|
2263
|
+
const rows = getBatchedMessageSearchStatement(db, ftsQueries.length, cutoff, dateRange, sessionFirst).all(...bindings);
|
|
2264
|
+
const result = Array.from({ length: ftsQueries.length }, () => []);
|
|
2265
|
+
for (const row of rows) {
|
|
2266
|
+
if (typeof row.queryIndex !== "number" || row.queryIndex < 0 || row.queryIndex >= result.length) {
|
|
2267
|
+
continue;
|
|
2268
|
+
}
|
|
2269
|
+
const normalized = normalizeMessageSearchRow(row, cutoff);
|
|
2270
|
+
if (normalized)
|
|
2271
|
+
result[row.queryIndex].push(normalized);
|
|
2272
|
+
}
|
|
2273
|
+
return result;
|
|
2274
|
+
}
|
|
2275
|
+
var RRF_K = 60;
|
|
2276
|
+
var VERBATIM_RANK_BONUS = 1 / RRF_K;
|
|
2277
|
+
var IDF_FALLOFF = 100;
|
|
2278
|
+
function probeDiscriminationWeight(df, corpusSize) {
|
|
2279
|
+
if (corpusSize <= 0 || df <= 0)
|
|
2280
|
+
return 1;
|
|
2281
|
+
return 1 / (1 + IDF_FALLOFF * df / corpusSize);
|
|
2282
|
+
}
|
|
2283
|
+
function searchMessages(args) {
|
|
2284
|
+
return withMessageFtsSessionFilter(args.db, args.sessionId, (sessionFirst) => searchMessagesInSnapshot(args, sessionFirst));
|
|
2285
|
+
}
|
|
2286
|
+
function searchMessagesInSnapshot(args, sessionFirst) {
|
|
2287
|
+
const cutoff = args.maxOrdinal != null && args.maxOrdinal >= 0 ? args.maxOrdinal : null;
|
|
2288
|
+
const fetchLimit = args.maxOrdinal != null && args.maxOrdinal >= 0 ? args.limit * 3 : args.limit;
|
|
2289
|
+
const baseQuery = sanitizeFtsQuery(args.query.trim());
|
|
2290
|
+
const probes = args.probes ?? [];
|
|
2291
|
+
if (probes.length === 0) {
|
|
2292
|
+
const outcome = args.diagnostics && cutoff !== null ? runMessageFtsQueryWithDiagnostics({
|
|
2293
|
+
db: args.db,
|
|
2294
|
+
sessionId: args.sessionId,
|
|
2295
|
+
ftsQuery: baseQuery,
|
|
2296
|
+
fetchLimit,
|
|
2297
|
+
cutoff,
|
|
2298
|
+
dateRange: args.dateRange,
|
|
2299
|
+
sessionFirst
|
|
2300
|
+
}) : {
|
|
2301
|
+
rows: runMessageFtsQuery(args.db, args.sessionId, baseQuery, fetchLimit, cutoff, args.dateRange, sessionFirst),
|
|
2302
|
+
suppressedCount: 0
|
|
2303
|
+
};
|
|
2304
|
+
if (args.diagnostics) {
|
|
2305
|
+
args.diagnostics.suppressedLiveMessageMatches = outcome.suppressedCount;
|
|
2306
|
+
}
|
|
2307
|
+
const rows = outcome.rows.length > 0 || !args.relaxedRecall ? outcome.rows : runMessageFtsQuery(args.db, args.sessionId, relaxedFtsQuery(args.query), fetchLimit, cutoff, args.dateRange, sessionFirst);
|
|
2308
|
+
const filtered = rows.slice(0, args.limit);
|
|
2309
|
+
return filtered.map((row, rank) => ({
|
|
2310
|
+
source: "message",
|
|
2311
|
+
content: previewText(row.content),
|
|
2312
|
+
score: linearDecayScore(rank, filtered.length),
|
|
2313
|
+
messageOrdinal: row.messageOrdinal,
|
|
2314
|
+
messageId: row.messageId,
|
|
2315
|
+
role: row.role
|
|
2316
|
+
}));
|
|
2317
|
+
}
|
|
2318
|
+
const sanitizedProbes = probes.map((probe) => ({ probe, query: sanitizeFtsQuery(probe) })).filter((entry) => entry.query.length > 0);
|
|
2319
|
+
const corpusSize = getIndexedMessageCorpusSize(args.db, args.sessionId, cutoff);
|
|
2320
|
+
const probeCounts = countSessionFtsMatchesBatch(args.db, args.sessionId, sanitizedProbes.map((entry) => entry.query), cutoff, args.dateRange, sessionFirst);
|
|
2321
|
+
const collectBaseDiagnostics = args.diagnostics !== undefined && cutoff !== null;
|
|
2322
|
+
const baseOutcome = collectBaseDiagnostics && baseQuery.length > 0 ? runMessageFtsQueryWithDiagnostics({
|
|
2323
|
+
db: args.db,
|
|
2324
|
+
sessionId: args.sessionId,
|
|
2325
|
+
ftsQuery: baseQuery,
|
|
2326
|
+
fetchLimit,
|
|
2327
|
+
cutoff,
|
|
2328
|
+
dateRange: args.dateRange,
|
|
2329
|
+
sessionFirst
|
|
2330
|
+
}) : null;
|
|
2331
|
+
if (args.diagnostics) {
|
|
2332
|
+
args.diagnostics.suppressedLiveMessageMatches = baseOutcome?.suppressedCount ?? 0;
|
|
2333
|
+
}
|
|
2334
|
+
const searchQueries = [
|
|
2335
|
+
...!collectBaseDiagnostics && baseQuery.length > 0 ? [baseQuery] : [],
|
|
2336
|
+
...sanitizedProbes.map((entry) => entry.query)
|
|
2337
|
+
];
|
|
2338
|
+
const rowsByQuery = runMessageFtsQueriesBatch(args.db, args.sessionId, searchQueries, fetchLimit, cutoff, args.dateRange, sessionFirst);
|
|
2339
|
+
const queryLists = [];
|
|
2340
|
+
let queryIndex = 0;
|
|
2341
|
+
if (baseQuery.length > 0) {
|
|
2342
|
+
queryLists.push({
|
|
2343
|
+
rows: baseOutcome?.rows ?? rowsByQuery[queryIndex] ?? [],
|
|
2344
|
+
weight: 1
|
|
2345
|
+
});
|
|
2346
|
+
if (!collectBaseDiagnostics)
|
|
2347
|
+
queryIndex += 1;
|
|
2348
|
+
}
|
|
2349
|
+
const probeWeights = new Map;
|
|
2350
|
+
sanitizedProbes.forEach((entry, probeIndex) => {
|
|
2351
|
+
const weight = probeDiscriminationWeight(probeCounts[probeIndex] ?? 0, corpusSize);
|
|
2352
|
+
probeWeights.set(entry.probe, weight);
|
|
2353
|
+
queryLists.push({ rows: rowsByQuery[queryIndex] ?? [], weight });
|
|
2354
|
+
queryIndex += 1;
|
|
2355
|
+
});
|
|
2356
|
+
const fused = new Map;
|
|
2357
|
+
for (const list of queryLists) {
|
|
2358
|
+
list.rows.forEach((row, rank) => {
|
|
2359
|
+
const rrf = list.weight / (RRF_K + rank);
|
|
2360
|
+
const existing = fused.get(row.messageId);
|
|
2361
|
+
if (existing) {
|
|
2362
|
+
existing.score += rrf;
|
|
2363
|
+
} else {
|
|
2364
|
+
fused.set(row.messageId, { row, score: rrf });
|
|
2365
|
+
}
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
for (const entry of fused.values()) {
|
|
2369
|
+
let best = 0;
|
|
2370
|
+
for (const probe of probes) {
|
|
2371
|
+
const weight = probeWeights.get(probe) ?? 0;
|
|
2372
|
+
if (weight > best && containsProbeVerbatim(entry.row.content, [probe])) {
|
|
2373
|
+
best = weight;
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
if (best > 0) {
|
|
2377
|
+
entry.score += best * VERBATIM_RANK_BONUS;
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
const ranked = [...fused.values()].sort((a, b) => b.score !== a.score ? b.score - a.score : a.row.messageOrdinal - b.row.messageOrdinal).slice(0, args.limit);
|
|
2381
|
+
return ranked.map((entry, rank) => ({
|
|
2382
|
+
source: "message",
|
|
2383
|
+
content: previewText(entry.row.content),
|
|
2384
|
+
score: linearDecayScore(rank, ranked.length),
|
|
2385
|
+
messageOrdinal: entry.row.messageOrdinal,
|
|
2386
|
+
messageId: entry.row.messageId,
|
|
2387
|
+
role: entry.row.role
|
|
2388
|
+
}));
|
|
2389
|
+
}
|
|
2390
|
+
var NOTE_SEARCHABLE_STATUSES = ["active", "pending", "ready"];
|
|
2391
|
+
var MAX_NOTE_KEYWORD_SCORE = 3.5;
|
|
2392
|
+
function noteSearchText(note) {
|
|
2393
|
+
const reason = note.readyReason?.trim();
|
|
2394
|
+
return reason ? `${note.content}
|
|
2395
|
+
Reason: ${reason}` : note.content;
|
|
2396
|
+
}
|
|
2397
|
+
function tokenizeKeywordNeedle(text) {
|
|
2398
|
+
const matches = text.toLowerCase().match(/[a-z0-9/._:-]+/g) ?? [];
|
|
2399
|
+
const seen = new Set;
|
|
2400
|
+
const tokens = [];
|
|
2401
|
+
for (const match of matches) {
|
|
2402
|
+
if (match.length <= 1 || !/[a-z0-9]/.test(match) || seen.has(match)) {
|
|
2403
|
+
continue;
|
|
2404
|
+
}
|
|
2405
|
+
seen.add(match);
|
|
2406
|
+
tokens.push(match);
|
|
2407
|
+
}
|
|
2408
|
+
return tokens;
|
|
2409
|
+
}
|
|
2410
|
+
function normalizeNoteKeywordScore(score) {
|
|
2411
|
+
return normalizeCosineScore(score / MAX_NOTE_KEYWORD_SCORE) * SINGLE_SOURCE_PENALTY;
|
|
2412
|
+
}
|
|
2413
|
+
function rankNotesForNeedle(notes, needle) {
|
|
2414
|
+
const normalizedNeedle = needle.trim().toLowerCase();
|
|
2415
|
+
if (normalizedNeedle.length === 0) {
|
|
2416
|
+
return [];
|
|
2417
|
+
}
|
|
2418
|
+
const needleTokens = tokenizeKeywordNeedle(normalizedNeedle);
|
|
2419
|
+
const ranked = [];
|
|
2420
|
+
for (const note of notes) {
|
|
2421
|
+
const text = noteSearchText(note);
|
|
2422
|
+
const normalizedText = text.toLowerCase();
|
|
2423
|
+
const noteTokens = new Set(tokenizeKeywordNeedle(normalizedText));
|
|
2424
|
+
const exact = normalizedText.includes(normalizedNeedle);
|
|
2425
|
+
const matchedTokens = needleTokens.filter((token) => noteTokens.has(token)).length;
|
|
2426
|
+
if (!exact && matchedTokens === 0) {
|
|
2427
|
+
continue;
|
|
2428
|
+
}
|
|
2429
|
+
const matchedUniqueTokens = new Set(needleTokens.filter((token) => noteTokens.has(token))).size;
|
|
2430
|
+
const coverage = needleTokens.length > 0 ? matchedTokens / needleTokens.length : 0;
|
|
2431
|
+
const density = noteTokens.size > 0 ? matchedUniqueTokens / noteTokens.size : 0;
|
|
2432
|
+
const exactPhrase = exact && (needleTokens.length > 1 || normalizedText.trim() === normalizedNeedle);
|
|
2433
|
+
const allTokens = needleTokens.length > 1 && matchedTokens === needleTokens.length;
|
|
2434
|
+
const score = (exactPhrase ? 2 : 0) + coverage * density + (allTokens ? 0.5 * density : 0);
|
|
2435
|
+
ranked.push({ note, score, text });
|
|
2436
|
+
}
|
|
2437
|
+
return ranked.sort((left, right) => {
|
|
2438
|
+
if (right.score !== left.score) {
|
|
2439
|
+
return right.score - left.score;
|
|
2440
|
+
}
|
|
2441
|
+
if (right.note.createdAt !== left.note.createdAt) {
|
|
2442
|
+
return right.note.createdAt - left.note.createdAt;
|
|
2443
|
+
}
|
|
2444
|
+
return left.note.id - right.note.id;
|
|
2445
|
+
});
|
|
2446
|
+
}
|
|
2447
|
+
function searchNotes(args) {
|
|
2448
|
+
if (args.limit <= 0) {
|
|
2449
|
+
return [];
|
|
2450
|
+
}
|
|
2451
|
+
const notes = [
|
|
2452
|
+
...getNotes(args.db, {
|
|
2453
|
+
sessionId: args.sessionId,
|
|
2454
|
+
type: "session",
|
|
2455
|
+
status: NOTE_SEARCHABLE_STATUSES
|
|
2456
|
+
}),
|
|
2457
|
+
...getNotes(args.db, {
|
|
2458
|
+
projectPath: args.projectPath,
|
|
2459
|
+
type: "smart",
|
|
2460
|
+
status: NOTE_SEARCHABLE_STATUSES
|
|
2461
|
+
})
|
|
2462
|
+
].filter((note) => timestampIsInRange(note.createdAt, args.dateRange));
|
|
2463
|
+
if (notes.length === 0) {
|
|
2464
|
+
return [];
|
|
2465
|
+
}
|
|
2466
|
+
const baseList = rankNotesForNeedle(notes, args.query);
|
|
2467
|
+
const probes = args.probes ?? [];
|
|
2468
|
+
if (probes.length === 0) {
|
|
2469
|
+
const ranked = baseList.slice(0, args.limit);
|
|
2470
|
+
return ranked.map((entry) => ({
|
|
2471
|
+
source: "note",
|
|
2472
|
+
content: previewText(entry.text),
|
|
2473
|
+
score: normalizeNoteKeywordScore(entry.score),
|
|
2474
|
+
noteId: entry.note.id,
|
|
2475
|
+
status: entry.note.status,
|
|
2476
|
+
createdAt: entry.note.createdAt,
|
|
2477
|
+
anchorOrdinal: entry.note.anchorOrdinal,
|
|
2478
|
+
sourceSessionId: entry.note.sessionId
|
|
2479
|
+
}));
|
|
2480
|
+
}
|
|
2481
|
+
const queryLists = [];
|
|
2482
|
+
if (baseList.length > 0) {
|
|
2483
|
+
queryLists.push({ rows: baseList, weight: 1 });
|
|
2484
|
+
}
|
|
2485
|
+
for (const probe of probes) {
|
|
2486
|
+
const rows = rankNotesForNeedle(notes, probe);
|
|
2487
|
+
if (rows.length === 0) {
|
|
2488
|
+
continue;
|
|
2489
|
+
}
|
|
2490
|
+
const weight = probeDiscriminationWeight(rows.length, notes.length);
|
|
2491
|
+
queryLists.push({ rows, weight });
|
|
2492
|
+
}
|
|
2493
|
+
const fused = new Map;
|
|
2494
|
+
for (const list of queryLists) {
|
|
2495
|
+
for (const row of list.rows) {
|
|
2496
|
+
const relevance = row.score * list.weight;
|
|
2497
|
+
const existing = fused.get(row.note.id);
|
|
2498
|
+
if (existing) {
|
|
2499
|
+
if (relevance > existing.score) {
|
|
2500
|
+
existing.entry = row;
|
|
2501
|
+
existing.score = relevance;
|
|
2502
|
+
}
|
|
2503
|
+
} else {
|
|
2504
|
+
fused.set(row.note.id, { entry: row, score: relevance });
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
const ranked = [...fused.values()].sort((left, right) => {
|
|
2509
|
+
if (right.score !== left.score) {
|
|
2510
|
+
return right.score - left.score;
|
|
2511
|
+
}
|
|
2512
|
+
if (right.entry.note.createdAt !== left.entry.note.createdAt) {
|
|
2513
|
+
return right.entry.note.createdAt - left.entry.note.createdAt;
|
|
2514
|
+
}
|
|
2515
|
+
return left.entry.note.id - right.entry.note.id;
|
|
2516
|
+
}).slice(0, args.limit);
|
|
2517
|
+
return ranked.map((entry) => ({
|
|
2518
|
+
source: "note",
|
|
2519
|
+
content: previewText(entry.entry.text),
|
|
2520
|
+
score: normalizeNoteKeywordScore(entry.score),
|
|
2521
|
+
noteId: entry.entry.note.id,
|
|
2522
|
+
status: entry.entry.note.status,
|
|
2523
|
+
createdAt: entry.entry.note.createdAt,
|
|
2524
|
+
anchorOrdinal: entry.entry.note.anchorOrdinal,
|
|
2525
|
+
sourceSessionId: entry.entry.note.sessionId
|
|
2526
|
+
}));
|
|
2527
|
+
}
|
|
2528
|
+
function searchCompartmentChunks(args) {
|
|
2529
|
+
if (!args.queryEmbedding || args.limit <= 0 || !args.modelId || args.modelId === "off")
|
|
2530
|
+
return [];
|
|
2531
|
+
const cutoff = args.maxOrdinal != null && args.maxOrdinal >= 0 ? args.maxOrdinal : null;
|
|
2532
|
+
const rows = loadCompartmentChunkEmbeddingsForSearch(args.db, args.sessionId, args.projectPath, args.modelId, args.dateRange);
|
|
2533
|
+
if (rows.length === 0)
|
|
2534
|
+
return [];
|
|
2535
|
+
const byCompartment = new Map;
|
|
2536
|
+
for (const row of rows) {
|
|
2537
|
+
if (cutoff !== null && row.endOrdinal > cutoff) {
|
|
2538
|
+
continue;
|
|
2539
|
+
}
|
|
2540
|
+
const score = normalizeCosineScore(cosineSimilarity(args.queryEmbedding, row.vector));
|
|
2541
|
+
if (score <= 0)
|
|
2542
|
+
continue;
|
|
2543
|
+
const existing = byCompartment.get(row.compartmentId);
|
|
2544
|
+
if (!existing || score > existing.score) {
|
|
2545
|
+
byCompartment.set(row.compartmentId, { row, score });
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
return [...byCompartment.values()].sort((left, right) => right.score !== left.score ? right.score - left.score : left.row.startOrdinal - right.row.startOrdinal).slice(0, args.limit).map(({ row, score }) => ({
|
|
2549
|
+
source: "compartment",
|
|
2550
|
+
content: previewText(row.title),
|
|
2551
|
+
score: score * SINGLE_SOURCE_PENALTY,
|
|
2552
|
+
compartmentId: row.compartmentId,
|
|
2553
|
+
sessionId: row.sessionId,
|
|
2554
|
+
title: row.title,
|
|
2555
|
+
startOrdinal: row.startOrdinal,
|
|
2556
|
+
endOrdinal: row.endOrdinal,
|
|
2557
|
+
matchType: "semantic"
|
|
2558
|
+
}));
|
|
2559
|
+
}
|
|
2560
|
+
function mergeMessageAndCompartmentResults(args) {
|
|
2561
|
+
if (args.compartments.length === 0)
|
|
2562
|
+
return args.messages;
|
|
2563
|
+
if (args.messages.length === 0)
|
|
2564
|
+
return args.compartments;
|
|
2565
|
+
const fused = new Map;
|
|
2566
|
+
const add = (key, result, score, tieOrdinal) => {
|
|
2567
|
+
const existing = fused.get(key);
|
|
2568
|
+
if (existing) {
|
|
2569
|
+
existing.score += score;
|
|
2570
|
+
return existing;
|
|
2571
|
+
}
|
|
2572
|
+
const entry = { result, score, tieOrdinal, snippetScore: -1 };
|
|
2573
|
+
fused.set(key, entry);
|
|
2574
|
+
return entry;
|
|
2575
|
+
};
|
|
2576
|
+
args.compartments.forEach((compartment, rank) => {
|
|
2577
|
+
add(`compartment:${compartment.compartmentId}`, compartment, 1 / (RRF_K + rank), compartment.startOrdinal);
|
|
2578
|
+
});
|
|
2579
|
+
for (const [rank, message] of args.messages.entries()) {
|
|
2580
|
+
const containing = args.compartments.find((compartment) => message.messageOrdinal >= compartment.startOrdinal && message.messageOrdinal <= compartment.endOrdinal);
|
|
2581
|
+
const contribution = 1 / (RRF_K + rank);
|
|
2582
|
+
if (!containing) {
|
|
2583
|
+
add(`message:${message.messageId}`, message, contribution, message.messageOrdinal);
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
const entry = add(`compartment:${containing.compartmentId}`, containing, contribution, containing.startOrdinal);
|
|
2587
|
+
if (message.score > entry.snippetScore && entry.result.source === "compartment") {
|
|
2588
|
+
entry.snippetScore = message.score;
|
|
2589
|
+
entry.result = {
|
|
2590
|
+
...entry.result,
|
|
2591
|
+
matchType: "hybrid",
|
|
2592
|
+
snippet: message.content
|
|
2593
|
+
};
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
const ranked = [...fused.values()].sort((left, right) => right.score !== left.score ? right.score - left.score : left.tieOrdinal - right.tieOrdinal).slice(0, args.limit);
|
|
2597
|
+
return ranked.map((entry, rank) => ({
|
|
2598
|
+
...entry.result,
|
|
2599
|
+
score: linearDecayScore(rank, ranked.length)
|
|
2600
|
+
}));
|
|
2601
|
+
}
|
|
2602
|
+
function getSourceBoost(result) {
|
|
2603
|
+
switch (result.source) {
|
|
2604
|
+
case "memory":
|
|
2605
|
+
return MEMORY_SOURCE_BOOST;
|
|
2606
|
+
case "message":
|
|
2607
|
+
case "compartment":
|
|
2608
|
+
return MESSAGE_SOURCE_BOOST;
|
|
2609
|
+
case "git_commit":
|
|
2610
|
+
return GIT_COMMIT_SOURCE_BOOST;
|
|
2611
|
+
case "primer":
|
|
2612
|
+
return PRIMER_SOURCE_BOOST;
|
|
2613
|
+
case "note":
|
|
2614
|
+
return 1;
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
function compareUnifiedResults(left, right) {
|
|
2618
|
+
const leftEffective = left.score * getSourceBoost(left);
|
|
2619
|
+
const rightEffective = right.score * getSourceBoost(right);
|
|
2620
|
+
if (rightEffective !== leftEffective) {
|
|
2621
|
+
return rightEffective - leftEffective;
|
|
2622
|
+
}
|
|
2623
|
+
if (left.source === "memory" && right.source === "memory") {
|
|
2624
|
+
return left.memoryId - right.memoryId;
|
|
2625
|
+
}
|
|
2626
|
+
if (left.source === "message" && right.source === "message") {
|
|
2627
|
+
return left.messageOrdinal - right.messageOrdinal;
|
|
2628
|
+
}
|
|
2629
|
+
if (left.source === "compartment" && right.source === "compartment") {
|
|
2630
|
+
return left.startOrdinal - right.startOrdinal;
|
|
2631
|
+
}
|
|
2632
|
+
if (left.source === "git_commit" && right.source === "git_commit") {
|
|
2633
|
+
return right.committedAtMs - left.committedAtMs;
|
|
2634
|
+
}
|
|
2635
|
+
if (left.source === "primer" && right.source === "primer") {
|
|
2636
|
+
return right.support - left.support || left.primerId - right.primerId;
|
|
2637
|
+
}
|
|
2638
|
+
if (left.source === "note" && right.source === "note") {
|
|
2639
|
+
return right.createdAt - left.createdAt || left.noteId - right.noteId;
|
|
2640
|
+
}
|
|
2641
|
+
return 0;
|
|
2642
|
+
}
|
|
2643
|
+
function toGitCommitResult(hit) {
|
|
2644
|
+
return {
|
|
2645
|
+
source: "git_commit",
|
|
2646
|
+
content: previewText(hit.commit.message),
|
|
2647
|
+
score: hit.score,
|
|
2648
|
+
sha: hit.commit.sha,
|
|
2649
|
+
shortSha: hit.commit.shortSha,
|
|
2650
|
+
author: hit.commit.author,
|
|
2651
|
+
committedAtMs: hit.commit.committedAtMs,
|
|
2652
|
+
matchType: hit.matchType
|
|
2653
|
+
};
|
|
2654
|
+
}
|
|
2655
|
+
function searchGitCommits(args) {
|
|
2656
|
+
if (args.limit <= 0)
|
|
2657
|
+
return [];
|
|
2658
|
+
const hits = searchGitCommitsSync(args.db, args.projectPath, args.query, {
|
|
2659
|
+
limit: args.limit,
|
|
2660
|
+
queryEmbedding: args.queryEmbedding,
|
|
2661
|
+
queryModelId: args.queryModelId,
|
|
2662
|
+
from: args.dateRange?.from,
|
|
2663
|
+
to: args.dateRange?.to
|
|
2664
|
+
});
|
|
2665
|
+
return hits.map(toGitCommitResult);
|
|
2666
|
+
}
|
|
2667
|
+
function primerText(primer) {
|
|
2668
|
+
const answer = primer.answer.trim();
|
|
2669
|
+
return answer ? `Q: ${primer.question}
|
|
2670
|
+
A: ${answer}` : `Q: ${primer.question}`;
|
|
2671
|
+
}
|
|
2672
|
+
function searchPrimers(args) {
|
|
2673
|
+
const primers = getActivePrimers(args.db, args.projectPath).filter((primer) => timestampIsInRange(primer.createdAt, args.dateRange));
|
|
2674
|
+
if (primers.length === 0 || args.limit <= 0)
|
|
2675
|
+
return [];
|
|
2676
|
+
const ftsQuery = sanitizeFtsQuery(args.query);
|
|
2677
|
+
const ftsRanks = new Map;
|
|
2678
|
+
if (ftsQuery) {
|
|
2679
|
+
const rows = args.db.prepare(`SELECT p.id AS id, bm25(primers_fts) AS rank
|
|
2680
|
+
FROM primers_fts
|
|
2681
|
+
JOIN primers p ON p.id = primers_fts.rowid
|
|
2682
|
+
WHERE primers_fts MATCH ? AND p.project_path = ? AND p.status = 'active'
|
|
2683
|
+
${args.dateRange === null ? "" : "AND p.created_at BETWEEN ? AND ?"}
|
|
2684
|
+
ORDER BY rank ASC
|
|
2685
|
+
LIMIT ?`).all(ftsQuery, args.projectPath, ...args.dateRange === null ? [] : [args.dateRange.from, args.dateRange.to], args.limit * 3);
|
|
2686
|
+
rows.forEach((row, index) => {
|
|
2687
|
+
ftsRanks.set(row.id, linearDecayScore(index, rows.length));
|
|
2688
|
+
});
|
|
2689
|
+
}
|
|
2690
|
+
const scored = primers.map((primer) => {
|
|
2691
|
+
const semantic = args.queryEmbedding && primer.questionEmbedding && primer.questionEmbeddingModelId === args.queryModelId ? normalizeCosineScore(cosineSimilarity(args.queryEmbedding, primer.questionEmbedding)) : 0;
|
|
2692
|
+
const fts = ftsRanks.get(primer.id) ?? 0;
|
|
2693
|
+
if (semantic <= 0 && fts <= 0)
|
|
2694
|
+
return null;
|
|
2695
|
+
const score = semantic > 0 && fts > 0 ? semantic * SEMANTIC_WEIGHT + fts * FTS_WEIGHT : Math.max(semantic, fts);
|
|
2696
|
+
return {
|
|
2697
|
+
source: "primer",
|
|
2698
|
+
content: previewText(primerText(primer)),
|
|
2699
|
+
score,
|
|
2700
|
+
primerId: primer.id,
|
|
2701
|
+
question: primer.question,
|
|
2702
|
+
support: primer.totalSupport,
|
|
2703
|
+
lastObservedAt: primer.lastObservedAt,
|
|
2704
|
+
matchType: semantic > 0 && fts > 0 ? "hybrid" : semantic > 0 ? "semantic" : "fts"
|
|
2705
|
+
};
|
|
2706
|
+
}).filter((result) => result !== null).sort((a, b) => b.score - a.score || b.support - a.support || a.primerId - b.primerId).slice(0, args.limit);
|
|
2707
|
+
return scored;
|
|
2708
|
+
}
|
|
2709
|
+
function resolveSources(sources) {
|
|
2710
|
+
if (sources === undefined) {
|
|
2711
|
+
return new Set(["memory", "message", "git_commit", "primer", "note"]);
|
|
2712
|
+
}
|
|
2713
|
+
const set = new Set;
|
|
2714
|
+
for (const source of sources) {
|
|
2715
|
+
if (source === "memory" || source === "message" || source === "git_commit" || source === "primer" || source === "note") {
|
|
2716
|
+
set.add(source);
|
|
2717
|
+
}
|
|
2718
|
+
}
|
|
2719
|
+
return set;
|
|
2720
|
+
}
|
|
2721
|
+
function memoriesToIdLookupResults(args) {
|
|
2722
|
+
const ordered = args.memories.slice(0, args.limit);
|
|
2723
|
+
return ordered.map((memory, rank) => ({
|
|
2724
|
+
source: "memory",
|
|
2725
|
+
content: previewText(memory.content),
|
|
2726
|
+
score: 1 - rank * 0.01,
|
|
2727
|
+
memoryId: memory.id,
|
|
2728
|
+
category: memory.category,
|
|
2729
|
+
matchType: "fts",
|
|
2730
|
+
sourceName: args.sourceNameByMemoryId?.get(memory.id)
|
|
2731
|
+
}));
|
|
2732
|
+
}
|
|
2733
|
+
function resolveMemoriesByIdsForSearch2(args) {
|
|
2734
|
+
if (args.diagnostics) {
|
|
2735
|
+
args.diagnostics.suppressedVisibleMemoryIds = [];
|
|
2736
|
+
}
|
|
2737
|
+
if (args.ids.length === 0) {
|
|
2738
|
+
return null;
|
|
2739
|
+
}
|
|
2740
|
+
const workspace = resolveSearchWorkspaceContext(args.db, args.projectPath);
|
|
2741
|
+
const fetched = workspace.isWorkspaced ? getMemoriesByProjects2(args.db, workspace.expandedIdentities, ["active", "permanent", "archived"], Date.now(), workspace.ownIdentities, workspace.shareCategories) : getMemoriesByProject2(args.db, args.projectPath, ["active", "permanent", "archived"]);
|
|
2742
|
+
if (fetched.length === 0) {
|
|
2743
|
+
return null;
|
|
2744
|
+
}
|
|
2745
|
+
const dateRange = normalizeDateRange(args.from, args.to);
|
|
2746
|
+
const memoriesById = new Map(fetched.filter((memory) => timestampIsInRange(memory.createdAt, dateRange)).map((memory) => [memory.id, memory]));
|
|
2747
|
+
const ordered = [];
|
|
2748
|
+
const suppressedVisibleIds = new Set;
|
|
2749
|
+
for (const id of args.ids) {
|
|
2750
|
+
const memory = memoriesById.get(id);
|
|
2751
|
+
if (!memory)
|
|
2752
|
+
continue;
|
|
2753
|
+
if (args.visibleMemoryIds?.has(id)) {
|
|
2754
|
+
suppressedVisibleIds.add(id);
|
|
2755
|
+
continue;
|
|
2756
|
+
}
|
|
2757
|
+
ordered.push(memory);
|
|
2758
|
+
if (ordered.length >= args.limit)
|
|
2759
|
+
break;
|
|
2760
|
+
}
|
|
2761
|
+
if (args.diagnostics) {
|
|
2762
|
+
args.diagnostics.suppressedVisibleMemoryIds = [...suppressedVisibleIds].sort((left, right) => left - right);
|
|
2763
|
+
}
|
|
2764
|
+
if (ordered.length === 0) {
|
|
2765
|
+
return null;
|
|
2766
|
+
}
|
|
2767
|
+
return memoriesToIdLookupResults({
|
|
2768
|
+
memories: ordered,
|
|
2769
|
+
limit: args.limit,
|
|
2770
|
+
sourceNameByMemoryId: sourceNamesForSearchMemories({
|
|
2771
|
+
memories: ordered,
|
|
2772
|
+
projectPath: args.projectPath,
|
|
2773
|
+
workspace
|
|
2774
|
+
})
|
|
2775
|
+
});
|
|
2776
|
+
}
|
|
2777
|
+
async function unifiedSearch2(db, sessionId, projectPath, query, options = {}) {
|
|
2778
|
+
const trimmedQuery = query.trim();
|
|
2779
|
+
const measurementStartedAt = Date.now();
|
|
2780
|
+
if (trimmedQuery.length === 0 || options.signal?.aborted) {
|
|
2781
|
+
return [];
|
|
2782
|
+
}
|
|
2783
|
+
const limit = normalizeLimit(options.limit);
|
|
2784
|
+
const dateRange = normalizeDateRange(options.from, options.to);
|
|
2785
|
+
const tierLimit = Math.max(limit * 3, DEFAULT_UNIFIED_SEARCH_LIMIT);
|
|
2786
|
+
if (options.diagnostics) {
|
|
2787
|
+
options.diagnostics.suppressedVisibleMemoryIds = [];
|
|
2788
|
+
options.diagnostics.suppressedLiveMessageMatches = 0;
|
|
2789
|
+
options.diagnostics.gitCommitUnavailable = null;
|
|
2790
|
+
}
|
|
2791
|
+
const embeddingEnabled = options.embeddingEnabled ?? true;
|
|
2792
|
+
const embedQuery = options.embedQuery ?? embedText;
|
|
2793
|
+
const isEmbeddingRuntimeEnabled = options.isEmbeddingRuntimeEnabled ?? isEmbeddingEnabled;
|
|
2794
|
+
const gitCommitsEnabled = options.gitCommitsEnabled ?? false;
|
|
2795
|
+
const activeSources = resolveSources(options.sources);
|
|
2796
|
+
const memoryFeatureEnabled = options.memoryEnabled ?? true;
|
|
2797
|
+
const runMemory = activeSources.has("memory") && memoryFeatureEnabled;
|
|
2798
|
+
const runMessages = activeSources.has("message");
|
|
2799
|
+
const runGitCommits = activeSources.has("git_commit") && gitCommitsEnabled;
|
|
2800
|
+
if (options.diagnostics && activeSources.has("git_commit") && options.gitRepositoryAvailable === false) {
|
|
2801
|
+
options.diagnostics.gitCommitUnavailable = "no_git_repository";
|
|
2802
|
+
}
|
|
2803
|
+
const runPrimers = activeSources.has("primer") && memoryFeatureEnabled;
|
|
2804
|
+
const runNotes = activeSources.has("note");
|
|
2805
|
+
const runCompartmentChunks = runMessages && embeddingEnabled;
|
|
2806
|
+
const needsEmbedding = (runMemory || runGitCommits || runCompartmentChunks || runPrimers) && embeddingEnabled && isEmbeddingRuntimeEnabled();
|
|
2807
|
+
const queryEmbeddingPromise = needsEmbedding ? embedQuery(trimmedQuery, options.signal).catch((error) => {
|
|
2808
|
+
log2(`[search] query embedding failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2809
|
+
return null;
|
|
2810
|
+
}) : Promise.resolve(null);
|
|
2811
|
+
await Promise.resolve();
|
|
2812
|
+
if (options.signal?.aborted)
|
|
2813
|
+
return [];
|
|
2814
|
+
const messageProbes = options.explicitSearch ? extractLiteralProbes(trimmedQuery) : [];
|
|
2815
|
+
let messageResults = [];
|
|
2816
|
+
if (runMessages) {
|
|
2817
|
+
const messageRequest = {
|
|
2818
|
+
sessionId,
|
|
2819
|
+
query: trimmedQuery,
|
|
2820
|
+
limit: tierLimit,
|
|
2821
|
+
maxOrdinal: options.maxMessageOrdinal,
|
|
2822
|
+
probes: messageProbes,
|
|
2823
|
+
relaxedRecall: options.explicitSearch,
|
|
2824
|
+
collectDiagnostics: options.diagnostics !== undefined,
|
|
2825
|
+
dateRange
|
|
2826
|
+
};
|
|
2827
|
+
const delegated = options.searchMessageHistory ? await options.searchMessageHistory(messageRequest).catch((error) => {
|
|
2828
|
+
log2(`[search] off-thread message search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2829
|
+
return null;
|
|
2830
|
+
}) : null;
|
|
2831
|
+
if (delegated && options.signal?.aborted)
|
|
2832
|
+
return [];
|
|
2833
|
+
const outcome = delegated ?? searchMessageHistory2(db, messageRequest);
|
|
2834
|
+
if (options.diagnostics) {
|
|
2835
|
+
options.diagnostics.suppressedLiveMessageMatches = outcome.suppressedLiveMessageMatches;
|
|
2836
|
+
}
|
|
2837
|
+
messageResults = outcome.results;
|
|
2838
|
+
}
|
|
2839
|
+
const capturedQuery = await queryEmbeddingPromise;
|
|
2840
|
+
if (options.signal?.aborted)
|
|
2841
|
+
return [];
|
|
2842
|
+
const embeddingSnapshot = getProjectEmbeddingSnapshot2(projectPath);
|
|
2843
|
+
const queryContract = capturedQuery instanceof Float32Array || capturedQuery === null ? null : capturedQuery;
|
|
2844
|
+
const generationIsCurrent = queryContract === null || embeddingSnapshot !== null && embeddingSnapshot.generation === queryContract.generation;
|
|
2845
|
+
const queryEmbedding = generationIsCurrent ? queryContract?.vector ?? (capturedQuery instanceof Float32Array ? capturedQuery : null) : null;
|
|
2846
|
+
const workspace = resolveSearchWorkspaceContext(db, projectPath);
|
|
2847
|
+
const embeddingModelId = queryContract?.modelId ?? options.embeddingModelIdOverride ?? embeddingSnapshot?.modelId;
|
|
2848
|
+
const chunkModelId = queryContract?.chunkModelId ?? options.chunkModelIdOverride ?? embeddingSnapshot?.chunkModelId;
|
|
2849
|
+
const compartmentResults = runCompartmentChunks ? searchCompartmentChunks({
|
|
2850
|
+
db,
|
|
2851
|
+
sessionId,
|
|
2852
|
+
projectPath,
|
|
2853
|
+
queryEmbedding,
|
|
2854
|
+
limit: tierLimit,
|
|
2855
|
+
maxOrdinal: options.maxMessageOrdinal,
|
|
2856
|
+
modelId: chunkModelId && chunkModelId !== "off" ? chunkModelId : null,
|
|
2857
|
+
dateRange
|
|
2858
|
+
}) : [];
|
|
2859
|
+
const messageLikeResults = mergeMessageAndCompartmentResults({
|
|
2860
|
+
messages: messageResults,
|
|
2861
|
+
compartments: compartmentResults,
|
|
2862
|
+
limit: tierLimit
|
|
2863
|
+
});
|
|
2864
|
+
const [memoryOutcome, gitCommitResults, primerResults, noteResults] = await Promise.all([
|
|
2865
|
+
runMemory ? searchMemories({
|
|
2866
|
+
db,
|
|
2867
|
+
projectPath,
|
|
2868
|
+
query: trimmedQuery,
|
|
2869
|
+
limit: tierLimit,
|
|
2870
|
+
memoryEnabled: true,
|
|
2871
|
+
queryEmbedding,
|
|
2872
|
+
queryModelId: embeddingModelId && embeddingModelId !== "off" ? embeddingModelId : null,
|
|
2873
|
+
workspace,
|
|
2874
|
+
visibleMemoryIds: options.visibleMemoryIds,
|
|
2875
|
+
backfillMemoryEmbeddings: options.backfillMemoryEmbeddings,
|
|
2876
|
+
dateRange
|
|
2877
|
+
}) : Promise.resolve({
|
|
2878
|
+
results: [],
|
|
2879
|
+
suppressedVisibleIds: []
|
|
2880
|
+
}),
|
|
2881
|
+
runGitCommits ? Promise.resolve(searchGitCommits({
|
|
2882
|
+
db,
|
|
2883
|
+
projectPath,
|
|
2884
|
+
query: trimmedQuery,
|
|
2885
|
+
limit: tierLimit,
|
|
2886
|
+
queryEmbedding,
|
|
2887
|
+
queryModelId: embeddingModelId && embeddingModelId !== "off" ? embeddingModelId : null,
|
|
2888
|
+
dateRange
|
|
2889
|
+
})) : Promise.resolve([]),
|
|
2890
|
+
runPrimers ? Promise.resolve(searchPrimers({
|
|
2891
|
+
db,
|
|
2892
|
+
projectPath,
|
|
2893
|
+
query: trimmedQuery,
|
|
2894
|
+
limit: tierLimit,
|
|
2895
|
+
queryEmbedding,
|
|
2896
|
+
queryModelId: embeddingModelId && embeddingModelId !== "off" ? embeddingModelId : null,
|
|
2897
|
+
dateRange
|
|
2898
|
+
})) : Promise.resolve([]),
|
|
2899
|
+
runNotes ? Promise.resolve(searchNotes({
|
|
2900
|
+
db,
|
|
2901
|
+
sessionId,
|
|
2902
|
+
projectPath,
|
|
2903
|
+
query: trimmedQuery,
|
|
2904
|
+
limit: tierLimit,
|
|
2905
|
+
probes: messageProbes,
|
|
2906
|
+
dateRange
|
|
2907
|
+
})) : Promise.resolve([])
|
|
2908
|
+
]);
|
|
2909
|
+
if (options.diagnostics) {
|
|
2910
|
+
options.diagnostics.suppressedVisibleMemoryIds = memoryOutcome.suppressedVisibleIds;
|
|
2911
|
+
}
|
|
2912
|
+
const results = [
|
|
2913
|
+
...memoryOutcome.results,
|
|
2914
|
+
...primerResults,
|
|
2915
|
+
...messageLikeResults,
|
|
2916
|
+
...gitCommitResults,
|
|
2917
|
+
...noteResults
|
|
2918
|
+
].sort(compareUnifiedResults).slice(0, limit);
|
|
2919
|
+
if (!options.measurementDisabled) {
|
|
2920
|
+
recordShadowMeasurement({
|
|
2921
|
+
db,
|
|
2922
|
+
sessionId,
|
|
2923
|
+
projectPath,
|
|
2924
|
+
query: trimmedQuery,
|
|
2925
|
+
options,
|
|
2926
|
+
primaryResults: results,
|
|
2927
|
+
primaryQuery: queryContract,
|
|
2928
|
+
primaryLatencyMs: Date.now() - measurementStartedAt,
|
|
2929
|
+
search: unifiedSearch2
|
|
2930
|
+
});
|
|
2931
|
+
}
|
|
2932
|
+
const countRetrievals = options.countRetrievals ?? true;
|
|
2933
|
+
if (countRetrievals) {
|
|
2934
|
+
const memoryIds = results.filter((result) => result.source === "memory").map((result) => result.memoryId);
|
|
2935
|
+
if (memoryIds.length > 0) {
|
|
2936
|
+
db.transaction(() => {
|
|
2937
|
+
for (const memoryId of memoryIds) {
|
|
2938
|
+
try {
|
|
2939
|
+
updateMemoryRetrievalCount2(db, memoryId);
|
|
2940
|
+
} catch (error) {
|
|
2941
|
+
if (error instanceof ModuleMemoryAuthorityError2)
|
|
2942
|
+
continue;
|
|
2943
|
+
throw error;
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
}).immediate();
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
return results;
|
|
2950
|
+
}
|
|
2951
|
+
var SEARCH_NOTE_EXPAND_HINT = "Use ctx_expand(start=N-10, end=N) around any note @msg anchor above to read the surrounding conversation context.";
|
|
2952
|
+
function formatSearchAge(timestampMs) {
|
|
2953
|
+
const ageMs = Date.now() - timestampMs;
|
|
2954
|
+
if (ageMs < 0)
|
|
2955
|
+
return "future";
|
|
2956
|
+
const days = Math.floor(ageMs / (24 * 60 * 60 * 1000));
|
|
2957
|
+
if (days <= 0)
|
|
2958
|
+
return "today";
|
|
2959
|
+
if (days === 1)
|
|
2960
|
+
return "1d ago";
|
|
2961
|
+
if (days < 30)
|
|
2962
|
+
return `${days}d ago`;
|
|
2963
|
+
const months = Math.floor(days / 30);
|
|
2964
|
+
if (months === 1)
|
|
2965
|
+
return "1mo ago";
|
|
2966
|
+
if (months < 12)
|
|
2967
|
+
return `${months}mo ago`;
|
|
2968
|
+
const years = Math.floor(days / 365);
|
|
2969
|
+
return years === 1 ? "1y ago" : `${years}y ago`;
|
|
2970
|
+
}
|
|
2971
|
+
function formatUnifiedSearchResult(result, index, currentSessionId) {
|
|
2972
|
+
if (result.source === "memory") {
|
|
2973
|
+
const source = result.sourceName ? ` source=${result.sourceName}` : "";
|
|
2974
|
+
return [
|
|
2975
|
+
`[${index}] [memory] score=${result.score.toFixed(2)} id=${result.memoryId} category=${result.category}${source} match=${result.matchType}`,
|
|
2976
|
+
result.content
|
|
2977
|
+
].join(`
|
|
2978
|
+
`);
|
|
2979
|
+
}
|
|
2980
|
+
if (result.source === "git_commit") {
|
|
2981
|
+
return [
|
|
2982
|
+
`[${index}] [git_commit] score=${result.score.toFixed(2)} sha=${result.shortSha} ${formatSearchAge(result.committedAtMs)} match=${result.matchType}`,
|
|
2983
|
+
result.content
|
|
2984
|
+
].join(`
|
|
2985
|
+
`);
|
|
2986
|
+
}
|
|
2987
|
+
if (result.source === "primer") {
|
|
2988
|
+
return [
|
|
2989
|
+
`[${index}] [primer] score=${result.score.toFixed(2)} id=${result.primerId} support=${result.support} match=${result.matchType}`,
|
|
2990
|
+
result.content
|
|
2991
|
+
].join(`
|
|
2992
|
+
`);
|
|
2993
|
+
}
|
|
2994
|
+
if (result.source === "note") {
|
|
2995
|
+
const anchor = result.anchorOrdinal !== null && result.sourceSessionId === currentSessionId ? ` @msg ${result.anchorOrdinal}` : "";
|
|
2996
|
+
return [
|
|
2997
|
+
`[${index}] [note] score=${result.score.toFixed(2)} id=#${result.noteId} status=${result.status} ${formatSearchAge(result.createdAt)}${anchor}`,
|
|
2998
|
+
result.content
|
|
2999
|
+
].join(`
|
|
3000
|
+
`);
|
|
3001
|
+
}
|
|
3002
|
+
if (result.source === "compartment") {
|
|
3003
|
+
return [
|
|
3004
|
+
`[${index}] [message] score=${result.score.toFixed(2)} compartment_id=${result.compartmentId} range=${result.startOrdinal}-${result.endOrdinal} match=${result.matchType} title=${result.title}`,
|
|
3005
|
+
result.snippet ? `Snippet: ${result.snippet}` : result.content
|
|
3006
|
+
].join(`
|
|
3007
|
+
`);
|
|
3008
|
+
}
|
|
3009
|
+
const expandStart = Math.max(1, result.messageOrdinal - 3);
|
|
3010
|
+
const expandEnd = result.messageOrdinal + 3;
|
|
3011
|
+
return [
|
|
3012
|
+
`[${index}] [message] score=${result.score.toFixed(2)} ordinal=${result.messageOrdinal} range=${expandStart}-${expandEnd} role=${result.role}`,
|
|
3013
|
+
result.content
|
|
3014
|
+
].join(`
|
|
3015
|
+
`);
|
|
3016
|
+
}
|
|
3017
|
+
function formatSearchDiagnosticLines(results, diagnostics) {
|
|
3018
|
+
if (!diagnostics)
|
|
3019
|
+
return [];
|
|
3020
|
+
const lines = [];
|
|
3021
|
+
const visibleIds = [...diagnostics.suppressedVisibleMemoryIds].sort((left, right) => left - right);
|
|
3022
|
+
if (visibleIds.length > 0) {
|
|
3023
|
+
const count = visibleIds.length;
|
|
3024
|
+
const noun = count === 1 ? "match" : "matches";
|
|
3025
|
+
const ids = visibleIds.join(", ");
|
|
3026
|
+
if (results.some((result) => result.source === "memory")) {
|
|
3027
|
+
lines.push(`Memories: ${count} additional ${noun} suppressed because ${count === 1 ? "it is" : "they are"} already visible in your project-memory block (ids ${ids}).`);
|
|
3028
|
+
} else {
|
|
3029
|
+
lines.push(`Memories: ${count} ${noun} found, all already visible in your project-memory block (ids ${ids}).`);
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
if (diagnostics.suppressedLiveMessageMatches > 0) {
|
|
3033
|
+
const count = diagnostics.suppressedLiveMessageMatches;
|
|
3034
|
+
lines.push(`Message history: ${count} raw-message ${count === 1 ? "match is" : "matches are"} newer than the last compartment boundary (already in your context).`);
|
|
3035
|
+
}
|
|
3036
|
+
if (diagnostics.gitCommitUnavailable === "no_git_repository") {
|
|
3037
|
+
lines.push("Git commits: no git repository — commit search unavailable for this project.");
|
|
3038
|
+
}
|
|
3039
|
+
return lines;
|
|
3040
|
+
}
|
|
3041
|
+
function formatSearchResults2(query, results, currentSessionId, diagnostics) {
|
|
3042
|
+
const diagnosticLines = formatSearchDiagnosticLines(results, diagnostics);
|
|
3043
|
+
if (results.length === 0) {
|
|
3044
|
+
if (diagnosticLines.length > 0) {
|
|
3045
|
+
return `No hidden results found for "${query}".
|
|
3046
|
+
|
|
3047
|
+
${diagnosticLines.join(`
|
|
3048
|
+
`)}`;
|
|
3049
|
+
}
|
|
3050
|
+
return `No results found for "${query}" across notes, memories, primers, git commits, or message history.`;
|
|
3051
|
+
}
|
|
3052
|
+
const bodyParts = results.map((result, index) => formatUnifiedSearchResult(result, index + 1, currentSessionId));
|
|
3053
|
+
if (diagnosticLines.length > 0)
|
|
3054
|
+
bodyParts.push(diagnosticLines.join(`
|
|
3055
|
+
`));
|
|
3056
|
+
if (results.some((result) => result.source === "message" || result.source === "compartment")) {
|
|
3057
|
+
bodyParts.push("Use ctx_expand(start, end) with the range from any message result above to read the full conversation context.");
|
|
3058
|
+
}
|
|
3059
|
+
if (results.some((result) => result.source === "note" && result.anchorOrdinal !== null && result.sourceSessionId === currentSessionId)) {
|
|
3060
|
+
bodyParts.push(SEARCH_NOTE_EXPAND_HINT);
|
|
3061
|
+
}
|
|
3062
|
+
return `Found ${results.length} result${results.length === 1 ? "" : "s"} for "${query}":
|
|
3063
|
+
|
|
3064
|
+
${bodyParts.join(`
|
|
3065
|
+
|
|
3066
|
+
`)}`;
|
|
3067
|
+
}
|
|
3068
|
+
|
|
3069
|
+
export { getNoteByIdInScope, getNotes, addNote, getSessionNotes, getPendingSmartNotes, getReadySmartNotes, updateNote, dismissNotes, dismissNote, markNoteReady, markNoteChecked, PRIMER_CANDIDATE_TTL_MS, PRIMER_CANDIDATE_MAX_AGE_MS, primerOccurrenceKey, primerOccurrenceUtcDay, insertPrimerCandidates, updatePrimerCandidateEmbedding, getPrimerCandidatesByIds, getPrimerCandidatesForPromotion, countPrimerCandidatesForProject, getActivePrimers, createPrimer, updatePrimerSupport, updatePrimerAnswer, promoteSessionFactsDurable, embedPromotedFacts, recordMemoryMapping, recordMemoryVerifications, getUnmappedMemoryIds, clearMemoryVerifications, getMemoryVerifications, resolveGitTopLevel, readGitHead, readGitChangedFilesSince, readGitFileChangeTimesSince, verificationFileExists, normalizeVerificationFiles, indexCommitsForProject, embedUnembeddedCommits, createUnifiedSearchDiagnostics2, searchMessageHistory2, parseIdShapedQuery2, resolveMemoriesByIdsForSearch2, unifiedSearch2, formatSearchResults2 };
|