@bojackduy/opencode-telescope 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/components/preview.tsx +30 -143
- package/components/result-list.tsx +17 -1
- package/package.json +2 -1
- package/search/db-path.ts +67 -0
- package/search/dependencies.test.ts +38 -0
- package/search/dependencies.ts +63 -0
- package/search/embedding.test.ts +83 -0
- package/search/embedding.ts +67 -0
- package/search/queries.ts +813 -0
- package/search/schema.ts +120 -0
- package/search/text.ts +348 -0
- package/search/types.ts +131 -0
- package/search/vector.ts +223 -0
- package/search.ts +47 -1510
- package/telescope.tsx +2 -2
- package/ui/format.test.ts +120 -0
- package/ui/preview-utils.test.ts +161 -0
- package/ui/preview-utils.ts +136 -0
- package/ui/render-target.test.ts +55 -0
package/search.ts
CHANGED
|
@@ -1,1510 +1,47 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export type ConversationPreviewPage = {
|
|
50
|
-
parts: ConversationPreviewPart[]
|
|
51
|
-
hasMoreBefore: boolean
|
|
52
|
-
hasMoreAfter: boolean
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
export type ConversationPreviewCursor = {
|
|
56
|
-
id: string
|
|
57
|
-
timeCreated: number
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export type ToolState = {
|
|
61
|
-
status: "pending" | "running" | "completed" | "error"
|
|
62
|
-
input?: unknown
|
|
63
|
-
metadata?: unknown
|
|
64
|
-
output?: string
|
|
65
|
-
error?: string
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
export type SemanticConfig = {
|
|
69
|
-
embedBaseUrl: string
|
|
70
|
-
embedModel?: string
|
|
71
|
-
disableVector: boolean
|
|
72
|
-
sqliteLibPath?: string
|
|
73
|
-
sqliteVecExtension?: string
|
|
74
|
-
hybridAlpha: number
|
|
75
|
-
documentPrefix: string
|
|
76
|
-
queryPrefix: string
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export type VectorState = "enabled" | "disabled" | "unavailable" | "stale"
|
|
80
|
-
|
|
81
|
-
export const DEFAULT_EMBED_BASE_URL = "http://127.0.0.1:8081"
|
|
82
|
-
export const DEFAULT_DOCUMENT_PREFIX = "search_document: "
|
|
83
|
-
export const DEFAULT_QUERY_PREFIX = "search_query: "
|
|
84
|
-
export const DEFAULT_HYBRID_ALPHA = 0.45
|
|
85
|
-
|
|
86
|
-
export function parseSemanticConfig(env: Record<string, string | undefined> = process.env): SemanticConfig {
|
|
87
|
-
return {
|
|
88
|
-
embedBaseUrl: env.OPENCODE_TELESCOPE_EMBED_BASE_URL ?? DEFAULT_EMBED_BASE_URL,
|
|
89
|
-
embedModel: env.OPENCODE_TELESCOPE_EMBED_MODEL || undefined,
|
|
90
|
-
disableVector: env.OPENCODE_TELESCOPE_DISABLE_VECTOR === "1" || env.OPENCODE_TELESCOPE_DISABLE_VECTOR === "true",
|
|
91
|
-
sqliteLibPath: env.OPENCODE_TELESCOPE_SQLITE_LIB || undefined,
|
|
92
|
-
sqliteVecExtension: env.OPENCODE_TELESCOPE_SQLITE_VEC_EXT || undefined,
|
|
93
|
-
hybridAlpha: parseAlpha(env.OPENCODE_TELESCOPE_HYBRID_ALPHA),
|
|
94
|
-
documentPrefix: DEFAULT_DOCUMENT_PREFIX,
|
|
95
|
-
queryPrefix: DEFAULT_QUERY_PREFIX,
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function parseAlpha(value: string | undefined) {
|
|
100
|
-
if (!value) return DEFAULT_HYBRID_ALPHA
|
|
101
|
-
const parsed = Number(value)
|
|
102
|
-
if (!Number.isFinite(parsed)) return DEFAULT_HYBRID_ALPHA
|
|
103
|
-
if (parsed < 0) return 0
|
|
104
|
-
if (parsed > 1) return 1
|
|
105
|
-
return parsed
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
type Row = {
|
|
109
|
-
id: string
|
|
110
|
-
message_id: string
|
|
111
|
-
session_id: string
|
|
112
|
-
session_title: string | null
|
|
113
|
-
directory: string
|
|
114
|
-
role: SearchRole
|
|
115
|
-
part_type?: SearchResult["partType"]
|
|
116
|
-
tool?: string | null
|
|
117
|
-
time_created: number
|
|
118
|
-
text: string
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
type IndexSourceRow = Omit<Row, "text"> & {
|
|
122
|
-
data: string
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export type DocumentRow = {
|
|
126
|
-
doc_id: string
|
|
127
|
-
part_id: string
|
|
128
|
-
message_id: string
|
|
129
|
-
session_id: string
|
|
130
|
-
session_title: string
|
|
131
|
-
directory: string
|
|
132
|
-
role: string
|
|
133
|
-
part_type: string
|
|
134
|
-
tool: string | null
|
|
135
|
-
time_created: number
|
|
136
|
-
chunk_index: number
|
|
137
|
-
text: string
|
|
138
|
-
source_hash: string
|
|
139
|
-
extractor_version: string
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
type ConversationRow = {
|
|
143
|
-
id: string
|
|
144
|
-
message_id: string
|
|
145
|
-
session_id: string
|
|
146
|
-
role: SearchRole
|
|
147
|
-
type: "text" | "reasoning" | "tool"
|
|
148
|
-
time_created: number
|
|
149
|
-
data: string
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
let cachedDbPath: string | undefined
|
|
153
|
-
let _db: Database | undefined
|
|
154
|
-
let _dbPath: string | undefined
|
|
155
|
-
let _indexDb: Database | undefined
|
|
156
|
-
let _indexDbPath: string | undefined
|
|
157
|
-
const backgroundIndexRebuilds = new Set<string>()
|
|
158
|
-
|
|
159
|
-
function getDb(dbPath?: string): Database {
|
|
160
|
-
const resolved = dbPath ?? resolveDatabasePath()
|
|
161
|
-
if (!_db || resolved !== _dbPath) {
|
|
162
|
-
_db?.close()
|
|
163
|
-
_db = new Database(resolved, { readonly: true })
|
|
164
|
-
_dbPath = resolved
|
|
165
|
-
tableCache.clear()
|
|
166
|
-
}
|
|
167
|
-
return _db
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
export function resolveDatabasePath() {
|
|
171
|
-
if (cachedDbPath) return cachedDbPath
|
|
172
|
-
if (process.env.OPENCODE_DB) {
|
|
173
|
-
if (process.env.OPENCODE_DB === ":memory:" || path.isAbsolute(process.env.OPENCODE_DB)) return cachedDbPath = process.env.OPENCODE_DB
|
|
174
|
-
return cachedDbPath = path.join(candidateDataDirs()[0] ?? defaultDataDir(), process.env.OPENCODE_DB)
|
|
175
|
-
}
|
|
176
|
-
if (process.env.OPENCODE_DISABLE_CHANNEL_DB === "1" || process.env.OPENCODE_DISABLE_CHANNEL_DB === "true") {
|
|
177
|
-
return cachedDbPath = requireExistingDatabase(["opencode.db"])
|
|
178
|
-
}
|
|
179
|
-
const stable = candidateDatabasePaths(["opencode.db"]).find(existsSync)
|
|
180
|
-
if (stable) return cachedDbPath = stable
|
|
181
|
-
if (process.env.OPENCODE_CHANNEL) {
|
|
182
|
-
const channel = process.env.OPENCODE_CHANNEL.replace(/[^a-zA-Z0-9._-]/g, "-")
|
|
183
|
-
const candidate = candidateDatabasePaths([`opencode-${channel}.db`]).find(existsSync)
|
|
184
|
-
if (candidate) return cachedDbPath = candidate
|
|
185
|
-
}
|
|
186
|
-
const fallback = candidateDatabasePaths(["opencode.db"])[0] ?? path.join(defaultDataDir(), "opencode.db")
|
|
187
|
-
try {
|
|
188
|
-
const discovered = candidateDataDirs()
|
|
189
|
-
.flatMap((dir) =>
|
|
190
|
-
readdirSync(dir, { withFileTypes: true })
|
|
191
|
-
.filter((entry) => entry.isFile() && /^opencode-.+\.db$/.test(entry.name))
|
|
192
|
-
.map((entry) => path.join(dir, entry.name)),
|
|
193
|
-
)
|
|
194
|
-
.at(0)
|
|
195
|
-
return cachedDbPath = discovered ?? fallback
|
|
196
|
-
} catch {
|
|
197
|
-
return cachedDbPath = fallback
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
export function searchSessionMessages(query: string, options?: { limit?: number; offset?: number; dbPath?: string; directory?: string; role?: SearchRole }) {
|
|
202
|
-
const term = query.trim()
|
|
203
|
-
if (!term) return []
|
|
204
|
-
if (options?.dbPath === ":memory:") return []
|
|
205
|
-
const dbPath = options?.dbPath ?? resolveDatabasePath()
|
|
206
|
-
const db = getDb(dbPath)
|
|
207
|
-
return searchRows(db, dbPath, term, options?.limit ?? 80, options?.directory, options?.offset, options?.role)
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
export function recentSessionMessages(options?: { limit?: number; offset?: number; dbPath?: string; directory?: string; role?: SearchRole }) {
|
|
211
|
-
const dbPath = options?.dbPath ?? resolveDatabasePath()
|
|
212
|
-
const db = getDb(dbPath)
|
|
213
|
-
const limit = options?.limit ?? 40
|
|
214
|
-
const indexed = dbPath === ":memory:" ? undefined : indexedRecentRows(db, dbPath, limit, options?.directory, options?.offset, options?.role)
|
|
215
|
-
if (indexed) return indexed.flatMap((row) => rowToSearchResult(row, "") ?? [])
|
|
216
|
-
debug.log("query:recent:source-fallback", { limit, offset: options?.offset ?? 0, directory: options?.directory, role: options?.role })
|
|
217
|
-
return visibleTextRows(db, limit, undefined, options?.directory, options?.offset, options?.role).flatMap(
|
|
218
|
-
(row) => rowToSearchResult(row, "") ?? [],
|
|
219
|
-
)
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export function loadConversationAround(result: SearchResult, options?: { before?: number; after?: number; dbPath?: string }): ConversationPreviewPage {
|
|
223
|
-
debug.time("preview:query:total")
|
|
224
|
-
const db = getDb(options?.dbPath)
|
|
225
|
-
const before = options?.before ?? 3
|
|
226
|
-
const after = options?.after ?? 6
|
|
227
|
-
|
|
228
|
-
debug.time("preview:query:hit")
|
|
229
|
-
const hit = db.query<{ time_created: number }, [string]>(
|
|
230
|
-
"SELECT time_created FROM part WHERE id = ?",
|
|
231
|
-
).get(result.id)
|
|
232
|
-
debug.timeEnd("preview:query:hit")
|
|
233
|
-
if (!hit) {
|
|
234
|
-
debug.log("preview:window", {
|
|
235
|
-
item: result.id,
|
|
236
|
-
session: result.sessionID,
|
|
237
|
-
before,
|
|
238
|
-
after,
|
|
239
|
-
hit: false,
|
|
240
|
-
})
|
|
241
|
-
debug.timeEnd("preview:query:total")
|
|
242
|
-
return { parts: [], hasMoreBefore: false, hasMoreAfter: false }
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const fetchBefore = Math.max(before * 4, before + 1, 30)
|
|
246
|
-
const fetchAfter = Math.max((after + 1) * 4, after + 2, 50)
|
|
247
|
-
|
|
248
|
-
debug.time("preview:query:before")
|
|
249
|
-
const beforeRows = db.query<ConversationRow, [string, number, number, string, number]>(`
|
|
250
|
-
SELECT p.id, p.message_id, p.session_id,
|
|
251
|
-
json_extract(m.data, '$.role') AS role,
|
|
252
|
-
json_extract(p.data, '$.type') AS type,
|
|
253
|
-
p.time_created, p.data
|
|
254
|
-
FROM part p
|
|
255
|
-
JOIN message m ON m.id = p.message_id
|
|
256
|
-
WHERE p.session_id = ?
|
|
257
|
-
AND (p.time_created < ? OR (p.time_created = ? AND p.id < ?))
|
|
258
|
-
ORDER BY p.time_created DESC, p.id DESC
|
|
259
|
-
LIMIT ?
|
|
260
|
-
`).all(result.sessionID, hit.time_created, hit.time_created, result.id, fetchBefore)
|
|
261
|
-
debug.timeEnd("preview:query:before")
|
|
262
|
-
|
|
263
|
-
debug.time("preview:query:after")
|
|
264
|
-
const afterRows = db.query<ConversationRow, [string, number, number, string, number]>(`
|
|
265
|
-
SELECT p.id, p.message_id, p.session_id,
|
|
266
|
-
json_extract(m.data, '$.role') AS role,
|
|
267
|
-
json_extract(p.data, '$.type') AS type,
|
|
268
|
-
p.time_created, p.data
|
|
269
|
-
FROM part p
|
|
270
|
-
JOIN message m ON m.id = p.message_id
|
|
271
|
-
WHERE p.session_id = ?
|
|
272
|
-
AND (p.time_created > ? OR (p.time_created = ? AND p.id >= ?))
|
|
273
|
-
ORDER BY p.time_created ASC, p.id ASC
|
|
274
|
-
LIMIT ?
|
|
275
|
-
`).all(result.sessionID, hit.time_created, hit.time_created, result.id, fetchAfter)
|
|
276
|
-
debug.timeEnd("preview:query:after")
|
|
277
|
-
|
|
278
|
-
debug.time("preview:query:parse")
|
|
279
|
-
const validBefore = beforeRows.filter(isPreviewRow)
|
|
280
|
-
const validAfter = afterRows.filter(isPreviewRow)
|
|
281
|
-
const parts = [
|
|
282
|
-
...validBefore.slice(0, before).reverse(),
|
|
283
|
-
...validAfter.slice(0, after + 1),
|
|
284
|
-
].flatMap((row) => parseConversationPart(row, row.id === result.id) ?? [])
|
|
285
|
-
const page = {
|
|
286
|
-
parts,
|
|
287
|
-
hasMoreBefore: validBefore.length > before,
|
|
288
|
-
hasMoreAfter: validAfter.length > after + 1,
|
|
289
|
-
}
|
|
290
|
-
debug.log("preview:window", {
|
|
291
|
-
item: result.id,
|
|
292
|
-
session: result.sessionID,
|
|
293
|
-
mode: "around",
|
|
294
|
-
before,
|
|
295
|
-
after,
|
|
296
|
-
fetchBefore,
|
|
297
|
-
fetchAfter,
|
|
298
|
-
beforeRows: beforeRows.length,
|
|
299
|
-
validBefore: validBefore.length,
|
|
300
|
-
invalidBefore: previewRowBreakdown(beforeRows, validBefore),
|
|
301
|
-
afterRows: afterRows.length,
|
|
302
|
-
validAfter: validAfter.length,
|
|
303
|
-
invalidAfter: previewRowBreakdown(afterRows, validAfter),
|
|
304
|
-
parts: parts.length,
|
|
305
|
-
hasMoreBefore: page.hasMoreBefore,
|
|
306
|
-
hasMoreAfter: page.hasMoreAfter,
|
|
307
|
-
first: parts[0]?.id,
|
|
308
|
-
last: parts.at(-1)?.id,
|
|
309
|
-
})
|
|
310
|
-
debug.timeEnd("preview:query:parse")
|
|
311
|
-
debug.timeEnd("preview:query:total")
|
|
312
|
-
return page
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
export function loadConversationBefore(result: SearchResult, cursor: ConversationPreviewCursor, options?: { limit?: number; dbPath?: string }) {
|
|
316
|
-
debug.time("preview:query:before-page")
|
|
317
|
-
const db = getDb(options?.dbPath)
|
|
318
|
-
const limit = options?.limit ?? 20
|
|
319
|
-
const fetchLimit = Math.max(limit * 4, limit + 1, 30)
|
|
320
|
-
const rows = db.query<ConversationRow, [string, number, number, string, number]>(`
|
|
321
|
-
SELECT p.id, p.message_id, p.session_id,
|
|
322
|
-
json_extract(m.data, '$.role') AS role,
|
|
323
|
-
json_extract(p.data, '$.type') AS type,
|
|
324
|
-
p.time_created, p.data
|
|
325
|
-
FROM part p
|
|
326
|
-
JOIN message m ON m.id = p.message_id
|
|
327
|
-
WHERE p.session_id = ?
|
|
328
|
-
AND (p.time_created < ? OR (p.time_created = ? AND p.id < ?))
|
|
329
|
-
ORDER BY p.time_created DESC, p.id DESC
|
|
330
|
-
LIMIT ?
|
|
331
|
-
`).all(result.sessionID, cursor.timeCreated, cursor.timeCreated, cursor.id, fetchLimit)
|
|
332
|
-
const valid = rows.filter(isPreviewRow)
|
|
333
|
-
const parts = valid.slice(0, limit).reverse().flatMap((row) => parseConversationPart(row, false) ?? [])
|
|
334
|
-
const parsedIds = new Set(parts.map((part) => part.id))
|
|
335
|
-
const page = { parts, hasMoreBefore: valid.length > limit }
|
|
336
|
-
debug.log("preview:window", {
|
|
337
|
-
item: result.id,
|
|
338
|
-
session: result.sessionID,
|
|
339
|
-
mode: "before",
|
|
340
|
-
cursor,
|
|
341
|
-
limit,
|
|
342
|
-
rows: rows.length,
|
|
343
|
-
valid: valid.length,
|
|
344
|
-
invalid: previewRowBreakdown(rows, valid),
|
|
345
|
-
parts: parts.length,
|
|
346
|
-
unparsedValid: valid.slice(0, limit).filter((row) => !parsedIds.has(row.id)).map((row) => ({ id: row.id, type: row.type, role: row.role })),
|
|
347
|
-
hasMoreBefore: page.hasMoreBefore,
|
|
348
|
-
first: parts[0]?.id,
|
|
349
|
-
last: parts.at(-1)?.id,
|
|
350
|
-
})
|
|
351
|
-
debug.timeEnd("preview:query:before-page")
|
|
352
|
-
return page
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
export function loadConversationAfter(result: SearchResult, cursor: ConversationPreviewCursor, options?: { limit?: number; dbPath?: string }) {
|
|
356
|
-
debug.time("preview:query:after-page")
|
|
357
|
-
const db = getDb(options?.dbPath)
|
|
358
|
-
const limit = options?.limit ?? 20
|
|
359
|
-
const fetchLimit = Math.max(limit * 4, limit + 1, 30)
|
|
360
|
-
const rows = db.query<ConversationRow, [string, number, number, string, number]>(`
|
|
361
|
-
SELECT p.id, p.message_id, p.session_id,
|
|
362
|
-
json_extract(m.data, '$.role') AS role,
|
|
363
|
-
json_extract(p.data, '$.type') AS type,
|
|
364
|
-
p.time_created, p.data
|
|
365
|
-
FROM part p
|
|
366
|
-
JOIN message m ON m.id = p.message_id
|
|
367
|
-
WHERE p.session_id = ?
|
|
368
|
-
AND (p.time_created > ? OR (p.time_created = ? AND p.id > ?))
|
|
369
|
-
ORDER BY p.time_created ASC, p.id ASC
|
|
370
|
-
LIMIT ?
|
|
371
|
-
`).all(result.sessionID, cursor.timeCreated, cursor.timeCreated, cursor.id, fetchLimit)
|
|
372
|
-
const valid = rows.filter(isPreviewRow)
|
|
373
|
-
const parts = valid.slice(0, limit).flatMap((row) => parseConversationPart(row, false) ?? [])
|
|
374
|
-
const parsedIds = new Set(parts.map((part) => part.id))
|
|
375
|
-
const page = { parts, hasMoreAfter: valid.length > limit }
|
|
376
|
-
debug.log("preview:window", {
|
|
377
|
-
item: result.id,
|
|
378
|
-
session: result.sessionID,
|
|
379
|
-
mode: "after",
|
|
380
|
-
cursor,
|
|
381
|
-
limit,
|
|
382
|
-
rows: rows.length,
|
|
383
|
-
valid: valid.length,
|
|
384
|
-
invalid: previewRowBreakdown(rows, valid),
|
|
385
|
-
parts: parts.length,
|
|
386
|
-
unparsedValid: valid.slice(0, limit).filter((row) => !parsedIds.has(row.id)).map((row) => ({ id: row.id, type: row.type, role: row.role })),
|
|
387
|
-
hasMoreAfter: page.hasMoreAfter,
|
|
388
|
-
first: parts[0]?.id,
|
|
389
|
-
last: parts.at(-1)?.id,
|
|
390
|
-
})
|
|
391
|
-
debug.timeEnd("preview:query:after-page")
|
|
392
|
-
return page
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
export function rowToSearchResult(row: Row, query: string): SearchResult | undefined {
|
|
396
|
-
const text = row.text.trim()
|
|
397
|
-
const match = findMatch(text, query)
|
|
398
|
-
if (!match) return
|
|
399
|
-
const preview = focusedPreview(text, match.start, match.end)
|
|
400
|
-
return {
|
|
401
|
-
id: row.id,
|
|
402
|
-
messageID: row.message_id,
|
|
403
|
-
sessionID: row.session_id,
|
|
404
|
-
sessionTitle: row.session_title || "Untitled session",
|
|
405
|
-
directory: row.directory,
|
|
406
|
-
role: row.role,
|
|
407
|
-
partType: row.part_type ?? "text",
|
|
408
|
-
tool: row.tool ?? undefined,
|
|
409
|
-
timeCreated: row.time_created,
|
|
410
|
-
snippet: makeSnippet(text, query),
|
|
411
|
-
matchStart: match.start,
|
|
412
|
-
matchEnd: match.end,
|
|
413
|
-
before: match.before,
|
|
414
|
-
match: match.match,
|
|
415
|
-
after: match.after,
|
|
416
|
-
excerpt: match.excerpt,
|
|
417
|
-
previewBefore: preview.before,
|
|
418
|
-
previewMatch: preview.match,
|
|
419
|
-
previewAfter: preview.after,
|
|
420
|
-
previewMode: preview.mode,
|
|
421
|
-
previewHighlight: preview.highlight,
|
|
422
|
-
text,
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
export function extractSearchText(data: string) {
|
|
427
|
-
try {
|
|
428
|
-
return extractFromValue(JSON.parse(data)).replace(/\s+/g, " ").trim()
|
|
429
|
-
} catch {
|
|
430
|
-
return data.replace(/\s+/g, " ").trim()
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
export function makeSnippet(text: string, query: string, radius = 72) {
|
|
435
|
-
const haystack = text.replace(/\s+/g, " ").trim()
|
|
436
|
-
const tokens = query.trim().split(/\s+/)
|
|
437
|
-
const lower = haystack.toLowerCase()
|
|
438
|
-
let searchPos = 0
|
|
439
|
-
let firstIndex = -1
|
|
440
|
-
let lastEnd = 0
|
|
441
|
-
for (const token of tokens) {
|
|
442
|
-
const index = lower.indexOf(token.toLowerCase(), searchPos)
|
|
443
|
-
if (index === -1) return truncate(haystack, radius * 2)
|
|
444
|
-
if (firstIndex === -1) firstIndex = index
|
|
445
|
-
searchPos = index + token.length
|
|
446
|
-
lastEnd = searchPos
|
|
447
|
-
}
|
|
448
|
-
const start = Math.max(0, firstIndex - radius)
|
|
449
|
-
const end = Math.min(haystack.length, lastEnd + radius)
|
|
450
|
-
return `${start > 0 ? "..." : ""}${haystack.slice(start, end)}${end < haystack.length ? "..." : ""}`
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
function findMatch(text: string, query: string, radius = 96) {
|
|
454
|
-
const needle = query.trim()
|
|
455
|
-
if (!needle) {
|
|
456
|
-
const collapsed = text.replace(/\s+/g, " ").trim()
|
|
457
|
-
const end = Math.min(collapsed.length, radius * 2)
|
|
458
|
-
return {
|
|
459
|
-
start: 0,
|
|
460
|
-
end: 0,
|
|
461
|
-
before: "",
|
|
462
|
-
match: "",
|
|
463
|
-
after: collapsed.slice(0, end),
|
|
464
|
-
excerpt: collapsed.slice(0, end),
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
const tokens = needle.split(/\s+/)
|
|
468
|
-
const lowerText = text.toLowerCase()
|
|
469
|
-
let searchPos = 0
|
|
470
|
-
let firstStart = -1
|
|
471
|
-
let lastEnd = -1
|
|
472
|
-
for (const token of tokens) {
|
|
473
|
-
const pos = lowerText.indexOf(token.toLowerCase(), searchPos)
|
|
474
|
-
if (pos === -1) return
|
|
475
|
-
if (firstStart === -1) firstStart = pos
|
|
476
|
-
searchPos = pos + token.length
|
|
477
|
-
lastEnd = searchPos
|
|
478
|
-
}
|
|
479
|
-
const start = firstStart
|
|
480
|
-
const end = lastEnd
|
|
481
|
-
const lineStart = text.lastIndexOf("\n", start - 1) + 1
|
|
482
|
-
const nextLine = text.indexOf("\n", end)
|
|
483
|
-
const lineEnd = nextLine === -1 ? text.length : nextLine
|
|
484
|
-
const line = text.slice(lineStart, lineEnd)
|
|
485
|
-
const matchLen = end - start
|
|
486
|
-
const lineMatchStart = Math.max(0, start - lineStart)
|
|
487
|
-
const lineMatchEnd = lineMatchStart + matchLen
|
|
488
|
-
const excerptStart = Math.max(0, lineMatchStart - radius)
|
|
489
|
-
const excerptEnd = Math.min(line.length, lineMatchEnd + radius)
|
|
490
|
-
const before = normalizeSnippetSegment(line.slice(excerptStart, lineMatchStart))
|
|
491
|
-
const after = normalizeSnippetSegment(line.slice(lineMatchEnd, excerptEnd))
|
|
492
|
-
return {
|
|
493
|
-
start,
|
|
494
|
-
end,
|
|
495
|
-
before: `${excerptStart > 0 ? "..." : ""}${before}`,
|
|
496
|
-
match: text.slice(start, end),
|
|
497
|
-
after: `${after}${excerptEnd < line.length ? "..." : ""}`,
|
|
498
|
-
excerpt: `${excerptStart > 0 ? "..." : ""}${normalizeSnippetSegment(line.slice(excerptStart, excerptEnd))}${excerptEnd < line.length ? "..." : ""}`,
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
function normalizeSnippetSegment(value: string) {
|
|
503
|
-
return value.replace(/\s+/g, " ")
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
function focusedPreview(text: string, matchStart: number, matchEnd: number) {
|
|
507
|
-
if (matchStart === matchEnd) {
|
|
508
|
-
const preview = text.slice(0, Math.min(text.length, 1400))
|
|
509
|
-
return { before: preview, match: "", after: "", mode: "markdown" as const, highlight: false }
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
const lineStart = text.lastIndexOf("\n", matchStart - 1) + 1
|
|
513
|
-
const nextLine = text.indexOf("\n", matchEnd)
|
|
514
|
-
const lineEnd = nextLine === -1 ? text.length : nextLine
|
|
515
|
-
const line = text.slice(lineStart, lineEnd)
|
|
516
|
-
|
|
517
|
-
if (line.length > 260) {
|
|
518
|
-
const beforeStart = Math.max(0, matchStart - 180)
|
|
519
|
-
const afterEnd = Math.min(text.length, matchEnd + 220)
|
|
520
|
-
return {
|
|
521
|
-
before: `${beforeStart > 0 ? "..." : ""}${text.slice(beforeStart, matchStart)}`,
|
|
522
|
-
match: text.slice(matchStart, matchEnd),
|
|
523
|
-
after: `${text.slice(matchEnd, afterEnd)}${afterEnd < text.length ? "..." : ""}`,
|
|
524
|
-
mode: "text" as const,
|
|
525
|
-
highlight: true,
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
const window = lineWindow(text, matchStart, matchEnd, 40, 80)
|
|
530
|
-
const windowText = text.slice(window.start, window.end)
|
|
531
|
-
const relativeStart = matchStart - window.start
|
|
532
|
-
const relativeEnd = matchEnd - window.start
|
|
533
|
-
|
|
534
|
-
const insideCodeFence = isInsideCodeFence(text, matchStart)
|
|
535
|
-
return {
|
|
536
|
-
before: `${window.start > 0 ? "...\n" : ""}${windowText.slice(0, relativeStart)}`,
|
|
537
|
-
match: windowText.slice(relativeStart, relativeEnd),
|
|
538
|
-
after: `${windowText.slice(relativeEnd)}${window.end < text.length ? "\n..." : ""}`,
|
|
539
|
-
mode: "markdown" as const,
|
|
540
|
-
highlight: !insideCodeFence,
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function lineWindow(text: string, matchStart: number, matchEnd: number, before: number, after: number) {
|
|
545
|
-
const starts = [0]
|
|
546
|
-
for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) {
|
|
547
|
-
starts.push(index + 1)
|
|
548
|
-
}
|
|
549
|
-
const matchLine = findLastStartIndex(starts, matchStart)
|
|
550
|
-
const startLine = Math.max(0, matchLine - before)
|
|
551
|
-
const endLine = Math.min(starts.length - 1, matchLine + after)
|
|
552
|
-
const start = starts[startLine] ?? 0
|
|
553
|
-
const end = starts[endLine + 1] ? starts[endLine + 1] - 1 : text.length
|
|
554
|
-
return { start, end: Math.max(end, matchEnd) }
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
function findLastStartIndex(starts: number[], offset: number) {
|
|
558
|
-
for (let index = starts.length - 1; index >= 0; index--) {
|
|
559
|
-
if (starts[index]! <= offset) return index
|
|
560
|
-
}
|
|
561
|
-
return 0
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
function isInsideCodeFence(text: string, offset: number) {
|
|
565
|
-
const before = text.slice(0, offset)
|
|
566
|
-
const fences = before.match(/^```/gm)
|
|
567
|
-
return Boolean(fences && fences.length % 2 === 1)
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
function parseConversationPart(row: ConversationRow, target: boolean): ConversationPreviewPart | undefined {
|
|
571
|
-
const data = parsePartData(row.data)
|
|
572
|
-
if (!data) return
|
|
573
|
-
if (row.type === "tool") {
|
|
574
|
-
return {
|
|
575
|
-
id: row.id,
|
|
576
|
-
messageID: row.message_id,
|
|
577
|
-
sessionID: row.session_id,
|
|
578
|
-
role: row.role,
|
|
579
|
-
type: row.type,
|
|
580
|
-
timeCreated: row.time_created,
|
|
581
|
-
text: target ? extractToolIndexText(data).trim() : "",
|
|
582
|
-
tool: typeof data.tool === "string" ? data.tool : "tool",
|
|
583
|
-
state: parseToolState(data.state),
|
|
584
|
-
target,
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
const text = typeof data.text === "string" ? data.text.trim() : ""
|
|
588
|
-
if (!text) return
|
|
589
|
-
return {
|
|
590
|
-
id: row.id,
|
|
591
|
-
messageID: row.message_id,
|
|
592
|
-
sessionID: row.session_id,
|
|
593
|
-
role: row.role,
|
|
594
|
-
type: row.type,
|
|
595
|
-
timeCreated: row.time_created,
|
|
596
|
-
text,
|
|
597
|
-
target,
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
function isPreviewRow(row: ConversationRow) {
|
|
602
|
-
return (row.role === "user" || row.role === "assistant") &&
|
|
603
|
-
(row.type === "text" || row.type === "reasoning" || row.type === "tool")
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
function previewRowBreakdown(rows: ConversationRow[], validRows: ConversationRow[]) {
|
|
607
|
-
const valid = new Set(validRows.map((row) => row.id))
|
|
608
|
-
const invalidRows = rows.filter((row) => !valid.has(row.id))
|
|
609
|
-
if (invalidRows.length === 0) return undefined
|
|
610
|
-
const byRole = countBy(invalidRows, (row) => String(row.role ?? "unknown"))
|
|
611
|
-
const byType = countBy(invalidRows, (row) => String(row.type ?? "unknown"))
|
|
612
|
-
return { count: invalidRows.length, byRole, byType }
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
function countBy<T>(items: T[], key: (item: T) => string) {
|
|
616
|
-
const counts: Record<string, number> = {}
|
|
617
|
-
for (const item of items) counts[key(item)] = (counts[key(item)] ?? 0) + 1
|
|
618
|
-
return counts
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
function parsePartData(data: string) {
|
|
622
|
-
try {
|
|
623
|
-
const value = JSON.parse(data) as unknown
|
|
624
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return
|
|
625
|
-
return value as Record<string, unknown>
|
|
626
|
-
} catch {
|
|
627
|
-
return
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
function parseToolState(value: unknown): ToolState | undefined {
|
|
632
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return
|
|
633
|
-
const state = value as Record<string, unknown>
|
|
634
|
-
if (!["pending", "running", "completed", "error"].includes(String(state.status))) return
|
|
635
|
-
return {
|
|
636
|
-
status: state.status as ToolState["status"],
|
|
637
|
-
input: state.input,
|
|
638
|
-
metadata: state.metadata,
|
|
639
|
-
output: typeof state.output === "string" ? state.output : undefined,
|
|
640
|
-
error: typeof state.error === "string" ? state.error : undefined,
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
function searchRows(db: Database, dbPath: string, query: string, limit: number, directory?: string, offset?: number, role?: SearchRole) {
|
|
645
|
-
if (!tableExists(db, "part") || !tableExists(db, "message")) return []
|
|
646
|
-
debug.time("query:sql")
|
|
647
|
-
const rows = indexedTextRows(db, dbPath, limit, query, directory, offset, role) ?? visibleTextRows(db, limit, query, directory, offset, role)
|
|
648
|
-
debug.timeEnd("query:sql")
|
|
649
|
-
debug.time("query:map")
|
|
650
|
-
const results = rows.flatMap((row) => rowToSearchResult(row, query) ?? [])
|
|
651
|
-
debug.timeEnd("query:map")
|
|
652
|
-
return results
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
function indexedTextRows(db: Database, dbPath: string, limit: number, query: string, directory?: string, offset?: number, role?: SearchRole) {
|
|
656
|
-
const match = ftsQuery(query)
|
|
657
|
-
if (!match) return []
|
|
658
|
-
try {
|
|
659
|
-
const index = ensureSearchIndex(db, dbPath)
|
|
660
|
-
if (!index) return []
|
|
661
|
-
const conditions = ["document_fts MATCH ?"]
|
|
662
|
-
const params: (string | number)[] = [match]
|
|
663
|
-
if (role) {
|
|
664
|
-
conditions.push("role = ?")
|
|
665
|
-
params.push(role)
|
|
666
|
-
}
|
|
667
|
-
if (directory) {
|
|
668
|
-
conditions.push("directory = ?")
|
|
669
|
-
params.push(directory)
|
|
670
|
-
}
|
|
671
|
-
params.push(limit)
|
|
672
|
-
if (offset) params.push(offset)
|
|
673
|
-
const offsetClause = offset ? "OFFSET ?" : ""
|
|
674
|
-
debug.time("query:fts:exec")
|
|
675
|
-
const rows = index.query<Row, (string | number)[]>(`
|
|
676
|
-
SELECT id, message_id, session_id, session_title, directory, role, part_type, tool,
|
|
677
|
-
CAST(time_created AS INTEGER) AS time_created, text
|
|
678
|
-
FROM document_fts
|
|
679
|
-
WHERE ${conditions.join(" AND ")}
|
|
680
|
-
ORDER BY bm25(document_fts), CAST(time_created AS INTEGER) DESC
|
|
681
|
-
LIMIT ? ${offsetClause}
|
|
682
|
-
`).all(...params as any[])
|
|
683
|
-
debug.timeEnd("query:fts:exec")
|
|
684
|
-
return rows
|
|
685
|
-
} catch (err) {
|
|
686
|
-
debug.log("fts:fallback", err instanceof Error ? err.message : String(err))
|
|
687
|
-
return
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
function indexedRecentRows(db: Database, dbPath: string, limit: number, directory?: string, offset?: number, role?: SearchRole) {
|
|
692
|
-
try {
|
|
693
|
-
const index = ensureSearchIndex(db, dbPath, { rebuild: false, useStale: true })
|
|
694
|
-
if (!index) {
|
|
695
|
-
debug.log("query:recent:index:missing", { dbPath })
|
|
696
|
-
return
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
const rowCount = index.query<{ count: number }, []>("SELECT COUNT(*) as count FROM document_index").get()?.count ?? 0
|
|
700
|
-
if (rowCount === 0) {
|
|
701
|
-
debug.log("query:recent:index:empty", { dbPath })
|
|
702
|
-
return
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
const state = sourceState(db, dbPath)
|
|
706
|
-
const currentDataVersion = getMeta(index, "source_data_version")
|
|
707
|
-
const currentMtimeMs = getMeta(index, "source_mtime_ms")
|
|
708
|
-
const currentPath = getMeta(index, "source_path")
|
|
709
|
-
const currentIndexVersion = getMeta(index, "index_version")
|
|
710
|
-
if (currentPath !== dbPath || currentDataVersion !== String(state.dataVersion) || currentMtimeMs !== String(state.mtimeMs) || currentIndexVersion !== SEARCH_INDEX_VERSION) {
|
|
711
|
-
debug.log("query:recent:index:stale", {
|
|
712
|
-
dbPath,
|
|
713
|
-
expectedDataVersion: state.dataVersion,
|
|
714
|
-
actualDataVersion: currentDataVersion,
|
|
715
|
-
})
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
const conditions: string[] = ["part_type = 'text'"]
|
|
719
|
-
const params: (string | number)[] = []
|
|
720
|
-
if (role) {
|
|
721
|
-
conditions.push("role = ?")
|
|
722
|
-
params.push(role)
|
|
723
|
-
}
|
|
724
|
-
if (directory) {
|
|
725
|
-
conditions.push("directory = ?")
|
|
726
|
-
params.push(directory)
|
|
727
|
-
}
|
|
728
|
-
params.push(limit)
|
|
729
|
-
if (offset) params.push(offset)
|
|
730
|
-
|
|
731
|
-
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""
|
|
732
|
-
const offsetClause = offset ? "OFFSET ?" : ""
|
|
733
|
-
debug.time("query:recent:index:exec")
|
|
734
|
-
const rows = index.query<Row, (string | number)[]>(`
|
|
735
|
-
SELECT id, message_id, session_id, session_title, directory, role, part_type, tool,
|
|
736
|
-
CAST(time_created AS INTEGER) AS time_created, text
|
|
737
|
-
FROM document_index
|
|
738
|
-
${where}
|
|
739
|
-
ORDER BY CAST(time_created AS INTEGER) DESC
|
|
740
|
-
LIMIT ? ${offsetClause}
|
|
741
|
-
`).all(...params as any[])
|
|
742
|
-
debug.timeEnd("query:recent:index:exec")
|
|
743
|
-
return rows
|
|
744
|
-
} catch (err) {
|
|
745
|
-
debug.log("recent:index:fallback", err instanceof Error ? err.message : String(err))
|
|
746
|
-
return
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
function visibleTextRows(db: Database, limit: number, query?: string, directory?: string, offset?: number, role?: SearchRole) {
|
|
751
|
-
const offsetClause = offset ? "OFFSET ?" : ""
|
|
752
|
-
const conditions: string[] = [
|
|
753
|
-
"json_extract(p.data, '$.type') = 'text'",
|
|
754
|
-
role ? "json_extract(m.data, '$.role') = ?" : "json_extract(m.data, '$.role') IN ('user', 'assistant')",
|
|
755
|
-
]
|
|
756
|
-
const params: (string | number)[] = []
|
|
757
|
-
|
|
758
|
-
if (role) params.push(role)
|
|
759
|
-
|
|
760
|
-
if (directory) {
|
|
761
|
-
conditions.push("s.directory = ?")
|
|
762
|
-
params.push(directory)
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
const tokens = query ? query.trim().split(/\s+/).filter(Boolean) : []
|
|
766
|
-
for (const token of tokens) {
|
|
767
|
-
conditions.push("json_extract(p.data, '$.text') LIKE ?")
|
|
768
|
-
params.push(`%${token}%`)
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
params.push(limit)
|
|
772
|
-
if (offset) params.push(offset)
|
|
773
|
-
|
|
774
|
-
const sql = `
|
|
775
|
-
SELECT p.id, p.message_id, p.session_id, s.title AS session_title, s.directory,
|
|
776
|
-
json_extract(m.data, '$.role') AS role,
|
|
777
|
-
p.time_created,
|
|
778
|
-
json_extract(p.data, '$.text') AS text
|
|
779
|
-
FROM part p
|
|
780
|
-
JOIN message m ON m.id = p.message_id
|
|
781
|
-
JOIN session s ON s.id = p.session_id
|
|
782
|
-
WHERE ${conditions.join(" AND ")}
|
|
783
|
-
ORDER BY p.time_created DESC
|
|
784
|
-
LIMIT ? ${offsetClause}
|
|
785
|
-
`
|
|
786
|
-
debug.time("query:sql:exec")
|
|
787
|
-
const rows = db.query<Row, (string | number)[]>(sql).all(...params as any[])
|
|
788
|
-
debug.timeEnd("query:sql:exec")
|
|
789
|
-
return rows
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
function ensureSearchIndex(source: Database, sourcePath: string, options?: { rebuild?: boolean; useStale?: boolean }) {
|
|
793
|
-
const indexPath = searchIndexPath(sourcePath)
|
|
794
|
-
if (!_indexDb || _indexDbPath !== indexPath) {
|
|
795
|
-
_indexDb?.close()
|
|
796
|
-
configureCustomSQLite()
|
|
797
|
-
_indexDb = new Database(indexPath)
|
|
798
|
-
_indexDbPath = indexPath
|
|
799
|
-
migrateSearchIndex(_indexDb)
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
const state = sourceState(source, sourcePath)
|
|
803
|
-
const currentDataVersion = getMeta(_indexDb, "source_data_version")
|
|
804
|
-
const currentMtimeMs = getMeta(_indexDb, "source_mtime_ms")
|
|
805
|
-
const currentPath = getMeta(_indexDb, "source_path")
|
|
806
|
-
const currentIndexVersion = getMeta(_indexDb, "index_version")
|
|
807
|
-
if (currentPath !== sourcePath || currentDataVersion !== String(state.dataVersion) || currentMtimeMs !== String(state.mtimeMs) || currentIndexVersion !== SEARCH_INDEX_VERSION) {
|
|
808
|
-
if (options?.rebuild === false) {
|
|
809
|
-
if (options?.useStale) return _indexDb
|
|
810
|
-
return
|
|
811
|
-
}
|
|
812
|
-
rebuildSearchIndex(source, _indexDb, sourcePath, state)
|
|
813
|
-
}
|
|
814
|
-
return _indexDb
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
function scheduleBackgroundIndexRebuild(dbPath: string) {
|
|
818
|
-
if (backgroundIndexRebuilds.has(dbPath)) return
|
|
819
|
-
backgroundIndexRebuilds.add(dbPath)
|
|
820
|
-
debug.log("fts:rebuild:background-scheduled", { dbPath })
|
|
821
|
-
const timer = setTimeout(() => {
|
|
822
|
-
debug.time("fts:rebuild:background")
|
|
823
|
-
try {
|
|
824
|
-
const db = getDb(dbPath)
|
|
825
|
-
ensureSearchIndex(db, dbPath)
|
|
826
|
-
} catch (err) {
|
|
827
|
-
debug.log("fts:rebuild:background:error", err instanceof Error ? err.message : String(err))
|
|
828
|
-
} finally {
|
|
829
|
-
backgroundIndexRebuilds.delete(dbPath)
|
|
830
|
-
debug.timeEnd("fts:rebuild:background")
|
|
831
|
-
}
|
|
832
|
-
}, 250)
|
|
833
|
-
;(timer as { unref?: () => void }).unref?.()
|
|
834
|
-
}
|
|
835
|
-
|
|
836
|
-
const SEARCH_INDEX_VERSION = "7"
|
|
837
|
-
const DOCUMENT_EXTRACTOR_VERSION = "1"
|
|
838
|
-
|
|
839
|
-
function searchIndexPath(sourcePath: string) {
|
|
840
|
-
const parsed = path.parse(sourcePath)
|
|
841
|
-
return path.join(parsed.dir, `${parsed.name}-telescope-search.db`)
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
function migrateSearchIndex(db: Database) {
|
|
845
|
-
db.exec(`
|
|
846
|
-
CREATE TABLE IF NOT EXISTS index_meta(
|
|
847
|
-
key TEXT PRIMARY KEY,
|
|
848
|
-
value TEXT NOT NULL
|
|
849
|
-
);
|
|
850
|
-
CREATE TABLE IF NOT EXISTS document(
|
|
851
|
-
rowid INTEGER PRIMARY KEY,
|
|
852
|
-
doc_id TEXT UNIQUE NOT NULL,
|
|
853
|
-
part_id TEXT NOT NULL,
|
|
854
|
-
message_id TEXT NOT NULL,
|
|
855
|
-
session_id TEXT NOT NULL,
|
|
856
|
-
session_title TEXT NOT NULL,
|
|
857
|
-
directory TEXT NOT NULL,
|
|
858
|
-
role TEXT NOT NULL,
|
|
859
|
-
part_type TEXT NOT NULL,
|
|
860
|
-
tool TEXT,
|
|
861
|
-
time_created INTEGER NOT NULL,
|
|
862
|
-
chunk_index INTEGER NOT NULL,
|
|
863
|
-
text TEXT NOT NULL,
|
|
864
|
-
source_hash TEXT NOT NULL,
|
|
865
|
-
extractor_version TEXT NOT NULL,
|
|
866
|
-
indexed_at INTEGER NOT NULL
|
|
867
|
-
);
|
|
868
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
|
|
869
|
-
id UNINDEXED,
|
|
870
|
-
message_id UNINDEXED,
|
|
871
|
-
session_id UNINDEXED,
|
|
872
|
-
session_title,
|
|
873
|
-
directory UNINDEXED,
|
|
874
|
-
role UNINDEXED,
|
|
875
|
-
part_type UNINDEXED,
|
|
876
|
-
tool UNINDEXED,
|
|
877
|
-
time_created UNINDEXED,
|
|
878
|
-
text,
|
|
879
|
-
tokenize='unicode61'
|
|
880
|
-
);
|
|
881
|
-
CREATE TABLE IF NOT EXISTS document_index(
|
|
882
|
-
id TEXT PRIMARY KEY,
|
|
883
|
-
message_id TEXT NOT NULL,
|
|
884
|
-
session_id TEXT NOT NULL,
|
|
885
|
-
session_title TEXT NOT NULL,
|
|
886
|
-
directory TEXT NOT NULL,
|
|
887
|
-
role TEXT NOT NULL,
|
|
888
|
-
part_type TEXT NOT NULL,
|
|
889
|
-
tool TEXT,
|
|
890
|
-
time_created INTEGER NOT NULL,
|
|
891
|
-
text TEXT NOT NULL
|
|
892
|
-
);
|
|
893
|
-
CREATE INDEX IF NOT EXISTS document_index_recent_idx
|
|
894
|
-
ON document_index(directory, role, time_created DESC);
|
|
895
|
-
CREATE INDEX IF NOT EXISTS document_index_recent_text_idx
|
|
896
|
-
ON document_index(directory, role, part_type, time_created DESC);
|
|
897
|
-
CREATE INDEX IF NOT EXISTS document_index_time_idx
|
|
898
|
-
ON document_index(time_created DESC);
|
|
899
|
-
`)
|
|
900
|
-
|
|
901
|
-
const columns = db.query<{ name: string }, []>("PRAGMA table_info(document_fts)").all().map((column) => column.name)
|
|
902
|
-
if (!columns.includes("part_type") || !columns.includes("tool")) {
|
|
903
|
-
db.exec("DROP TABLE document_fts")
|
|
904
|
-
db.exec(`
|
|
905
|
-
CREATE VIRTUAL TABLE document_fts USING fts5(
|
|
906
|
-
id UNINDEXED,
|
|
907
|
-
message_id UNINDEXED,
|
|
908
|
-
session_id UNINDEXED,
|
|
909
|
-
session_title,
|
|
910
|
-
directory UNINDEXED,
|
|
911
|
-
role UNINDEXED,
|
|
912
|
-
part_type UNINDEXED,
|
|
913
|
-
tool UNINDEXED,
|
|
914
|
-
time_created UNINDEXED,
|
|
915
|
-
text,
|
|
916
|
-
tokenize='unicode61'
|
|
917
|
-
);
|
|
918
|
-
`)
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
const indexColumns = db.query<{ name: string }, []>("PRAGMA table_info(document_index)").all().map((column) => column.name)
|
|
922
|
-
if (!["part_type", "tool", "time_created", "text"].every((name) => indexColumns.includes(name))) {
|
|
923
|
-
db.exec("DROP TABLE IF EXISTS document_index")
|
|
924
|
-
db.exec(`
|
|
925
|
-
CREATE TABLE document_index(
|
|
926
|
-
id TEXT PRIMARY KEY,
|
|
927
|
-
message_id TEXT NOT NULL,
|
|
928
|
-
session_id TEXT NOT NULL,
|
|
929
|
-
session_title TEXT NOT NULL,
|
|
930
|
-
directory TEXT NOT NULL,
|
|
931
|
-
role TEXT NOT NULL,
|
|
932
|
-
part_type TEXT NOT NULL,
|
|
933
|
-
tool TEXT,
|
|
934
|
-
time_created INTEGER NOT NULL,
|
|
935
|
-
text TEXT NOT NULL
|
|
936
|
-
);
|
|
937
|
-
`)
|
|
938
|
-
}
|
|
939
|
-
db.exec(`
|
|
940
|
-
CREATE INDEX IF NOT EXISTS document_index_recent_idx
|
|
941
|
-
ON document_index(directory, role, time_created DESC);
|
|
942
|
-
CREATE INDEX IF NOT EXISTS document_index_recent_text_idx
|
|
943
|
-
ON document_index(directory, role, part_type, time_created DESC);
|
|
944
|
-
CREATE INDEX IF NOT EXISTS document_index_time_idx
|
|
945
|
-
ON document_index(time_created DESC);
|
|
946
|
-
`)
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
function sourceState(db: Database, sourcePath: string) {
|
|
950
|
-
const stat = statSync(sourcePath)
|
|
951
|
-
const dataVersion = db.query<{ data_version: number }, []>("PRAGMA data_version").get()?.data_version ?? 0
|
|
952
|
-
return { dataVersion, mtimeMs: stat.mtimeMs }
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
function rebuildSearchIndex(source: Database, index: Database, sourcePath: string, state: { dataVersion: number; mtimeMs: number }) {
|
|
956
|
-
debug.time("fts:rebuild")
|
|
957
|
-
const rows = source.query<IndexSourceRow, []>(`
|
|
958
|
-
SELECT p.id, p.message_id, p.session_id, s.title AS session_title, s.directory,
|
|
959
|
-
json_extract(m.data, '$.role') AS role,
|
|
960
|
-
json_extract(p.data, '$.type') AS part_type,
|
|
961
|
-
json_extract(p.data, '$.tool') AS tool,
|
|
962
|
-
p.time_created,
|
|
963
|
-
p.data
|
|
964
|
-
FROM part p
|
|
965
|
-
JOIN message m ON m.id = p.message_id
|
|
966
|
-
JOIN session s ON s.id = p.session_id
|
|
967
|
-
WHERE (
|
|
968
|
-
json_extract(p.data, '$.type') = 'text'
|
|
969
|
-
OR (
|
|
970
|
-
json_extract(p.data, '$.type') = 'tool'
|
|
971
|
-
AND json_extract(p.data, '$.tool') IN ('apply_patch', 'edit', 'write')
|
|
972
|
-
)
|
|
973
|
-
)
|
|
974
|
-
AND json_extract(m.data, '$.role') IN ('user', 'assistant')
|
|
975
|
-
ORDER BY p.time_created DESC
|
|
976
|
-
`).all()
|
|
977
|
-
const now = Date.now()
|
|
978
|
-
const config = parseSemanticConfig()
|
|
979
|
-
const insertDoc = index.query<unknown, [string, string, string, string, string, string, string, string, string | null, number, number, string, string, string, number]>(`
|
|
980
|
-
INSERT INTO document(doc_id, part_id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, chunk_index, text, source_hash, extractor_version, indexed_at)
|
|
981
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
982
|
-
`)
|
|
983
|
-
const insertFts = index.query<Row, [string, string, string, string, string, string, SearchResult["partType"], string | null, number, string]>(`
|
|
984
|
-
INSERT INTO document_fts(id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, text)
|
|
985
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
986
|
-
`)
|
|
987
|
-
const insertIndex = index.query<Row, [string, string, string, string, string, string, SearchResult["partType"], string | null, number, string]>(`
|
|
988
|
-
INSERT INTO document_index(id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, text)
|
|
989
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
990
|
-
`)
|
|
991
|
-
index.exec("BEGIN IMMEDIATE")
|
|
992
|
-
try {
|
|
993
|
-
index.exec("DELETE FROM document")
|
|
994
|
-
index.exec("DELETE FROM document_fts")
|
|
995
|
-
index.exec("DELETE FROM document_index")
|
|
996
|
-
for (const row of rows.flatMap(indexSourceRowToRows)) {
|
|
997
|
-
const docID = `telescope:${row.session_id}:${row.message_id}:${row.id}:0`
|
|
998
|
-
const sourceHash = hashPartData({ session_id: row.session_id, message_id: row.message_id, part_id: row.id })
|
|
999
|
-
insertDoc.run(
|
|
1000
|
-
docID,
|
|
1001
|
-
row.id,
|
|
1002
|
-
row.message_id,
|
|
1003
|
-
row.session_id,
|
|
1004
|
-
row.session_title ?? "Untitled session",
|
|
1005
|
-
row.directory,
|
|
1006
|
-
row.role,
|
|
1007
|
-
row.part_type ?? "text",
|
|
1008
|
-
row.tool ?? null,
|
|
1009
|
-
row.time_created,
|
|
1010
|
-
0,
|
|
1011
|
-
row.text,
|
|
1012
|
-
sourceHash,
|
|
1013
|
-
DOCUMENT_EXTRACTOR_VERSION,
|
|
1014
|
-
now,
|
|
1015
|
-
)
|
|
1016
|
-
insertFts.run(
|
|
1017
|
-
row.id,
|
|
1018
|
-
row.message_id,
|
|
1019
|
-
row.session_id,
|
|
1020
|
-
row.session_title ?? "Untitled session",
|
|
1021
|
-
row.directory,
|
|
1022
|
-
row.role,
|
|
1023
|
-
row.part_type ?? "text",
|
|
1024
|
-
row.tool ?? null,
|
|
1025
|
-
row.time_created,
|
|
1026
|
-
row.text,
|
|
1027
|
-
)
|
|
1028
|
-
insertIndex.run(
|
|
1029
|
-
row.id,
|
|
1030
|
-
row.message_id,
|
|
1031
|
-
row.session_id,
|
|
1032
|
-
row.session_title ?? "Untitled session",
|
|
1033
|
-
row.directory,
|
|
1034
|
-
row.role,
|
|
1035
|
-
row.part_type ?? "text",
|
|
1036
|
-
row.tool ?? null,
|
|
1037
|
-
row.time_created,
|
|
1038
|
-
row.text,
|
|
1039
|
-
)
|
|
1040
|
-
}
|
|
1041
|
-
setMeta(index, "source_path", sourcePath)
|
|
1042
|
-
setMeta(index, "source_data_version", String(state.dataVersion))
|
|
1043
|
-
setMeta(index, "source_mtime_ms", String(state.mtimeMs))
|
|
1044
|
-
setMeta(index, "index_version", SEARCH_INDEX_VERSION)
|
|
1045
|
-
setMeta(index, "schema_version", "2")
|
|
1046
|
-
setMeta(index, "extractor_version", DOCUMENT_EXTRACTOR_VERSION)
|
|
1047
|
-
setMeta(index, "ranking_version", "1")
|
|
1048
|
-
setMeta(index, "embedding_base_url", config.embedBaseUrl)
|
|
1049
|
-
setMeta(index, "document_prefix", config.documentPrefix)
|
|
1050
|
-
setMeta(index, "query_prefix", config.queryPrefix)
|
|
1051
|
-
if (config.embedModel) setMeta(index, "embedding_model", config.embedModel)
|
|
1052
|
-
index.exec("COMMIT")
|
|
1053
|
-
} catch (err) {
|
|
1054
|
-
index.exec("ROLLBACK")
|
|
1055
|
-
throw err
|
|
1056
|
-
} finally {
|
|
1057
|
-
debug.timeEnd("fts:rebuild")
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
|
-
if (config.disableVector) {
|
|
1061
|
-
setMeta(index, "vector_state", "disabled")
|
|
1062
|
-
} else {
|
|
1063
|
-
setupVectorTable(index, config, searchIndexPath(sourcePath))
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
function hashPartData(value: { session_id: string; message_id: string; part_id: string }) {
|
|
1068
|
-
return createHash("sha256").update(`${value.session_id}:${value.message_id}:${value.part_id}`).digest("hex")
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
let customSQLiteConfigured = false
|
|
1072
|
-
|
|
1073
|
-
function configureCustomSQLite() {
|
|
1074
|
-
if (customSQLiteConfigured) return
|
|
1075
|
-
const config = parseSemanticConfig()
|
|
1076
|
-
if (config.disableVector) return
|
|
1077
|
-
customSQLiteConfigured = true
|
|
1078
|
-
|
|
1079
|
-
const candidates = [
|
|
1080
|
-
config.sqliteLibPath,
|
|
1081
|
-
process.platform === "darwin" ? "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib" : undefined,
|
|
1082
|
-
process.platform === "darwin" ? "/usr/local/opt/sqlite/lib/libsqlite3.dylib" : undefined,
|
|
1083
|
-
].filter((item): item is string => Boolean(item))
|
|
1084
|
-
|
|
1085
|
-
for (const candidate of candidates) {
|
|
1086
|
-
if (existsSync(candidate)) {
|
|
1087
|
-
try {
|
|
1088
|
-
Database.setCustomSQLite(candidate)
|
|
1089
|
-
debug.log("custom-sqlite:set", { path: candidate })
|
|
1090
|
-
return
|
|
1091
|
-
} catch (err) {
|
|
1092
|
-
debug.log("custom-sqlite:error", { path: candidate, error: err instanceof Error ? err.message : String(err) })
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1097
|
-
|
|
1098
|
-
function importPackage(specifier: string) {
|
|
1099
|
-
return new Function("specifier", "return import(specifier)")(specifier) as Promise<{ default?: { load?: (db: Database) => void } }>
|
|
1100
|
-
}
|
|
1101
|
-
|
|
1102
|
-
async function loadVecExtension(db: Database): Promise<boolean> {
|
|
1103
|
-
try {
|
|
1104
|
-
const sqliteVec = await importPackage("sqlite-vec").catch(() => undefined)
|
|
1105
|
-
if (sqliteVec?.default?.load) {
|
|
1106
|
-
sqliteVec.default.load(db)
|
|
1107
|
-
debug.log("vector:extension:loaded", { source: "npm" })
|
|
1108
|
-
return true
|
|
1109
|
-
}
|
|
1110
|
-
} catch {
|
|
1111
|
-
debug.log("vector:extension:npm-failed")
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
const config = parseSemanticConfig()
|
|
1115
|
-
const explicitPath = config.sqliteVecExtension || process.env.OPENCODE_TELESCOPE_SQLITE_VEC_EXT
|
|
1116
|
-
if (explicitPath && existsSync(explicitPath)) {
|
|
1117
|
-
try {
|
|
1118
|
-
db.loadExtension(explicitPath)
|
|
1119
|
-
debug.log("vector:extension:loaded", { source: "path", path: explicitPath })
|
|
1120
|
-
return true
|
|
1121
|
-
} catch (err) {
|
|
1122
|
-
debug.log("vector:extension:path-failed", { path: explicitPath, error: err instanceof Error ? err.message : String(err) })
|
|
1123
|
-
}
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
return false
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
function setupVectorTable(index: Database, config: SemanticConfig, indexPath: string) {
|
|
1130
|
-
setMeta(index, "vector_state", "stale")
|
|
1131
|
-
rebuildVectorIndex(indexPath, config).catch((err: Error) => {
|
|
1132
|
-
debug.log("vector:rebuild:error", err instanceof Error ? err.message : String(err))
|
|
1133
|
-
})
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
async function rebuildVectorIndex(indexPath: string, config: SemanticConfig) {
|
|
1137
|
-
configureCustomSQLite()
|
|
1138
|
-
const db = new Database(indexPath)
|
|
1139
|
-
try {
|
|
1140
|
-
const loaded = await loadVecExtension(db)
|
|
1141
|
-
if (!loaded) {
|
|
1142
|
-
setMeta(db, "vector_state", "unavailable")
|
|
1143
|
-
return
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
const client = new LlamaEmbeddingClient({
|
|
1147
|
-
baseUrl: config.embedBaseUrl,
|
|
1148
|
-
model: config.embedModel,
|
|
1149
|
-
documentPrefix: config.documentPrefix,
|
|
1150
|
-
queryPrefix: config.queryPrefix,
|
|
1151
|
-
})
|
|
1152
|
-
const healthy = await client.health()
|
|
1153
|
-
if (!healthy) {
|
|
1154
|
-
setMeta(db, "vector_state", "unavailable")
|
|
1155
|
-
return
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
const docs = db.query<{ rowid: number; text: string }, []>("SELECT rowid, text FROM document ORDER BY rowid ASC").all()
|
|
1159
|
-
if (!docs.length) {
|
|
1160
|
-
setMeta(db, "vector_state", "enabled")
|
|
1161
|
-
return
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
debug.log("vector:embed:start", { count: docs.length })
|
|
1165
|
-
const embeddings = await client.embedDocuments(docs.map((d) => d.text))
|
|
1166
|
-
const dims = embeddings[0]?.length
|
|
1167
|
-
if (!dims) {
|
|
1168
|
-
setMeta(db, "vector_state", "unavailable")
|
|
1169
|
-
return
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
db.exec("DROP TABLE IF EXISTS document_vec")
|
|
1173
|
-
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS document_vec USING vec0(embedding float[${dims}])`)
|
|
1174
|
-
debug.log("vector:table:recreated", { dimensions: dims })
|
|
1175
|
-
|
|
1176
|
-
const insert = db.prepare<unknown, [number, Float32Array]>("INSERT INTO document_vec(rowid, embedding) VALUES (?, vec_f32(?))")
|
|
1177
|
-
const transaction = db.transaction(() => {
|
|
1178
|
-
for (const [i, doc] of docs.entries()) {
|
|
1179
|
-
insert.run(doc.rowid, embeddings[i])
|
|
1180
|
-
}
|
|
1181
|
-
})
|
|
1182
|
-
transaction()
|
|
1183
|
-
|
|
1184
|
-
setMeta(db, "vector_state", "enabled")
|
|
1185
|
-
setMeta(db, "embedding_dimensions", String(dims))
|
|
1186
|
-
if (config.embedModel) setMeta(db, "embedding_model", config.embedModel)
|
|
1187
|
-
debug.log("vector:rebuild:done", { vectors: docs.length, dimensions: dims })
|
|
1188
|
-
} catch (err) {
|
|
1189
|
-
setMeta(db, "vector_state", "unavailable")
|
|
1190
|
-
debug.log("vector:rebuild:error", err instanceof Error ? err.message : String(err))
|
|
1191
|
-
} finally {
|
|
1192
|
-
db.close()
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
export type ScoredRow = Row & {
|
|
1197
|
-
score: number
|
|
1198
|
-
keywordScore: number
|
|
1199
|
-
vectorScore: number
|
|
1200
|
-
}
|
|
1201
|
-
|
|
1202
|
-
function searchVector(index: Database, embedding: Float32Array, limit: number): Row[] {
|
|
1203
|
-
const count = index.query<{ count: number }, []>("SELECT COUNT(*) as count FROM document_vec").get()?.count ?? 0
|
|
1204
|
-
if (!count) return []
|
|
1205
|
-
const k = Math.min(count, Math.max(limit * 4, 200))
|
|
1206
|
-
return index.query<Row, [Float32Array, number]>(`
|
|
1207
|
-
SELECT d.part_id AS id, d.message_id, d.session_id, d.session_title, d.directory, d.role,
|
|
1208
|
-
d.part_type, d.tool, CAST(d.time_created AS INTEGER) AS time_created, d.text
|
|
1209
|
-
FROM document_vec v
|
|
1210
|
-
JOIN document d ON d.rowid = v.rowid
|
|
1211
|
-
WHERE v.embedding MATCH vec_f32(?) AND k = ?
|
|
1212
|
-
ORDER BY v.distance
|
|
1213
|
-
`).all(embedding, k)
|
|
1214
|
-
}
|
|
1215
|
-
|
|
1216
|
-
export function hybridBlend(keyword: Row[], vector: Row[], alpha: number): ScoredRow[] {
|
|
1217
|
-
const merged = new Map<string, ScoredRow>()
|
|
1218
|
-
const defaultScore = 1 / Math.max(keyword.length, 1)
|
|
1219
|
-
|
|
1220
|
-
for (const [i, row] of keyword.entries()) {
|
|
1221
|
-
merged.set(row.id, {
|
|
1222
|
-
...row,
|
|
1223
|
-
score: 0,
|
|
1224
|
-
keywordScore: keyword.length > 1 ? 1 - i / (keyword.length - 1) : 1,
|
|
1225
|
-
vectorScore: 0,
|
|
1226
|
-
})
|
|
1227
|
-
}
|
|
1228
|
-
|
|
1229
|
-
for (const [i, row] of vector.entries()) {
|
|
1230
|
-
const existing = merged.get(row.id)
|
|
1231
|
-
const vectorScore = vector.length > 1 ? 1 - i / (vector.length - 1) : 1
|
|
1232
|
-
if (existing) {
|
|
1233
|
-
existing.vectorScore = vectorScore
|
|
1234
|
-
} else {
|
|
1235
|
-
merged.set(row.id, {
|
|
1236
|
-
...row,
|
|
1237
|
-
score: 0,
|
|
1238
|
-
keywordScore: defaultScore,
|
|
1239
|
-
vectorScore,
|
|
1240
|
-
})
|
|
1241
|
-
}
|
|
1242
|
-
}
|
|
1243
|
-
|
|
1244
|
-
const keywordValues = [...merged.values()].map((r) => r.keywordScore)
|
|
1245
|
-
const vectorValues = [...merged.values()].map((r) => r.vectorScore)
|
|
1246
|
-
const kwMin = Math.min(...keywordValues)
|
|
1247
|
-
const kwMax = Math.max(...keywordValues)
|
|
1248
|
-
const vecMin = Math.min(...vectorValues)
|
|
1249
|
-
const vecMax = Math.max(...vectorValues)
|
|
1250
|
-
|
|
1251
|
-
const normalized = [...merged.values()].map((r) => {
|
|
1252
|
-
const kn = kwMax === kwMin ? 0.5 : (r.keywordScore - kwMin) / (kwMax - kwMin)
|
|
1253
|
-
const vn = vecMax === vecMin ? 0.5 : (r.vectorScore - vecMin) / (vecMax - vecMin)
|
|
1254
|
-
return {
|
|
1255
|
-
...r,
|
|
1256
|
-
keywordScore: kn,
|
|
1257
|
-
vectorScore: vn,
|
|
1258
|
-
score: (1 - alpha) * kn + alpha * vn,
|
|
1259
|
-
}
|
|
1260
|
-
})
|
|
1261
|
-
|
|
1262
|
-
return normalized.sort((a, b) => b.score - a.score)
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
export type HybridSearchOptions = {
|
|
1266
|
-
limit?: number
|
|
1267
|
-
offset?: number
|
|
1268
|
-
directory?: string
|
|
1269
|
-
role?: SearchRole
|
|
1270
|
-
dbPath?: string
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
export async function performSearch(query: string, options?: HybridSearchOptions): Promise<SearchResult[]> {
|
|
1274
|
-
if (!query.trim()) return searchSessionMessages(query, options)
|
|
1275
|
-
const config = parseSemanticConfig()
|
|
1276
|
-
if (!config.disableVector) {
|
|
1277
|
-
try {
|
|
1278
|
-
return await semanticSearchSessionMessages(query, options)
|
|
1279
|
-
} catch {
|
|
1280
|
-
debug.log("query:hybrid:fallback", { message: "hybrid search failed, falling back to keyword" })
|
|
1281
|
-
}
|
|
1282
|
-
}
|
|
1283
|
-
return searchSessionMessages(query, options)
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
export async function semanticSearchSessionMessages(query: string, options?: HybridSearchOptions): Promise<SearchResult[]> {
|
|
1287
|
-
const term = query.trim()
|
|
1288
|
-
if (!term) return []
|
|
1289
|
-
if (options?.dbPath === ":memory:") return []
|
|
1290
|
-
const dbPath = options?.dbPath ?? resolveDatabasePath()
|
|
1291
|
-
const db = getDb(dbPath)
|
|
1292
|
-
const limit = options?.limit ?? 80
|
|
1293
|
-
const dir = options?.directory
|
|
1294
|
-
const role = options?.role
|
|
1295
|
-
|
|
1296
|
-
const index = ensureSearchIndex(db, dbPath)
|
|
1297
|
-
if (!index) return []
|
|
1298
|
-
|
|
1299
|
-
const keyword = indexedTextRows(db, dbPath, limit, term, dir, options?.offset, role) ?? []
|
|
1300
|
-
const config = parseSemanticConfig()
|
|
1301
|
-
|
|
1302
|
-
let vector: Row[] = []
|
|
1303
|
-
if (!config.disableVector) {
|
|
1304
|
-
const vecState = getMeta(index, "vector_state")
|
|
1305
|
-
if (vecState === "enabled") {
|
|
1306
|
-
try {
|
|
1307
|
-
const client = new LlamaEmbeddingClient({
|
|
1308
|
-
baseUrl: config.embedBaseUrl,
|
|
1309
|
-
model: config.embedModel,
|
|
1310
|
-
documentPrefix: config.documentPrefix,
|
|
1311
|
-
queryPrefix: config.queryPrefix,
|
|
1312
|
-
})
|
|
1313
|
-
const embedding = await client.embedQuery(term)
|
|
1314
|
-
vector = searchVector(index, embedding, limit)
|
|
1315
|
-
debug.log("query:vector:results", { count: vector.length })
|
|
1316
|
-
} catch (err) {
|
|
1317
|
-
debug.log("query:vector:error", err instanceof Error ? err.message : String(err))
|
|
1318
|
-
}
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
|
|
1322
|
-
const merged = keyword.length || vector.length
|
|
1323
|
-
? hybridBlend(keyword, vector, config.hybridAlpha)
|
|
1324
|
-
: []
|
|
1325
|
-
|
|
1326
|
-
const results: SearchResult[] = []
|
|
1327
|
-
const seen = new Set<string>()
|
|
1328
|
-
for (const row of merged) {
|
|
1329
|
-
const result = rowToSearchResult(row, term) ?? rowToVectorResult(row)
|
|
1330
|
-
if (result) {
|
|
1331
|
-
seen.add(row.id)
|
|
1332
|
-
results.push(result)
|
|
1333
|
-
}
|
|
1334
|
-
}
|
|
1335
|
-
|
|
1336
|
-
for (const row of vector) {
|
|
1337
|
-
if (!seen.has(row.id)) {
|
|
1338
|
-
const result = rowToSearchResult(row, term) ?? rowToVectorResult(row)
|
|
1339
|
-
if (result) results.push(result)
|
|
1340
|
-
}
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
return results
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
|
-
export function rowToVectorResult(row: Row): SearchResult | undefined {
|
|
1347
|
-
const text = row.text.trim()
|
|
1348
|
-
if (!text) return
|
|
1349
|
-
const excerpt = text.slice(0, 200)
|
|
1350
|
-
return {
|
|
1351
|
-
id: row.id,
|
|
1352
|
-
messageID: row.message_id,
|
|
1353
|
-
sessionID: row.session_id,
|
|
1354
|
-
sessionTitle: row.session_title || "Untitled session",
|
|
1355
|
-
directory: row.directory,
|
|
1356
|
-
role: row.role,
|
|
1357
|
-
partType: row.part_type ?? "text",
|
|
1358
|
-
tool: row.tool ?? undefined,
|
|
1359
|
-
timeCreated: row.time_created,
|
|
1360
|
-
snippet: excerpt,
|
|
1361
|
-
matchStart: -1,
|
|
1362
|
-
matchEnd: -1,
|
|
1363
|
-
before: "",
|
|
1364
|
-
match: "",
|
|
1365
|
-
after: excerpt,
|
|
1366
|
-
excerpt,
|
|
1367
|
-
previewBefore: text.slice(0, Math.min(text.length, 1400)),
|
|
1368
|
-
previewMatch: "",
|
|
1369
|
-
previewAfter: "",
|
|
1370
|
-
previewMode: "markdown" as const,
|
|
1371
|
-
previewHighlight: false,
|
|
1372
|
-
text,
|
|
1373
|
-
}
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
function ftsQuery(query: string) {
|
|
1377
|
-
const tokens = query.trim().split(/\s+/)
|
|
1378
|
-
.map((token) => token.replace(/["*^:()]/g, " ").trim())
|
|
1379
|
-
.filter(Boolean)
|
|
1380
|
-
if (!tokens.length) return ""
|
|
1381
|
-
return tokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(" AND ")
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
function getMeta(db: Database, key: string) {
|
|
1385
|
-
return db.query<{ value: string }, [string]>("SELECT value FROM index_meta WHERE key = ?").get(key)?.value
|
|
1386
|
-
}
|
|
1387
|
-
|
|
1388
|
-
function setMeta(db: Database, key: string, value: string) {
|
|
1389
|
-
db.query<unknown, [string, string]>(`
|
|
1390
|
-
INSERT INTO index_meta(key, value) VALUES (?, ?)
|
|
1391
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
1392
|
-
`).run(key, value)
|
|
1393
|
-
}
|
|
1394
|
-
|
|
1395
|
-
function indexSourceRowToRows(row: IndexSourceRow): Row[] {
|
|
1396
|
-
const text = extractIndexText(row.data)
|
|
1397
|
-
if (!text) return []
|
|
1398
|
-
return [{ ...row, text }]
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
function extractIndexText(data: string) {
|
|
1402
|
-
try {
|
|
1403
|
-
const value = JSON.parse(data) as unknown
|
|
1404
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return ""
|
|
1405
|
-
const record = value as Record<string, unknown>
|
|
1406
|
-
if (record.type === "text") return typeof record.text === "string" ? record.text.trim() : ""
|
|
1407
|
-
if (record.type !== "tool") return ""
|
|
1408
|
-
return extractToolIndexText(record).replace(/\s+/g, " ").trim()
|
|
1409
|
-
} catch {
|
|
1410
|
-
return ""
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
function extractToolIndexText(part: Record<string, unknown>) {
|
|
1415
|
-
const tool = typeof part.tool === "string" ? part.tool : ""
|
|
1416
|
-
const state = recordValue(part.state)
|
|
1417
|
-
const input = recordValue(state?.input)
|
|
1418
|
-
const metadata = recordValue(state?.metadata)
|
|
1419
|
-
|
|
1420
|
-
if (tool === "apply_patch") {
|
|
1421
|
-
const files = Array.isArray(metadata?.files) ? metadata.files : []
|
|
1422
|
-
const renderedPatches = files.map(applyPatchFileIndexText).filter(Boolean).join("\n")
|
|
1423
|
-
const patchText = stringValue(input?.patchText)
|
|
1424
|
-
return [renderedPatches, patchText].filter(Boolean).join("\n")
|
|
1425
|
-
}
|
|
1426
|
-
|
|
1427
|
-
if (tool === "edit") {
|
|
1428
|
-
const filediff = recordValue(metadata?.filediff)
|
|
1429
|
-
return [
|
|
1430
|
-
stringValue(input?.filePath),
|
|
1431
|
-
stringValue(metadata?.diff),
|
|
1432
|
-
stringValue(filediff?.patch),
|
|
1433
|
-
stringValue(input?.oldString),
|
|
1434
|
-
stringValue(input?.newString),
|
|
1435
|
-
].filter(Boolean).join("\n")
|
|
1436
|
-
}
|
|
1437
|
-
|
|
1438
|
-
if (tool === "write") {
|
|
1439
|
-
return [stringValue(input?.filePath), stringValue(input?.content)].filter(Boolean).join("\n")
|
|
1440
|
-
}
|
|
1441
|
-
|
|
1442
|
-
return ""
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
function applyPatchFileIndexText(value: unknown) {
|
|
1446
|
-
const file = recordValue(value)
|
|
1447
|
-
if (!file) return ""
|
|
1448
|
-
return [
|
|
1449
|
-
stringValue(file.filePath),
|
|
1450
|
-
stringValue(file.relativePath),
|
|
1451
|
-
stringValue(file.patch),
|
|
1452
|
-
].filter(Boolean).join("\n")
|
|
1453
|
-
}
|
|
1454
|
-
|
|
1455
|
-
function recordValue(value: unknown): Record<string, unknown> | undefined {
|
|
1456
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return
|
|
1457
|
-
return value as Record<string, unknown>
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
function stringValue(value: unknown) {
|
|
1461
|
-
return typeof value === "string" ? value : undefined
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
|
-
const tableCache = new Map<string, boolean>()
|
|
1465
|
-
function tableExists(db: Database, name: string) {
|
|
1466
|
-
if (tableCache.has(name)) return tableCache.get(name)!
|
|
1467
|
-
const exists = Boolean(db.query<{ name: string }, [string]>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name))
|
|
1468
|
-
tableCache.set(name, exists)
|
|
1469
|
-
return exists
|
|
1470
|
-
}
|
|
1471
|
-
|
|
1472
|
-
function defaultDataDir() {
|
|
1473
|
-
if (process.env.XDG_DATA_HOME) return path.join(process.env.XDG_DATA_HOME, "opencode")
|
|
1474
|
-
return path.join(homedir(), ".local", "share", "opencode")
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
function candidateDataDirs() {
|
|
1478
|
-
return [
|
|
1479
|
-
defaultDataDir(),
|
|
1480
|
-
path.join(homedir(), ".local", "share", "opencode"),
|
|
1481
|
-
process.platform === "darwin" ? path.join(homedir(), "Library", "Application Support", "opencode") : undefined,
|
|
1482
|
-
process.platform === "win32" ? path.join(process.env.APPDATA ?? path.join(homedir(), "AppData", "Roaming"), "opencode") : undefined,
|
|
1483
|
-
].filter((item, index, list): item is string => Boolean(item) && list.indexOf(item) === index)
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
function candidateDatabasePaths(names: string[]) {
|
|
1487
|
-
return candidateDataDirs().flatMap((dir) => names.map((name) => path.join(dir, name)))
|
|
1488
|
-
}
|
|
1489
|
-
|
|
1490
|
-
function requireExistingDatabase(names: string[]) {
|
|
1491
|
-
return candidateDatabasePaths(names).find(existsSync) ?? candidateDatabasePaths(names)[0]!
|
|
1492
|
-
}
|
|
1493
|
-
|
|
1494
|
-
function extractFromValue(value: unknown): string {
|
|
1495
|
-
if (typeof value === "string") return value
|
|
1496
|
-
if (typeof value === "number" || typeof value === "boolean") return String(value)
|
|
1497
|
-
if (!value) return ""
|
|
1498
|
-
if (Array.isArray(value)) return value.map(extractFromValue).filter(Boolean).join("\n")
|
|
1499
|
-
if (typeof value !== "object") return ""
|
|
1500
|
-
return Object.entries(value)
|
|
1501
|
-
.filter(([key]) => !["id", "sessionID", "messageID", "time", "timeCreated", "timeUpdated", "tokens", "cost"].includes(key))
|
|
1502
|
-
.map(([, item]) => extractFromValue(item))
|
|
1503
|
-
.filter(Boolean)
|
|
1504
|
-
.join("\n")
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
|
-
function truncate(value: string, length: number) {
|
|
1508
|
-
if (value.length <= length) return value
|
|
1509
|
-
return `${value.slice(0, length - 3)}...`
|
|
1510
|
-
}
|
|
1
|
+
// Re-exported types
|
|
2
|
+
export type {
|
|
3
|
+
SearchResult,
|
|
4
|
+
SearchRole,
|
|
5
|
+
ConversationPreviewPart,
|
|
6
|
+
ConversationPreviewPage,
|
|
7
|
+
ConversationPreviewCursor,
|
|
8
|
+
ToolState,
|
|
9
|
+
SemanticConfig,
|
|
10
|
+
VectorState,
|
|
11
|
+
DocumentRow,
|
|
12
|
+
ScoredRow,
|
|
13
|
+
HybridSearchOptions,
|
|
14
|
+
} from "./search/types.ts"
|
|
15
|
+
|
|
16
|
+
// Re-exported query functions
|
|
17
|
+
export {
|
|
18
|
+
searchSessionMessages,
|
|
19
|
+
recentSessionMessages,
|
|
20
|
+
loadConversationAround,
|
|
21
|
+
loadConversationBefore,
|
|
22
|
+
loadConversationAfter,
|
|
23
|
+
performSearch,
|
|
24
|
+
semanticSearchSessionMessages,
|
|
25
|
+
parseSemanticConfig,
|
|
26
|
+
} from "./search/queries.ts"
|
|
27
|
+
|
|
28
|
+
// Re-exported text/snippet functions
|
|
29
|
+
export {
|
|
30
|
+
rowToSearchResult,
|
|
31
|
+
rowToVectorResult,
|
|
32
|
+
makeSnippet,
|
|
33
|
+
extractSearchText,
|
|
34
|
+
ftsQuery,
|
|
35
|
+
expandQuery,
|
|
36
|
+
} from "./search/text.ts"
|
|
37
|
+
|
|
38
|
+
// Re-exported vector/blend functions
|
|
39
|
+
export {
|
|
40
|
+
hybridBlend,
|
|
41
|
+
} from "./search/vector.ts"
|
|
42
|
+
|
|
43
|
+
// Re-exported path utilities
|
|
44
|
+
export {
|
|
45
|
+
resolveDatabasePath,
|
|
46
|
+
searchIndexPath,
|
|
47
|
+
} from "./search/db-path.ts"
|