@bojackduy/opencode-telescope 0.1.24 → 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/search.ts CHANGED
@@ -1,1510 +1,47 @@
1
- import { Database } from "bun:sqlite"
2
- import { createHash } from "node:crypto"
3
- import { existsSync, readdirSync, statSync } from "node:fs"
4
- import { homedir } from "node:os"
5
- import path from "node:path"
6
- import { debug } from "./ui/debug.ts"
7
- import { LlamaEmbeddingClient } from "./search/embedding.ts"
8
-
9
- export type SearchResult = {
10
- id: string
11
- messageID: string
12
- sessionID: string
13
- sessionTitle: string
14
- directory: string
15
- role: "user" | "assistant"
16
- partType: "text" | "reasoning" | "tool"
17
- tool?: string
18
- timeCreated: number
19
- snippet: string
20
- matchStart: number
21
- matchEnd: number
22
- before: string
23
- match: string
24
- after: string
25
- excerpt: string
26
- previewBefore: string
27
- previewMatch: string
28
- previewAfter: string
29
- previewMode: "markdown" | "text"
30
- previewHighlight: boolean
31
- text: string
32
- }
33
-
34
- export type SearchRole = "user" | "assistant"
35
-
36
- export type ConversationPreviewPart = {
37
- id: string
38
- messageID: string
39
- sessionID: string
40
- role: "user" | "assistant"
41
- type: "text" | "reasoning" | "tool"
42
- timeCreated: number
43
- text: string
44
- tool?: string
45
- state?: ToolState
46
- target: boolean
47
- }
48
-
49
- export type ConversationPreviewPage = {
50
- parts: ConversationPreviewPart[]
51
- hasMoreBefore: boolean
52
- hasMoreAfter: boolean
53
- }
54
-
55
- export type ConversationPreviewCursor = {
56
- id: string
57
- timeCreated: number
58
- }
59
-
60
- export type ToolState = {
61
- status: "pending" | "running" | "completed" | "error"
62
- input?: unknown
63
- metadata?: unknown
64
- output?: string
65
- error?: string
66
- }
67
-
68
- export type SemanticConfig = {
69
- embedBaseUrl: string
70
- embedModel?: string
71
- disableVector: boolean
72
- sqliteLibPath?: string
73
- sqliteVecExtension?: string
74
- hybridAlpha: number
75
- documentPrefix: string
76
- queryPrefix: string
77
- }
78
-
79
- export type VectorState = "enabled" | "disabled" | "unavailable" | "stale"
80
-
81
- export const DEFAULT_EMBED_BASE_URL = "http://127.0.0.1:8081"
82
- export const DEFAULT_DOCUMENT_PREFIX = "search_document: "
83
- export const DEFAULT_QUERY_PREFIX = "search_query: "
84
- export const DEFAULT_HYBRID_ALPHA = 0.45
85
-
86
- export function parseSemanticConfig(env: Record<string, string | undefined> = process.env): SemanticConfig {
87
- return {
88
- embedBaseUrl: env.OPENCODE_TELESCOPE_EMBED_BASE_URL ?? DEFAULT_EMBED_BASE_URL,
89
- embedModel: env.OPENCODE_TELESCOPE_EMBED_MODEL || undefined,
90
- disableVector: env.OPENCODE_TELESCOPE_DISABLE_VECTOR === "1" || env.OPENCODE_TELESCOPE_DISABLE_VECTOR === "true",
91
- sqliteLibPath: env.OPENCODE_TELESCOPE_SQLITE_LIB || undefined,
92
- sqliteVecExtension: env.OPENCODE_TELESCOPE_SQLITE_VEC_EXT || undefined,
93
- hybridAlpha: parseAlpha(env.OPENCODE_TELESCOPE_HYBRID_ALPHA),
94
- documentPrefix: DEFAULT_DOCUMENT_PREFIX,
95
- queryPrefix: DEFAULT_QUERY_PREFIX,
96
- }
97
- }
98
-
99
- function parseAlpha(value: string | undefined) {
100
- if (!value) return DEFAULT_HYBRID_ALPHA
101
- const parsed = Number(value)
102
- if (!Number.isFinite(parsed)) return DEFAULT_HYBRID_ALPHA
103
- if (parsed < 0) return 0
104
- if (parsed > 1) return 1
105
- return parsed
106
- }
107
-
108
- type Row = {
109
- id: string
110
- message_id: string
111
- session_id: string
112
- session_title: string | null
113
- directory: string
114
- role: SearchRole
115
- part_type?: SearchResult["partType"]
116
- tool?: string | null
117
- time_created: number
118
- text: string
119
- }
120
-
121
- type IndexSourceRow = Omit<Row, "text"> & {
122
- data: string
123
- }
124
-
125
- export type DocumentRow = {
126
- doc_id: string
127
- part_id: string
128
- message_id: string
129
- session_id: string
130
- session_title: string
131
- directory: string
132
- role: string
133
- part_type: string
134
- tool: string | null
135
- time_created: number
136
- chunk_index: number
137
- text: string
138
- source_hash: string
139
- extractor_version: string
140
- }
141
-
142
- type ConversationRow = {
143
- id: string
144
- message_id: string
145
- session_id: string
146
- role: SearchRole
147
- type: "text" | "reasoning" | "tool"
148
- time_created: number
149
- data: string
150
- }
151
-
152
- let cachedDbPath: string | undefined
153
- let _db: Database | undefined
154
- let _dbPath: string | undefined
155
- let _indexDb: Database | undefined
156
- let _indexDbPath: string | undefined
157
- const backgroundIndexRebuilds = new Set<string>()
158
-
159
- function getDb(dbPath?: string): Database {
160
- const resolved = dbPath ?? resolveDatabasePath()
161
- if (!_db || resolved !== _dbPath) {
162
- _db?.close()
163
- _db = new Database(resolved, { readonly: true })
164
- _dbPath = resolved
165
- tableCache.clear()
166
- }
167
- return _db
168
- }
169
-
170
- export function resolveDatabasePath() {
171
- if (cachedDbPath) return cachedDbPath
172
- if (process.env.OPENCODE_DB) {
173
- if (process.env.OPENCODE_DB === ":memory:" || path.isAbsolute(process.env.OPENCODE_DB)) return cachedDbPath = process.env.OPENCODE_DB
174
- return cachedDbPath = path.join(candidateDataDirs()[0] ?? defaultDataDir(), process.env.OPENCODE_DB)
175
- }
176
- if (process.env.OPENCODE_DISABLE_CHANNEL_DB === "1" || process.env.OPENCODE_DISABLE_CHANNEL_DB === "true") {
177
- return cachedDbPath = requireExistingDatabase(["opencode.db"])
178
- }
179
- const stable = candidateDatabasePaths(["opencode.db"]).find(existsSync)
180
- if (stable) return cachedDbPath = stable
181
- if (process.env.OPENCODE_CHANNEL) {
182
- const channel = process.env.OPENCODE_CHANNEL.replace(/[^a-zA-Z0-9._-]/g, "-")
183
- const candidate = candidateDatabasePaths([`opencode-${channel}.db`]).find(existsSync)
184
- if (candidate) return cachedDbPath = candidate
185
- }
186
- const fallback = candidateDatabasePaths(["opencode.db"])[0] ?? path.join(defaultDataDir(), "opencode.db")
187
- try {
188
- const discovered = candidateDataDirs()
189
- .flatMap((dir) =>
190
- readdirSync(dir, { withFileTypes: true })
191
- .filter((entry) => entry.isFile() && /^opencode-.+\.db$/.test(entry.name))
192
- .map((entry) => path.join(dir, entry.name)),
193
- )
194
- .at(0)
195
- return cachedDbPath = discovered ?? fallback
196
- } catch {
197
- return cachedDbPath = fallback
198
- }
199
- }
200
-
201
- export function searchSessionMessages(query: string, options?: { limit?: number; offset?: number; dbPath?: string; directory?: string; role?: SearchRole }) {
202
- const term = query.trim()
203
- if (!term) return []
204
- if (options?.dbPath === ":memory:") return []
205
- const dbPath = options?.dbPath ?? resolveDatabasePath()
206
- const db = getDb(dbPath)
207
- return searchRows(db, dbPath, term, options?.limit ?? 80, options?.directory, options?.offset, options?.role)
208
- }
209
-
210
- export function recentSessionMessages(options?: { limit?: number; offset?: number; dbPath?: string; directory?: string; role?: SearchRole }) {
211
- const dbPath = options?.dbPath ?? resolveDatabasePath()
212
- const db = getDb(dbPath)
213
- const limit = options?.limit ?? 40
214
- const indexed = dbPath === ":memory:" ? undefined : indexedRecentRows(db, dbPath, limit, options?.directory, options?.offset, options?.role)
215
- if (indexed) return indexed.flatMap((row) => rowToSearchResult(row, "") ?? [])
216
- debug.log("query:recent:source-fallback", { limit, offset: options?.offset ?? 0, directory: options?.directory, role: options?.role })
217
- return visibleTextRows(db, limit, undefined, options?.directory, options?.offset, options?.role).flatMap(
218
- (row) => rowToSearchResult(row, "") ?? [],
219
- )
220
- }
221
-
222
- export function loadConversationAround(result: SearchResult, options?: { before?: number; after?: number; dbPath?: string }): ConversationPreviewPage {
223
- debug.time("preview:query:total")
224
- const db = getDb(options?.dbPath)
225
- const before = options?.before ?? 3
226
- const after = options?.after ?? 6
227
-
228
- debug.time("preview:query:hit")
229
- const hit = db.query<{ time_created: number }, [string]>(
230
- "SELECT time_created FROM part WHERE id = ?",
231
- ).get(result.id)
232
- debug.timeEnd("preview:query:hit")
233
- if (!hit) {
234
- debug.log("preview:window", {
235
- item: result.id,
236
- session: result.sessionID,
237
- before,
238
- after,
239
- hit: false,
240
- })
241
- debug.timeEnd("preview:query:total")
242
- return { parts: [], hasMoreBefore: false, hasMoreAfter: false }
243
- }
244
-
245
- const fetchBefore = Math.max(before * 4, before + 1, 30)
246
- const fetchAfter = Math.max((after + 1) * 4, after + 2, 50)
247
-
248
- debug.time("preview:query:before")
249
- const beforeRows = db.query<ConversationRow, [string, number, number, string, number]>(`
250
- SELECT p.id, p.message_id, p.session_id,
251
- json_extract(m.data, '$.role') AS role,
252
- json_extract(p.data, '$.type') AS type,
253
- p.time_created, p.data
254
- FROM part p
255
- JOIN message m ON m.id = p.message_id
256
- WHERE p.session_id = ?
257
- AND (p.time_created < ? OR (p.time_created = ? AND p.id < ?))
258
- ORDER BY p.time_created DESC, p.id DESC
259
- LIMIT ?
260
- `).all(result.sessionID, hit.time_created, hit.time_created, result.id, fetchBefore)
261
- debug.timeEnd("preview:query:before")
262
-
263
- debug.time("preview:query:after")
264
- const afterRows = db.query<ConversationRow, [string, number, number, string, number]>(`
265
- SELECT p.id, p.message_id, p.session_id,
266
- json_extract(m.data, '$.role') AS role,
267
- json_extract(p.data, '$.type') AS type,
268
- p.time_created, p.data
269
- FROM part p
270
- JOIN message m ON m.id = p.message_id
271
- WHERE p.session_id = ?
272
- AND (p.time_created > ? OR (p.time_created = ? AND p.id >= ?))
273
- ORDER BY p.time_created ASC, p.id ASC
274
- LIMIT ?
275
- `).all(result.sessionID, hit.time_created, hit.time_created, result.id, fetchAfter)
276
- debug.timeEnd("preview:query:after")
277
-
278
- debug.time("preview:query:parse")
279
- const validBefore = beforeRows.filter(isPreviewRow)
280
- const validAfter = afterRows.filter(isPreviewRow)
281
- const parts = [
282
- ...validBefore.slice(0, before).reverse(),
283
- ...validAfter.slice(0, after + 1),
284
- ].flatMap((row) => parseConversationPart(row, row.id === result.id) ?? [])
285
- const page = {
286
- parts,
287
- hasMoreBefore: validBefore.length > before,
288
- hasMoreAfter: validAfter.length > after + 1,
289
- }
290
- debug.log("preview:window", {
291
- item: result.id,
292
- session: result.sessionID,
293
- mode: "around",
294
- before,
295
- after,
296
- fetchBefore,
297
- fetchAfter,
298
- beforeRows: beforeRows.length,
299
- validBefore: validBefore.length,
300
- invalidBefore: previewRowBreakdown(beforeRows, validBefore),
301
- afterRows: afterRows.length,
302
- validAfter: validAfter.length,
303
- invalidAfter: previewRowBreakdown(afterRows, validAfter),
304
- parts: parts.length,
305
- hasMoreBefore: page.hasMoreBefore,
306
- hasMoreAfter: page.hasMoreAfter,
307
- first: parts[0]?.id,
308
- last: parts.at(-1)?.id,
309
- })
310
- debug.timeEnd("preview:query:parse")
311
- debug.timeEnd("preview:query:total")
312
- return page
313
- }
314
-
315
- export function loadConversationBefore(result: SearchResult, cursor: ConversationPreviewCursor, options?: { limit?: number; dbPath?: string }) {
316
- debug.time("preview:query:before-page")
317
- const db = getDb(options?.dbPath)
318
- const limit = options?.limit ?? 20
319
- const fetchLimit = Math.max(limit * 4, limit + 1, 30)
320
- const rows = db.query<ConversationRow, [string, number, number, string, number]>(`
321
- SELECT p.id, p.message_id, p.session_id,
322
- json_extract(m.data, '$.role') AS role,
323
- json_extract(p.data, '$.type') AS type,
324
- p.time_created, p.data
325
- FROM part p
326
- JOIN message m ON m.id = p.message_id
327
- WHERE p.session_id = ?
328
- AND (p.time_created < ? OR (p.time_created = ? AND p.id < ?))
329
- ORDER BY p.time_created DESC, p.id DESC
330
- LIMIT ?
331
- `).all(result.sessionID, cursor.timeCreated, cursor.timeCreated, cursor.id, fetchLimit)
332
- const valid = rows.filter(isPreviewRow)
333
- const parts = valid.slice(0, limit).reverse().flatMap((row) => parseConversationPart(row, false) ?? [])
334
- const parsedIds = new Set(parts.map((part) => part.id))
335
- const page = { parts, hasMoreBefore: valid.length > limit }
336
- debug.log("preview:window", {
337
- item: result.id,
338
- session: result.sessionID,
339
- mode: "before",
340
- cursor,
341
- limit,
342
- rows: rows.length,
343
- valid: valid.length,
344
- invalid: previewRowBreakdown(rows, valid),
345
- parts: parts.length,
346
- unparsedValid: valid.slice(0, limit).filter((row) => !parsedIds.has(row.id)).map((row) => ({ id: row.id, type: row.type, role: row.role })),
347
- hasMoreBefore: page.hasMoreBefore,
348
- first: parts[0]?.id,
349
- last: parts.at(-1)?.id,
350
- })
351
- debug.timeEnd("preview:query:before-page")
352
- return page
353
- }
354
-
355
- export function loadConversationAfter(result: SearchResult, cursor: ConversationPreviewCursor, options?: { limit?: number; dbPath?: string }) {
356
- debug.time("preview:query:after-page")
357
- const db = getDb(options?.dbPath)
358
- const limit = options?.limit ?? 20
359
- const fetchLimit = Math.max(limit * 4, limit + 1, 30)
360
- const rows = db.query<ConversationRow, [string, number, number, string, number]>(`
361
- SELECT p.id, p.message_id, p.session_id,
362
- json_extract(m.data, '$.role') AS role,
363
- json_extract(p.data, '$.type') AS type,
364
- p.time_created, p.data
365
- FROM part p
366
- JOIN message m ON m.id = p.message_id
367
- WHERE p.session_id = ?
368
- AND (p.time_created > ? OR (p.time_created = ? AND p.id > ?))
369
- ORDER BY p.time_created ASC, p.id ASC
370
- LIMIT ?
371
- `).all(result.sessionID, cursor.timeCreated, cursor.timeCreated, cursor.id, fetchLimit)
372
- const valid = rows.filter(isPreviewRow)
373
- const parts = valid.slice(0, limit).flatMap((row) => parseConversationPart(row, false) ?? [])
374
- const parsedIds = new Set(parts.map((part) => part.id))
375
- const page = { parts, hasMoreAfter: valid.length > limit }
376
- debug.log("preview:window", {
377
- item: result.id,
378
- session: result.sessionID,
379
- mode: "after",
380
- cursor,
381
- limit,
382
- rows: rows.length,
383
- valid: valid.length,
384
- invalid: previewRowBreakdown(rows, valid),
385
- parts: parts.length,
386
- unparsedValid: valid.slice(0, limit).filter((row) => !parsedIds.has(row.id)).map((row) => ({ id: row.id, type: row.type, role: row.role })),
387
- hasMoreAfter: page.hasMoreAfter,
388
- first: parts[0]?.id,
389
- last: parts.at(-1)?.id,
390
- })
391
- debug.timeEnd("preview:query:after-page")
392
- return page
393
- }
394
-
395
- export function rowToSearchResult(row: Row, query: string): SearchResult | undefined {
396
- const text = row.text.trim()
397
- const match = findMatch(text, query)
398
- if (!match) return
399
- const preview = focusedPreview(text, match.start, match.end)
400
- return {
401
- id: row.id,
402
- messageID: row.message_id,
403
- sessionID: row.session_id,
404
- sessionTitle: row.session_title || "Untitled session",
405
- directory: row.directory,
406
- role: row.role,
407
- partType: row.part_type ?? "text",
408
- tool: row.tool ?? undefined,
409
- timeCreated: row.time_created,
410
- snippet: makeSnippet(text, query),
411
- matchStart: match.start,
412
- matchEnd: match.end,
413
- before: match.before,
414
- match: match.match,
415
- after: match.after,
416
- excerpt: match.excerpt,
417
- previewBefore: preview.before,
418
- previewMatch: preview.match,
419
- previewAfter: preview.after,
420
- previewMode: preview.mode,
421
- previewHighlight: preview.highlight,
422
- text,
423
- }
424
- }
425
-
426
- export function extractSearchText(data: string) {
427
- try {
428
- return extractFromValue(JSON.parse(data)).replace(/\s+/g, " ").trim()
429
- } catch {
430
- return data.replace(/\s+/g, " ").trim()
431
- }
432
- }
433
-
434
- export function makeSnippet(text: string, query: string, radius = 72) {
435
- const haystack = text.replace(/\s+/g, " ").trim()
436
- const tokens = query.trim().split(/\s+/)
437
- const lower = haystack.toLowerCase()
438
- let searchPos = 0
439
- let firstIndex = -1
440
- let lastEnd = 0
441
- for (const token of tokens) {
442
- const index = lower.indexOf(token.toLowerCase(), searchPos)
443
- if (index === -1) return truncate(haystack, radius * 2)
444
- if (firstIndex === -1) firstIndex = index
445
- searchPos = index + token.length
446
- lastEnd = searchPos
447
- }
448
- const start = Math.max(0, firstIndex - radius)
449
- const end = Math.min(haystack.length, lastEnd + radius)
450
- return `${start > 0 ? "..." : ""}${haystack.slice(start, end)}${end < haystack.length ? "..." : ""}`
451
- }
452
-
453
- function findMatch(text: string, query: string, radius = 96) {
454
- const needle = query.trim()
455
- if (!needle) {
456
- const collapsed = text.replace(/\s+/g, " ").trim()
457
- const end = Math.min(collapsed.length, radius * 2)
458
- return {
459
- start: 0,
460
- end: 0,
461
- before: "",
462
- match: "",
463
- after: collapsed.slice(0, end),
464
- excerpt: collapsed.slice(0, end),
465
- }
466
- }
467
- const tokens = needle.split(/\s+/)
468
- const lowerText = text.toLowerCase()
469
- let searchPos = 0
470
- let firstStart = -1
471
- let lastEnd = -1
472
- for (const token of tokens) {
473
- const pos = lowerText.indexOf(token.toLowerCase(), searchPos)
474
- if (pos === -1) return
475
- if (firstStart === -1) firstStart = pos
476
- searchPos = pos + token.length
477
- lastEnd = searchPos
478
- }
479
- const start = firstStart
480
- const end = lastEnd
481
- const lineStart = text.lastIndexOf("\n", start - 1) + 1
482
- const nextLine = text.indexOf("\n", end)
483
- const lineEnd = nextLine === -1 ? text.length : nextLine
484
- const line = text.slice(lineStart, lineEnd)
485
- const matchLen = end - start
486
- const lineMatchStart = Math.max(0, start - lineStart)
487
- const lineMatchEnd = lineMatchStart + matchLen
488
- const excerptStart = Math.max(0, lineMatchStart - radius)
489
- const excerptEnd = Math.min(line.length, lineMatchEnd + radius)
490
- const before = normalizeSnippetSegment(line.slice(excerptStart, lineMatchStart))
491
- const after = normalizeSnippetSegment(line.slice(lineMatchEnd, excerptEnd))
492
- return {
493
- start,
494
- end,
495
- before: `${excerptStart > 0 ? "..." : ""}${before}`,
496
- match: text.slice(start, end),
497
- after: `${after}${excerptEnd < line.length ? "..." : ""}`,
498
- excerpt: `${excerptStart > 0 ? "..." : ""}${normalizeSnippetSegment(line.slice(excerptStart, excerptEnd))}${excerptEnd < line.length ? "..." : ""}`,
499
- }
500
- }
501
-
502
- function normalizeSnippetSegment(value: string) {
503
- return value.replace(/\s+/g, " ")
504
- }
505
-
506
- function focusedPreview(text: string, matchStart: number, matchEnd: number) {
507
- if (matchStart === matchEnd) {
508
- const preview = text.slice(0, Math.min(text.length, 1400))
509
- return { before: preview, match: "", after: "", mode: "markdown" as const, highlight: false }
510
- }
511
-
512
- const lineStart = text.lastIndexOf("\n", matchStart - 1) + 1
513
- const nextLine = text.indexOf("\n", matchEnd)
514
- const lineEnd = nextLine === -1 ? text.length : nextLine
515
- const line = text.slice(lineStart, lineEnd)
516
-
517
- if (line.length > 260) {
518
- const beforeStart = Math.max(0, matchStart - 180)
519
- const afterEnd = Math.min(text.length, matchEnd + 220)
520
- return {
521
- before: `${beforeStart > 0 ? "..." : ""}${text.slice(beforeStart, matchStart)}`,
522
- match: text.slice(matchStart, matchEnd),
523
- after: `${text.slice(matchEnd, afterEnd)}${afterEnd < text.length ? "..." : ""}`,
524
- mode: "text" as const,
525
- highlight: true,
526
- }
527
- }
528
-
529
- const window = lineWindow(text, matchStart, matchEnd, 40, 80)
530
- const windowText = text.slice(window.start, window.end)
531
- const relativeStart = matchStart - window.start
532
- const relativeEnd = matchEnd - window.start
533
-
534
- const insideCodeFence = isInsideCodeFence(text, matchStart)
535
- return {
536
- before: `${window.start > 0 ? "...\n" : ""}${windowText.slice(0, relativeStart)}`,
537
- match: windowText.slice(relativeStart, relativeEnd),
538
- after: `${windowText.slice(relativeEnd)}${window.end < text.length ? "\n..." : ""}`,
539
- mode: "markdown" as const,
540
- highlight: !insideCodeFence,
541
- }
542
- }
543
-
544
- function lineWindow(text: string, matchStart: number, matchEnd: number, before: number, after: number) {
545
- const starts = [0]
546
- for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) {
547
- starts.push(index + 1)
548
- }
549
- const matchLine = findLastStartIndex(starts, matchStart)
550
- const startLine = Math.max(0, matchLine - before)
551
- const endLine = Math.min(starts.length - 1, matchLine + after)
552
- const start = starts[startLine] ?? 0
553
- const end = starts[endLine + 1] ? starts[endLine + 1] - 1 : text.length
554
- return { start, end: Math.max(end, matchEnd) }
555
- }
556
-
557
- function findLastStartIndex(starts: number[], offset: number) {
558
- for (let index = starts.length - 1; index >= 0; index--) {
559
- if (starts[index]! <= offset) return index
560
- }
561
- return 0
562
- }
563
-
564
- function isInsideCodeFence(text: string, offset: number) {
565
- const before = text.slice(0, offset)
566
- const fences = before.match(/^```/gm)
567
- return Boolean(fences && fences.length % 2 === 1)
568
- }
569
-
570
- function parseConversationPart(row: ConversationRow, target: boolean): ConversationPreviewPart | undefined {
571
- const data = parsePartData(row.data)
572
- if (!data) return
573
- if (row.type === "tool") {
574
- return {
575
- id: row.id,
576
- messageID: row.message_id,
577
- sessionID: row.session_id,
578
- role: row.role,
579
- type: row.type,
580
- timeCreated: row.time_created,
581
- text: target ? extractToolIndexText(data).trim() : "",
582
- tool: typeof data.tool === "string" ? data.tool : "tool",
583
- state: parseToolState(data.state),
584
- target,
585
- }
586
- }
587
- const text = typeof data.text === "string" ? data.text.trim() : ""
588
- if (!text) return
589
- return {
590
- id: row.id,
591
- messageID: row.message_id,
592
- sessionID: row.session_id,
593
- role: row.role,
594
- type: row.type,
595
- timeCreated: row.time_created,
596
- text,
597
- target,
598
- }
599
- }
600
-
601
- function isPreviewRow(row: ConversationRow) {
602
- return (row.role === "user" || row.role === "assistant") &&
603
- (row.type === "text" || row.type === "reasoning" || row.type === "tool")
604
- }
605
-
606
- function previewRowBreakdown(rows: ConversationRow[], validRows: ConversationRow[]) {
607
- const valid = new Set(validRows.map((row) => row.id))
608
- const invalidRows = rows.filter((row) => !valid.has(row.id))
609
- if (invalidRows.length === 0) return undefined
610
- const byRole = countBy(invalidRows, (row) => String(row.role ?? "unknown"))
611
- const byType = countBy(invalidRows, (row) => String(row.type ?? "unknown"))
612
- return { count: invalidRows.length, byRole, byType }
613
- }
614
-
615
- function countBy<T>(items: T[], key: (item: T) => string) {
616
- const counts: Record<string, number> = {}
617
- for (const item of items) counts[key(item)] = (counts[key(item)] ?? 0) + 1
618
- return counts
619
- }
620
-
621
- function parsePartData(data: string) {
622
- try {
623
- const value = JSON.parse(data) as unknown
624
- if (!value || typeof value !== "object" || Array.isArray(value)) return
625
- return value as Record<string, unknown>
626
- } catch {
627
- return
628
- }
629
- }
630
-
631
- function parseToolState(value: unknown): ToolState | undefined {
632
- if (!value || typeof value !== "object" || Array.isArray(value)) return
633
- const state = value as Record<string, unknown>
634
- if (!["pending", "running", "completed", "error"].includes(String(state.status))) return
635
- return {
636
- status: state.status as ToolState["status"],
637
- input: state.input,
638
- metadata: state.metadata,
639
- output: typeof state.output === "string" ? state.output : undefined,
640
- error: typeof state.error === "string" ? state.error : undefined,
641
- }
642
- }
643
-
644
- function searchRows(db: Database, dbPath: string, query: string, limit: number, directory?: string, offset?: number, role?: SearchRole) {
645
- if (!tableExists(db, "part") || !tableExists(db, "message")) return []
646
- debug.time("query:sql")
647
- const rows = indexedTextRows(db, dbPath, limit, query, directory, offset, role) ?? visibleTextRows(db, limit, query, directory, offset, role)
648
- debug.timeEnd("query:sql")
649
- debug.time("query:map")
650
- const results = rows.flatMap((row) => rowToSearchResult(row, query) ?? [])
651
- debug.timeEnd("query:map")
652
- return results
653
- }
654
-
655
- function indexedTextRows(db: Database, dbPath: string, limit: number, query: string, directory?: string, offset?: number, role?: SearchRole) {
656
- const match = ftsQuery(query)
657
- if (!match) return []
658
- try {
659
- const index = ensureSearchIndex(db, dbPath)
660
- if (!index) return []
661
- const conditions = ["document_fts MATCH ?"]
662
- const params: (string | number)[] = [match]
663
- if (role) {
664
- conditions.push("role = ?")
665
- params.push(role)
666
- }
667
- if (directory) {
668
- conditions.push("directory = ?")
669
- params.push(directory)
670
- }
671
- params.push(limit)
672
- if (offset) params.push(offset)
673
- const offsetClause = offset ? "OFFSET ?" : ""
674
- debug.time("query:fts:exec")
675
- const rows = index.query<Row, (string | number)[]>(`
676
- SELECT id, message_id, session_id, session_title, directory, role, part_type, tool,
677
- CAST(time_created AS INTEGER) AS time_created, text
678
- FROM document_fts
679
- WHERE ${conditions.join(" AND ")}
680
- ORDER BY bm25(document_fts), CAST(time_created AS INTEGER) DESC
681
- LIMIT ? ${offsetClause}
682
- `).all(...params as any[])
683
- debug.timeEnd("query:fts:exec")
684
- return rows
685
- } catch (err) {
686
- debug.log("fts:fallback", err instanceof Error ? err.message : String(err))
687
- return
688
- }
689
- }
690
-
691
- function indexedRecentRows(db: Database, dbPath: string, limit: number, directory?: string, offset?: number, role?: SearchRole) {
692
- try {
693
- const index = ensureSearchIndex(db, dbPath, { rebuild: false, useStale: true })
694
- if (!index) {
695
- debug.log("query:recent:index:missing", { dbPath })
696
- return
697
- }
698
-
699
- const rowCount = index.query<{ count: number }, []>("SELECT COUNT(*) as count FROM document_index").get()?.count ?? 0
700
- if (rowCount === 0) {
701
- debug.log("query:recent:index:empty", { dbPath })
702
- return
703
- }
704
-
705
- const state = sourceState(db, dbPath)
706
- const currentDataVersion = getMeta(index, "source_data_version")
707
- const currentMtimeMs = getMeta(index, "source_mtime_ms")
708
- const currentPath = getMeta(index, "source_path")
709
- const currentIndexVersion = getMeta(index, "index_version")
710
- if (currentPath !== dbPath || currentDataVersion !== String(state.dataVersion) || currentMtimeMs !== String(state.mtimeMs) || currentIndexVersion !== SEARCH_INDEX_VERSION) {
711
- debug.log("query:recent:index:stale", {
712
- dbPath,
713
- expectedDataVersion: state.dataVersion,
714
- actualDataVersion: currentDataVersion,
715
- })
716
- }
717
-
718
- const conditions: string[] = ["part_type = 'text'"]
719
- const params: (string | number)[] = []
720
- if (role) {
721
- conditions.push("role = ?")
722
- params.push(role)
723
- }
724
- if (directory) {
725
- conditions.push("directory = ?")
726
- params.push(directory)
727
- }
728
- params.push(limit)
729
- if (offset) params.push(offset)
730
-
731
- const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""
732
- const offsetClause = offset ? "OFFSET ?" : ""
733
- debug.time("query:recent:index:exec")
734
- const rows = index.query<Row, (string | number)[]>(`
735
- SELECT id, message_id, session_id, session_title, directory, role, part_type, tool,
736
- CAST(time_created AS INTEGER) AS time_created, text
737
- FROM document_index
738
- ${where}
739
- ORDER BY CAST(time_created AS INTEGER) DESC
740
- LIMIT ? ${offsetClause}
741
- `).all(...params as any[])
742
- debug.timeEnd("query:recent:index:exec")
743
- return rows
744
- } catch (err) {
745
- debug.log("recent:index:fallback", err instanceof Error ? err.message : String(err))
746
- return
747
- }
748
- }
749
-
750
- function visibleTextRows(db: Database, limit: number, query?: string, directory?: string, offset?: number, role?: SearchRole) {
751
- const offsetClause = offset ? "OFFSET ?" : ""
752
- const conditions: string[] = [
753
- "json_extract(p.data, '$.type') = 'text'",
754
- role ? "json_extract(m.data, '$.role') = ?" : "json_extract(m.data, '$.role') IN ('user', 'assistant')",
755
- ]
756
- const params: (string | number)[] = []
757
-
758
- if (role) params.push(role)
759
-
760
- if (directory) {
761
- conditions.push("s.directory = ?")
762
- params.push(directory)
763
- }
764
-
765
- const tokens = query ? query.trim().split(/\s+/).filter(Boolean) : []
766
- for (const token of tokens) {
767
- conditions.push("json_extract(p.data, '$.text') LIKE ?")
768
- params.push(`%${token}%`)
769
- }
770
-
771
- params.push(limit)
772
- if (offset) params.push(offset)
773
-
774
- const sql = `
775
- SELECT p.id, p.message_id, p.session_id, s.title AS session_title, s.directory,
776
- json_extract(m.data, '$.role') AS role,
777
- p.time_created,
778
- json_extract(p.data, '$.text') AS text
779
- FROM part p
780
- JOIN message m ON m.id = p.message_id
781
- JOIN session s ON s.id = p.session_id
782
- WHERE ${conditions.join(" AND ")}
783
- ORDER BY p.time_created DESC
784
- LIMIT ? ${offsetClause}
785
- `
786
- debug.time("query:sql:exec")
787
- const rows = db.query<Row, (string | number)[]>(sql).all(...params as any[])
788
- debug.timeEnd("query:sql:exec")
789
- return rows
790
- }
791
-
792
- function ensureSearchIndex(source: Database, sourcePath: string, options?: { rebuild?: boolean; useStale?: boolean }) {
793
- const indexPath = searchIndexPath(sourcePath)
794
- if (!_indexDb || _indexDbPath !== indexPath) {
795
- _indexDb?.close()
796
- configureCustomSQLite()
797
- _indexDb = new Database(indexPath)
798
- _indexDbPath = indexPath
799
- migrateSearchIndex(_indexDb)
800
- }
801
-
802
- const state = sourceState(source, sourcePath)
803
- const currentDataVersion = getMeta(_indexDb, "source_data_version")
804
- const currentMtimeMs = getMeta(_indexDb, "source_mtime_ms")
805
- const currentPath = getMeta(_indexDb, "source_path")
806
- const currentIndexVersion = getMeta(_indexDb, "index_version")
807
- if (currentPath !== sourcePath || currentDataVersion !== String(state.dataVersion) || currentMtimeMs !== String(state.mtimeMs) || currentIndexVersion !== SEARCH_INDEX_VERSION) {
808
- if (options?.rebuild === false) {
809
- if (options?.useStale) return _indexDb
810
- return
811
- }
812
- rebuildSearchIndex(source, _indexDb, sourcePath, state)
813
- }
814
- return _indexDb
815
- }
816
-
817
- function scheduleBackgroundIndexRebuild(dbPath: string) {
818
- if (backgroundIndexRebuilds.has(dbPath)) return
819
- backgroundIndexRebuilds.add(dbPath)
820
- debug.log("fts:rebuild:background-scheduled", { dbPath })
821
- const timer = setTimeout(() => {
822
- debug.time("fts:rebuild:background")
823
- try {
824
- const db = getDb(dbPath)
825
- ensureSearchIndex(db, dbPath)
826
- } catch (err) {
827
- debug.log("fts:rebuild:background:error", err instanceof Error ? err.message : String(err))
828
- } finally {
829
- backgroundIndexRebuilds.delete(dbPath)
830
- debug.timeEnd("fts:rebuild:background")
831
- }
832
- }, 250)
833
- ;(timer as { unref?: () => void }).unref?.()
834
- }
835
-
836
- const SEARCH_INDEX_VERSION = "7"
837
- const DOCUMENT_EXTRACTOR_VERSION = "1"
838
-
839
- function searchIndexPath(sourcePath: string) {
840
- const parsed = path.parse(sourcePath)
841
- return path.join(parsed.dir, `${parsed.name}-telescope-search.db`)
842
- }
843
-
844
- function migrateSearchIndex(db: Database) {
845
- db.exec(`
846
- CREATE TABLE IF NOT EXISTS index_meta(
847
- key TEXT PRIMARY KEY,
848
- value TEXT NOT NULL
849
- );
850
- CREATE TABLE IF NOT EXISTS document(
851
- rowid INTEGER PRIMARY KEY,
852
- doc_id TEXT UNIQUE NOT NULL,
853
- part_id TEXT NOT NULL,
854
- message_id TEXT NOT NULL,
855
- session_id TEXT NOT NULL,
856
- session_title TEXT NOT NULL,
857
- directory TEXT NOT NULL,
858
- role TEXT NOT NULL,
859
- part_type TEXT NOT NULL,
860
- tool TEXT,
861
- time_created INTEGER NOT NULL,
862
- chunk_index INTEGER NOT NULL,
863
- text TEXT NOT NULL,
864
- source_hash TEXT NOT NULL,
865
- extractor_version TEXT NOT NULL,
866
- indexed_at INTEGER NOT NULL
867
- );
868
- CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
869
- id UNINDEXED,
870
- message_id UNINDEXED,
871
- session_id UNINDEXED,
872
- session_title,
873
- directory UNINDEXED,
874
- role UNINDEXED,
875
- part_type UNINDEXED,
876
- tool UNINDEXED,
877
- time_created UNINDEXED,
878
- text,
879
- tokenize='unicode61'
880
- );
881
- CREATE TABLE IF NOT EXISTS document_index(
882
- id TEXT PRIMARY KEY,
883
- message_id TEXT NOT NULL,
884
- session_id TEXT NOT NULL,
885
- session_title TEXT NOT NULL,
886
- directory TEXT NOT NULL,
887
- role TEXT NOT NULL,
888
- part_type TEXT NOT NULL,
889
- tool TEXT,
890
- time_created INTEGER NOT NULL,
891
- text TEXT NOT NULL
892
- );
893
- CREATE INDEX IF NOT EXISTS document_index_recent_idx
894
- ON document_index(directory, role, time_created DESC);
895
- CREATE INDEX IF NOT EXISTS document_index_recent_text_idx
896
- ON document_index(directory, role, part_type, time_created DESC);
897
- CREATE INDEX IF NOT EXISTS document_index_time_idx
898
- ON document_index(time_created DESC);
899
- `)
900
-
901
- const columns = db.query<{ name: string }, []>("PRAGMA table_info(document_fts)").all().map((column) => column.name)
902
- if (!columns.includes("part_type") || !columns.includes("tool")) {
903
- db.exec("DROP TABLE document_fts")
904
- db.exec(`
905
- CREATE VIRTUAL TABLE document_fts USING fts5(
906
- id UNINDEXED,
907
- message_id UNINDEXED,
908
- session_id UNINDEXED,
909
- session_title,
910
- directory UNINDEXED,
911
- role UNINDEXED,
912
- part_type UNINDEXED,
913
- tool UNINDEXED,
914
- time_created UNINDEXED,
915
- text,
916
- tokenize='unicode61'
917
- );
918
- `)
919
- }
920
-
921
- const indexColumns = db.query<{ name: string }, []>("PRAGMA table_info(document_index)").all().map((column) => column.name)
922
- if (!["part_type", "tool", "time_created", "text"].every((name) => indexColumns.includes(name))) {
923
- db.exec("DROP TABLE IF EXISTS document_index")
924
- db.exec(`
925
- CREATE TABLE document_index(
926
- id TEXT PRIMARY KEY,
927
- message_id TEXT NOT NULL,
928
- session_id TEXT NOT NULL,
929
- session_title TEXT NOT NULL,
930
- directory TEXT NOT NULL,
931
- role TEXT NOT NULL,
932
- part_type TEXT NOT NULL,
933
- tool TEXT,
934
- time_created INTEGER NOT NULL,
935
- text TEXT NOT NULL
936
- );
937
- `)
938
- }
939
- db.exec(`
940
- CREATE INDEX IF NOT EXISTS document_index_recent_idx
941
- ON document_index(directory, role, time_created DESC);
942
- CREATE INDEX IF NOT EXISTS document_index_recent_text_idx
943
- ON document_index(directory, role, part_type, time_created DESC);
944
- CREATE INDEX IF NOT EXISTS document_index_time_idx
945
- ON document_index(time_created DESC);
946
- `)
947
- }
948
-
949
- function sourceState(db: Database, sourcePath: string) {
950
- const stat = statSync(sourcePath)
951
- const dataVersion = db.query<{ data_version: number }, []>("PRAGMA data_version").get()?.data_version ?? 0
952
- return { dataVersion, mtimeMs: stat.mtimeMs }
953
- }
954
-
955
- function rebuildSearchIndex(source: Database, index: Database, sourcePath: string, state: { dataVersion: number; mtimeMs: number }) {
956
- debug.time("fts:rebuild")
957
- const rows = source.query<IndexSourceRow, []>(`
958
- SELECT p.id, p.message_id, p.session_id, s.title AS session_title, s.directory,
959
- json_extract(m.data, '$.role') AS role,
960
- json_extract(p.data, '$.type') AS part_type,
961
- json_extract(p.data, '$.tool') AS tool,
962
- p.time_created,
963
- p.data
964
- FROM part p
965
- JOIN message m ON m.id = p.message_id
966
- JOIN session s ON s.id = p.session_id
967
- WHERE (
968
- json_extract(p.data, '$.type') = 'text'
969
- OR (
970
- json_extract(p.data, '$.type') = 'tool'
971
- AND json_extract(p.data, '$.tool') IN ('apply_patch', 'edit', 'write')
972
- )
973
- )
974
- AND json_extract(m.data, '$.role') IN ('user', 'assistant')
975
- ORDER BY p.time_created DESC
976
- `).all()
977
- const now = Date.now()
978
- const config = parseSemanticConfig()
979
- const insertDoc = index.query<unknown, [string, string, string, string, string, string, string, string, string | null, number, number, string, string, string, number]>(`
980
- INSERT INTO document(doc_id, part_id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, chunk_index, text, source_hash, extractor_version, indexed_at)
981
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
982
- `)
983
- const insertFts = index.query<Row, [string, string, string, string, string, string, SearchResult["partType"], string | null, number, string]>(`
984
- INSERT INTO document_fts(id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, text)
985
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
986
- `)
987
- const insertIndex = index.query<Row, [string, string, string, string, string, string, SearchResult["partType"], string | null, number, string]>(`
988
- INSERT INTO document_index(id, message_id, session_id, session_title, directory, role, part_type, tool, time_created, text)
989
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
990
- `)
991
- index.exec("BEGIN IMMEDIATE")
992
- try {
993
- index.exec("DELETE FROM document")
994
- index.exec("DELETE FROM document_fts")
995
- index.exec("DELETE FROM document_index")
996
- for (const row of rows.flatMap(indexSourceRowToRows)) {
997
- const docID = `telescope:${row.session_id}:${row.message_id}:${row.id}:0`
998
- const sourceHash = hashPartData({ session_id: row.session_id, message_id: row.message_id, part_id: row.id })
999
- insertDoc.run(
1000
- docID,
1001
- row.id,
1002
- row.message_id,
1003
- row.session_id,
1004
- row.session_title ?? "Untitled session",
1005
- row.directory,
1006
- row.role,
1007
- row.part_type ?? "text",
1008
- row.tool ?? null,
1009
- row.time_created,
1010
- 0,
1011
- row.text,
1012
- sourceHash,
1013
- DOCUMENT_EXTRACTOR_VERSION,
1014
- now,
1015
- )
1016
- insertFts.run(
1017
- row.id,
1018
- row.message_id,
1019
- row.session_id,
1020
- row.session_title ?? "Untitled session",
1021
- row.directory,
1022
- row.role,
1023
- row.part_type ?? "text",
1024
- row.tool ?? null,
1025
- row.time_created,
1026
- row.text,
1027
- )
1028
- insertIndex.run(
1029
- row.id,
1030
- row.message_id,
1031
- row.session_id,
1032
- row.session_title ?? "Untitled session",
1033
- row.directory,
1034
- row.role,
1035
- row.part_type ?? "text",
1036
- row.tool ?? null,
1037
- row.time_created,
1038
- row.text,
1039
- )
1040
- }
1041
- setMeta(index, "source_path", sourcePath)
1042
- setMeta(index, "source_data_version", String(state.dataVersion))
1043
- setMeta(index, "source_mtime_ms", String(state.mtimeMs))
1044
- setMeta(index, "index_version", SEARCH_INDEX_VERSION)
1045
- setMeta(index, "schema_version", "2")
1046
- setMeta(index, "extractor_version", DOCUMENT_EXTRACTOR_VERSION)
1047
- setMeta(index, "ranking_version", "1")
1048
- setMeta(index, "embedding_base_url", config.embedBaseUrl)
1049
- setMeta(index, "document_prefix", config.documentPrefix)
1050
- setMeta(index, "query_prefix", config.queryPrefix)
1051
- if (config.embedModel) setMeta(index, "embedding_model", config.embedModel)
1052
- index.exec("COMMIT")
1053
- } catch (err) {
1054
- index.exec("ROLLBACK")
1055
- throw err
1056
- } finally {
1057
- debug.timeEnd("fts:rebuild")
1058
- }
1059
-
1060
- if (config.disableVector) {
1061
- setMeta(index, "vector_state", "disabled")
1062
- } else {
1063
- setupVectorTable(index, config, searchIndexPath(sourcePath))
1064
- }
1065
- }
1066
-
1067
- function hashPartData(value: { session_id: string; message_id: string; part_id: string }) {
1068
- return createHash("sha256").update(`${value.session_id}:${value.message_id}:${value.part_id}`).digest("hex")
1069
- }
1070
-
1071
- let customSQLiteConfigured = false
1072
-
1073
- function configureCustomSQLite() {
1074
- if (customSQLiteConfigured) return
1075
- const config = parseSemanticConfig()
1076
- if (config.disableVector) return
1077
- customSQLiteConfigured = true
1078
-
1079
- const candidates = [
1080
- config.sqliteLibPath,
1081
- process.platform === "darwin" ? "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib" : undefined,
1082
- process.platform === "darwin" ? "/usr/local/opt/sqlite/lib/libsqlite3.dylib" : undefined,
1083
- ].filter((item): item is string => Boolean(item))
1084
-
1085
- for (const candidate of candidates) {
1086
- if (existsSync(candidate)) {
1087
- try {
1088
- Database.setCustomSQLite(candidate)
1089
- debug.log("custom-sqlite:set", { path: candidate })
1090
- return
1091
- } catch (err) {
1092
- debug.log("custom-sqlite:error", { path: candidate, error: err instanceof Error ? err.message : String(err) })
1093
- }
1094
- }
1095
- }
1096
- }
1097
-
1098
- function importPackage(specifier: string) {
1099
- return new Function("specifier", "return import(specifier)")(specifier) as Promise<{ default?: { load?: (db: Database) => void } }>
1100
- }
1101
-
1102
- async function loadVecExtension(db: Database): Promise<boolean> {
1103
- try {
1104
- const sqliteVec = await importPackage("sqlite-vec").catch(() => undefined)
1105
- if (sqliteVec?.default?.load) {
1106
- sqliteVec.default.load(db)
1107
- debug.log("vector:extension:loaded", { source: "npm" })
1108
- return true
1109
- }
1110
- } catch {
1111
- debug.log("vector:extension:npm-failed")
1112
- }
1113
-
1114
- const config = parseSemanticConfig()
1115
- const explicitPath = config.sqliteVecExtension || process.env.OPENCODE_TELESCOPE_SQLITE_VEC_EXT
1116
- if (explicitPath && existsSync(explicitPath)) {
1117
- try {
1118
- db.loadExtension(explicitPath)
1119
- debug.log("vector:extension:loaded", { source: "path", path: explicitPath })
1120
- return true
1121
- } catch (err) {
1122
- debug.log("vector:extension:path-failed", { path: explicitPath, error: err instanceof Error ? err.message : String(err) })
1123
- }
1124
- }
1125
-
1126
- return false
1127
- }
1128
-
1129
- function setupVectorTable(index: Database, config: SemanticConfig, indexPath: string) {
1130
- setMeta(index, "vector_state", "stale")
1131
- rebuildVectorIndex(indexPath, config).catch((err: Error) => {
1132
- debug.log("vector:rebuild:error", err instanceof Error ? err.message : String(err))
1133
- })
1134
- }
1135
-
1136
- async function rebuildVectorIndex(indexPath: string, config: SemanticConfig) {
1137
- configureCustomSQLite()
1138
- const db = new Database(indexPath)
1139
- try {
1140
- const loaded = await loadVecExtension(db)
1141
- if (!loaded) {
1142
- setMeta(db, "vector_state", "unavailable")
1143
- return
1144
- }
1145
-
1146
- const client = new LlamaEmbeddingClient({
1147
- baseUrl: config.embedBaseUrl,
1148
- model: config.embedModel,
1149
- documentPrefix: config.documentPrefix,
1150
- queryPrefix: config.queryPrefix,
1151
- })
1152
- const healthy = await client.health()
1153
- if (!healthy) {
1154
- setMeta(db, "vector_state", "unavailable")
1155
- return
1156
- }
1157
-
1158
- const docs = db.query<{ rowid: number; text: string }, []>("SELECT rowid, text FROM document ORDER BY rowid ASC").all()
1159
- if (!docs.length) {
1160
- setMeta(db, "vector_state", "enabled")
1161
- return
1162
- }
1163
-
1164
- debug.log("vector:embed:start", { count: docs.length })
1165
- const embeddings = await client.embedDocuments(docs.map((d) => d.text))
1166
- const dims = embeddings[0]?.length
1167
- if (!dims) {
1168
- setMeta(db, "vector_state", "unavailable")
1169
- return
1170
- }
1171
-
1172
- db.exec("DROP TABLE IF EXISTS document_vec")
1173
- db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS document_vec USING vec0(embedding float[${dims}])`)
1174
- debug.log("vector:table:recreated", { dimensions: dims })
1175
-
1176
- const insert = db.prepare<unknown, [number, Float32Array]>("INSERT INTO document_vec(rowid, embedding) VALUES (?, vec_f32(?))")
1177
- const transaction = db.transaction(() => {
1178
- for (const [i, doc] of docs.entries()) {
1179
- insert.run(doc.rowid, embeddings[i])
1180
- }
1181
- })
1182
- transaction()
1183
-
1184
- setMeta(db, "vector_state", "enabled")
1185
- setMeta(db, "embedding_dimensions", String(dims))
1186
- if (config.embedModel) setMeta(db, "embedding_model", config.embedModel)
1187
- debug.log("vector:rebuild:done", { vectors: docs.length, dimensions: dims })
1188
- } catch (err) {
1189
- setMeta(db, "vector_state", "unavailable")
1190
- debug.log("vector:rebuild:error", err instanceof Error ? err.message : String(err))
1191
- } finally {
1192
- db.close()
1193
- }
1194
- }
1195
-
1196
- export type ScoredRow = Row & {
1197
- score: number
1198
- keywordScore: number
1199
- vectorScore: number
1200
- }
1201
-
1202
- function searchVector(index: Database, embedding: Float32Array, limit: number): Row[] {
1203
- const count = index.query<{ count: number }, []>("SELECT COUNT(*) as count FROM document_vec").get()?.count ?? 0
1204
- if (!count) return []
1205
- const k = Math.min(count, Math.max(limit * 4, 200))
1206
- return index.query<Row, [Float32Array, number]>(`
1207
- SELECT d.part_id AS id, d.message_id, d.session_id, d.session_title, d.directory, d.role,
1208
- d.part_type, d.tool, CAST(d.time_created AS INTEGER) AS time_created, d.text
1209
- FROM document_vec v
1210
- JOIN document d ON d.rowid = v.rowid
1211
- WHERE v.embedding MATCH vec_f32(?) AND k = ?
1212
- ORDER BY v.distance
1213
- `).all(embedding, k)
1214
- }
1215
-
1216
- export function hybridBlend(keyword: Row[], vector: Row[], alpha: number): ScoredRow[] {
1217
- const merged = new Map<string, ScoredRow>()
1218
- const defaultScore = 1 / Math.max(keyword.length, 1)
1219
-
1220
- for (const [i, row] of keyword.entries()) {
1221
- merged.set(row.id, {
1222
- ...row,
1223
- score: 0,
1224
- keywordScore: keyword.length > 1 ? 1 - i / (keyword.length - 1) : 1,
1225
- vectorScore: 0,
1226
- })
1227
- }
1228
-
1229
- for (const [i, row] of vector.entries()) {
1230
- const existing = merged.get(row.id)
1231
- const vectorScore = vector.length > 1 ? 1 - i / (vector.length - 1) : 1
1232
- if (existing) {
1233
- existing.vectorScore = vectorScore
1234
- } else {
1235
- merged.set(row.id, {
1236
- ...row,
1237
- score: 0,
1238
- keywordScore: defaultScore,
1239
- vectorScore,
1240
- })
1241
- }
1242
- }
1243
-
1244
- const keywordValues = [...merged.values()].map((r) => r.keywordScore)
1245
- const vectorValues = [...merged.values()].map((r) => r.vectorScore)
1246
- const kwMin = Math.min(...keywordValues)
1247
- const kwMax = Math.max(...keywordValues)
1248
- const vecMin = Math.min(...vectorValues)
1249
- const vecMax = Math.max(...vectorValues)
1250
-
1251
- const normalized = [...merged.values()].map((r) => {
1252
- const kn = kwMax === kwMin ? 0.5 : (r.keywordScore - kwMin) / (kwMax - kwMin)
1253
- const vn = vecMax === vecMin ? 0.5 : (r.vectorScore - vecMin) / (vecMax - vecMin)
1254
- return {
1255
- ...r,
1256
- keywordScore: kn,
1257
- vectorScore: vn,
1258
- score: (1 - alpha) * kn + alpha * vn,
1259
- }
1260
- })
1261
-
1262
- return normalized.sort((a, b) => b.score - a.score)
1263
- }
1264
-
1265
- export type HybridSearchOptions = {
1266
- limit?: number
1267
- offset?: number
1268
- directory?: string
1269
- role?: SearchRole
1270
- dbPath?: string
1271
- }
1272
-
1273
- export async function performSearch(query: string, options?: HybridSearchOptions): Promise<SearchResult[]> {
1274
- if (!query.trim()) return searchSessionMessages(query, options)
1275
- const config = parseSemanticConfig()
1276
- if (!config.disableVector) {
1277
- try {
1278
- return await semanticSearchSessionMessages(query, options)
1279
- } catch {
1280
- debug.log("query:hybrid:fallback", { message: "hybrid search failed, falling back to keyword" })
1281
- }
1282
- }
1283
- return searchSessionMessages(query, options)
1284
- }
1285
-
1286
- export async function semanticSearchSessionMessages(query: string, options?: HybridSearchOptions): Promise<SearchResult[]> {
1287
- const term = query.trim()
1288
- if (!term) return []
1289
- if (options?.dbPath === ":memory:") return []
1290
- const dbPath = options?.dbPath ?? resolveDatabasePath()
1291
- const db = getDb(dbPath)
1292
- const limit = options?.limit ?? 80
1293
- const dir = options?.directory
1294
- const role = options?.role
1295
-
1296
- const index = ensureSearchIndex(db, dbPath)
1297
- if (!index) return []
1298
-
1299
- const keyword = indexedTextRows(db, dbPath, limit, term, dir, options?.offset, role) ?? []
1300
- const config = parseSemanticConfig()
1301
-
1302
- let vector: Row[] = []
1303
- if (!config.disableVector) {
1304
- const vecState = getMeta(index, "vector_state")
1305
- if (vecState === "enabled") {
1306
- try {
1307
- const client = new LlamaEmbeddingClient({
1308
- baseUrl: config.embedBaseUrl,
1309
- model: config.embedModel,
1310
- documentPrefix: config.documentPrefix,
1311
- queryPrefix: config.queryPrefix,
1312
- })
1313
- const embedding = await client.embedQuery(term)
1314
- vector = searchVector(index, embedding, limit)
1315
- debug.log("query:vector:results", { count: vector.length })
1316
- } catch (err) {
1317
- debug.log("query:vector:error", err instanceof Error ? err.message : String(err))
1318
- }
1319
- }
1320
- }
1321
-
1322
- const merged = keyword.length || vector.length
1323
- ? hybridBlend(keyword, vector, config.hybridAlpha)
1324
- : []
1325
-
1326
- const results: SearchResult[] = []
1327
- const seen = new Set<string>()
1328
- for (const row of merged) {
1329
- const result = rowToSearchResult(row, term) ?? rowToVectorResult(row)
1330
- if (result) {
1331
- seen.add(row.id)
1332
- results.push(result)
1333
- }
1334
- }
1335
-
1336
- for (const row of vector) {
1337
- if (!seen.has(row.id)) {
1338
- const result = rowToSearchResult(row, term) ?? rowToVectorResult(row)
1339
- if (result) results.push(result)
1340
- }
1341
- }
1342
-
1343
- return results
1344
- }
1345
-
1346
- export function rowToVectorResult(row: Row): SearchResult | undefined {
1347
- const text = row.text.trim()
1348
- if (!text) return
1349
- const excerpt = text.slice(0, 200)
1350
- return {
1351
- id: row.id,
1352
- messageID: row.message_id,
1353
- sessionID: row.session_id,
1354
- sessionTitle: row.session_title || "Untitled session",
1355
- directory: row.directory,
1356
- role: row.role,
1357
- partType: row.part_type ?? "text",
1358
- tool: row.tool ?? undefined,
1359
- timeCreated: row.time_created,
1360
- snippet: excerpt,
1361
- matchStart: -1,
1362
- matchEnd: -1,
1363
- before: "",
1364
- match: "",
1365
- after: excerpt,
1366
- excerpt,
1367
- previewBefore: text.slice(0, Math.min(text.length, 1400)),
1368
- previewMatch: "",
1369
- previewAfter: "",
1370
- previewMode: "markdown" as const,
1371
- previewHighlight: false,
1372
- text,
1373
- }
1374
- }
1375
-
1376
- function ftsQuery(query: string) {
1377
- const tokens = query.trim().split(/\s+/)
1378
- .map((token) => token.replace(/["*^:()]/g, " ").trim())
1379
- .filter(Boolean)
1380
- if (!tokens.length) return ""
1381
- return tokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(" AND ")
1382
- }
1383
-
1384
- function getMeta(db: Database, key: string) {
1385
- return db.query<{ value: string }, [string]>("SELECT value FROM index_meta WHERE key = ?").get(key)?.value
1386
- }
1387
-
1388
- function setMeta(db: Database, key: string, value: string) {
1389
- db.query<unknown, [string, string]>(`
1390
- INSERT INTO index_meta(key, value) VALUES (?, ?)
1391
- ON CONFLICT(key) DO UPDATE SET value = excluded.value
1392
- `).run(key, value)
1393
- }
1394
-
1395
- function indexSourceRowToRows(row: IndexSourceRow): Row[] {
1396
- const text = extractIndexText(row.data)
1397
- if (!text) return []
1398
- return [{ ...row, text }]
1399
- }
1400
-
1401
- function extractIndexText(data: string) {
1402
- try {
1403
- const value = JSON.parse(data) as unknown
1404
- if (!value || typeof value !== "object" || Array.isArray(value)) return ""
1405
- const record = value as Record<string, unknown>
1406
- if (record.type === "text") return typeof record.text === "string" ? record.text.trim() : ""
1407
- if (record.type !== "tool") return ""
1408
- return extractToolIndexText(record).replace(/\s+/g, " ").trim()
1409
- } catch {
1410
- return ""
1411
- }
1412
- }
1413
-
1414
- function extractToolIndexText(part: Record<string, unknown>) {
1415
- const tool = typeof part.tool === "string" ? part.tool : ""
1416
- const state = recordValue(part.state)
1417
- const input = recordValue(state?.input)
1418
- const metadata = recordValue(state?.metadata)
1419
-
1420
- if (tool === "apply_patch") {
1421
- const files = Array.isArray(metadata?.files) ? metadata.files : []
1422
- const renderedPatches = files.map(applyPatchFileIndexText).filter(Boolean).join("\n")
1423
- const patchText = stringValue(input?.patchText)
1424
- return [renderedPatches, patchText].filter(Boolean).join("\n")
1425
- }
1426
-
1427
- if (tool === "edit") {
1428
- const filediff = recordValue(metadata?.filediff)
1429
- return [
1430
- stringValue(input?.filePath),
1431
- stringValue(metadata?.diff),
1432
- stringValue(filediff?.patch),
1433
- stringValue(input?.oldString),
1434
- stringValue(input?.newString),
1435
- ].filter(Boolean).join("\n")
1436
- }
1437
-
1438
- if (tool === "write") {
1439
- return [stringValue(input?.filePath), stringValue(input?.content)].filter(Boolean).join("\n")
1440
- }
1441
-
1442
- return ""
1443
- }
1444
-
1445
- function applyPatchFileIndexText(value: unknown) {
1446
- const file = recordValue(value)
1447
- if (!file) return ""
1448
- return [
1449
- stringValue(file.filePath),
1450
- stringValue(file.relativePath),
1451
- stringValue(file.patch),
1452
- ].filter(Boolean).join("\n")
1453
- }
1454
-
1455
- function recordValue(value: unknown): Record<string, unknown> | undefined {
1456
- if (!value || typeof value !== "object" || Array.isArray(value)) return
1457
- return value as Record<string, unknown>
1458
- }
1459
-
1460
- function stringValue(value: unknown) {
1461
- return typeof value === "string" ? value : undefined
1462
- }
1463
-
1464
- const tableCache = new Map<string, boolean>()
1465
- function tableExists(db: Database, name: string) {
1466
- if (tableCache.has(name)) return tableCache.get(name)!
1467
- const exists = Boolean(db.query<{ name: string }, [string]>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name))
1468
- tableCache.set(name, exists)
1469
- return exists
1470
- }
1471
-
1472
- function defaultDataDir() {
1473
- if (process.env.XDG_DATA_HOME) return path.join(process.env.XDG_DATA_HOME, "opencode")
1474
- return path.join(homedir(), ".local", "share", "opencode")
1475
- }
1476
-
1477
- function candidateDataDirs() {
1478
- return [
1479
- defaultDataDir(),
1480
- path.join(homedir(), ".local", "share", "opencode"),
1481
- process.platform === "darwin" ? path.join(homedir(), "Library", "Application Support", "opencode") : undefined,
1482
- process.platform === "win32" ? path.join(process.env.APPDATA ?? path.join(homedir(), "AppData", "Roaming"), "opencode") : undefined,
1483
- ].filter((item, index, list): item is string => Boolean(item) && list.indexOf(item) === index)
1484
- }
1485
-
1486
- function candidateDatabasePaths(names: string[]) {
1487
- return candidateDataDirs().flatMap((dir) => names.map((name) => path.join(dir, name)))
1488
- }
1489
-
1490
- function requireExistingDatabase(names: string[]) {
1491
- return candidateDatabasePaths(names).find(existsSync) ?? candidateDatabasePaths(names)[0]!
1492
- }
1493
-
1494
- function extractFromValue(value: unknown): string {
1495
- if (typeof value === "string") return value
1496
- if (typeof value === "number" || typeof value === "boolean") return String(value)
1497
- if (!value) return ""
1498
- if (Array.isArray(value)) return value.map(extractFromValue).filter(Boolean).join("\n")
1499
- if (typeof value !== "object") return ""
1500
- return Object.entries(value)
1501
- .filter(([key]) => !["id", "sessionID", "messageID", "time", "timeCreated", "timeUpdated", "tokens", "cost"].includes(key))
1502
- .map(([, item]) => extractFromValue(item))
1503
- .filter(Boolean)
1504
- .join("\n")
1505
- }
1506
-
1507
- function truncate(value: string, length: number) {
1508
- if (value.length <= length) return value
1509
- return `${value.slice(0, length - 3)}...`
1510
- }
1
+ // Re-exported types
2
+ export type {
3
+ SearchResult,
4
+ SearchRole,
5
+ ConversationPreviewPart,
6
+ ConversationPreviewPage,
7
+ ConversationPreviewCursor,
8
+ ToolState,
9
+ SemanticConfig,
10
+ VectorState,
11
+ DocumentRow,
12
+ ScoredRow,
13
+ HybridSearchOptions,
14
+ } from "./search/types.ts"
15
+
16
+ // Re-exported query functions
17
+ export {
18
+ searchSessionMessages,
19
+ recentSessionMessages,
20
+ loadConversationAround,
21
+ loadConversationBefore,
22
+ loadConversationAfter,
23
+ performSearch,
24
+ semanticSearchSessionMessages,
25
+ parseSemanticConfig,
26
+ } from "./search/queries.ts"
27
+
28
+ // Re-exported text/snippet functions
29
+ export {
30
+ rowToSearchResult,
31
+ rowToVectorResult,
32
+ makeSnippet,
33
+ extractSearchText,
34
+ ftsQuery,
35
+ expandQuery,
36
+ } from "./search/text.ts"
37
+
38
+ // Re-exported vector/blend functions
39
+ export {
40
+ hybridBlend,
41
+ } from "./search/vector.ts"
42
+
43
+ // Re-exported path utilities
44
+ export {
45
+ resolveDatabasePath,
46
+ searchIndexPath,
47
+ } from "./search/db-path.ts"