@saadnvd1/agent-os 0.3.673 → 0.3.680

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +2 -1
  2. package/app/api/bus/read/route.ts +34 -0
  3. package/app/api/search/conversations/context/route.ts +26 -0
  4. package/app/api/search/conversations/route.test.ts +139 -0
  5. package/app/api/search/conversations/route.ts +25 -0
  6. package/app/globals.css +25 -0
  7. package/components/Bookmarks/BookmarksDialog.tsx +2 -8
  8. package/components/Chat/useGoToItem.ts +11 -0
  9. package/components/CommandPalette/CommandPalette.tsx +23 -156
  10. package/components/CommandPalette/CommandsBody.tsx +216 -0
  11. package/components/CommandPalette/conversations/ConversationSearch.tsx +246 -0
  12. package/components/CommandPalette/conversations/EmptyState.tsx +95 -0
  13. package/components/CommandPalette/conversations/Preview.tsx +129 -0
  14. package/components/CommandPalette/conversations/ResultList.tsx +91 -0
  15. package/components/CommandPalette/conversations/Snippet.tsx +24 -0
  16. package/components/CommandPalette/conversations/format.ts +30 -0
  17. package/components/CommandPalette/conversations/recent.test.ts +20 -0
  18. package/components/CommandPalette/conversations/recent.ts +35 -0
  19. package/components/CommandPalette/conversations/useOpenHit.ts +30 -0
  20. package/data/search/index.ts +65 -0
  21. package/data/search/keys.ts +7 -0
  22. package/hooks/useDebouncedValue.ts +11 -0
  23. package/lib/agents/brief.ts +8 -0
  24. package/lib/bus/condense.ts +242 -0
  25. package/lib/bus/read-scope.ts +109 -0
  26. package/lib/bus/read.ts +181 -0
  27. package/lib/chat/worker/host.ts +3 -1
  28. package/lib/db/migrations/20261019T1500_conversation_search.ts +12 -0
  29. package/lib/demo-reseed.ts +4 -0
  30. package/lib/health/facts.ts +13 -3
  31. package/lib/health/spin-tick.ts +293 -0
  32. package/lib/health/spin.ts +158 -0
  33. package/lib/health/tick.ts +10 -2
  34. package/lib/health/types.ts +2 -0
  35. package/lib/orchestrator/read.ts +18 -4
  36. package/lib/palette/registry.ts +4 -0
  37. package/lib/resurrect/index.ts +2 -3
  38. package/lib/search/conversations/backfill.ts +117 -0
  39. package/lib/search/conversations/buildMatchQuery.ts +31 -0
  40. package/lib/search/conversations/index.ts +9 -0
  41. package/lib/search/conversations/messageContext.ts +45 -0
  42. package/lib/search/conversations/schema.ts +71 -0
  43. package/lib/search/conversations/searchConversations.ts +151 -0
  44. package/lib/search/conversations/types.ts +45 -0
  45. package/lib/sidebar/row-actions.ts +19 -8
  46. package/lib/sidebar/row-menu.ts +1 -1
  47. package/lib/tasks/restorable.ts +16 -0
  48. package/package.json +1 -1
  49. package/scripts/bench-conversation-search.ts +73 -0
  50. package/server.ts +3 -0
  51. package/stores/chatJump.ts +10 -0
  52. package/stores/palette.ts +22 -4
@@ -0,0 +1,158 @@
1
+ /**
2
+ * bg-spin: a background job whose shell burns a core with nothing printed is a
3
+ * wait loop that can never exit (`until grep -q "" /dev/null; do :; done`).
4
+ * The pure half: deciding it, picking what to stop, and the words about it.
5
+ * No I/O here; spin-tick.ts reads the processes and stops them.
6
+ *
7
+ * Only the CPU of shells in the job's tree is counted. A loop's work happens
8
+ * in the shell itself (builtins, or children reaped into its total), while a
9
+ * build's lives in node, cc or cargo, so a silent heavy build never reads as
10
+ * spinning.
11
+ */
12
+
13
+ import path from "path";
14
+ import { WAIT_ADVICE } from "../agents/brief";
15
+ import type { ProcRow } from "../process-table";
16
+ import { MIN } from "./types";
17
+
18
+ // Share of one core, every interval of the window.
19
+ export const SPIN_CORES = 0.8;
20
+ // How long it has to hold, with nothing new in the output.
21
+ export const SPIN_WINDOW_MS = 4 * MIN;
22
+ // Younger jobs aren't sampled.
23
+ export const SPIN_MIN_AGE_MS = SPIN_WINDOW_MS;
24
+ // A longer gap between samples (missed ticks, a sleeping laptop) starts over.
25
+ export const SPIN_MAX_GAP_MS = 3 * MIN;
26
+ // SIGTERM, then SIGKILL for whatever is still there.
27
+ export const KILL_GRACE_MS = 3000;
28
+ // Samples older than this are no use to the decision.
29
+ export const SPIN_KEEP_MS = SPIN_WINDOW_MS + 2 * MIN;
30
+
31
+ const SHELLS = new Set(["sh", "bash", "zsh", "dash", "fish", "ksh"]);
32
+ const AGENTS = new Set(["claude"]);
33
+ // Never part of a job's tree to stop.
34
+ const NEVER = new Set(["tmux", "claude"]);
35
+
36
+ export const baseOf = (comm: string): string =>
37
+ path.basename(comm).replace(/^-/, "");
38
+
39
+ export interface SpinSample {
40
+ at: number;
41
+ // Cumulative CPU seconds of the tree's shells.
42
+ cpuSec: number;
43
+ // The output file's last write, if it has one.
44
+ outputAt: number | null;
45
+ }
46
+
47
+ export type SpinClass =
48
+ | "warming"
49
+ | "sleeping"
50
+ | "busy-with-output"
51
+ | "spinning";
52
+
53
+ export function classifySpin(samples: SpinSample[]): SpinClass {
54
+ if (samples.length < 2) return "warming";
55
+ // The unbroken run of samples ending now.
56
+ let from = samples.length - 1;
57
+ while (from > 0 && samples[from].at - samples[from - 1].at <= SPIN_MAX_GAP_MS)
58
+ from--;
59
+ const run = samples.slice(from);
60
+ const last = run[run.length - 1];
61
+ // Start at the newest sample at least a window old.
62
+ let start = -1;
63
+ for (let i = 0; i < run.length; i++)
64
+ if (last.at - run[i].at >= SPIN_WINDOW_MS) start = i;
65
+ if (start < 0) return "warming";
66
+ const win = run.slice(start);
67
+ for (let i = 1; i < win.length; i++) {
68
+ const wall = (win[i].at - win[i - 1].at) / 1000;
69
+ const cpu = Math.max(0, win[i].cpuSec - win[i - 1].cpuSec);
70
+ if (wall <= 0 || cpu / wall < SPIN_CORES) return "sleeping";
71
+ }
72
+ const wrote = win.some((s) => s.outputAt !== null && s.outputAt > win[0].at);
73
+ return wrote ? "busy-with-output" : "spinning";
74
+ }
75
+
76
+ // Of the processes holding a task's output file open, the job's root: the
77
+ // one whose parent doesn't hold it too.
78
+ export function pickRoot(holders: number[], table: ProcRow[]): number | null {
79
+ const held = new Set(holders);
80
+ const roots = table.filter((r) => held.has(r.pid) && !held.has(r.ppid));
81
+ return roots.length === 1 ? roots[0].pid : null;
82
+ }
83
+
84
+ export function jobTree(root: number, table: ProcRow[]): ProcRow[] {
85
+ const kids = new Map<number, ProcRow[]>();
86
+ for (const r of table) {
87
+ const list = kids.get(r.ppid);
88
+ if (list) list.push(r);
89
+ else kids.set(r.ppid, [r]);
90
+ }
91
+ const byPid = new Map(table.map((r) => [r.pid, r]));
92
+ const out: ProcRow[] = [];
93
+ const seen = new Set<number>();
94
+ const stack = [byPid.get(root)].filter((r): r is ProcRow => !!r);
95
+ while (stack.length) {
96
+ const at = stack.shift() as ProcRow;
97
+ if (seen.has(at.pid)) continue;
98
+ seen.add(at.pid);
99
+ out.push(at);
100
+ stack.push(...(kids.get(at.pid) ?? []));
101
+ }
102
+ return out;
103
+ }
104
+
105
+ export const shellPids = (tree: ProcRow[]): number[] =>
106
+ tree.filter((r) => SHELLS.has(baseOf(r.comm))).map((r) => r.pid);
107
+
108
+ // Whether this tree is a background job of an agent and nothing else: a
109
+ // shell directly under the agent, with nothing of AgentOS, tmux or the agent
110
+ // inside it.
111
+ export function safeToKill(
112
+ root: number,
113
+ table: ProcRow[],
114
+ protectedPids: number[] = [process.pid, process.ppid]
115
+ ): boolean {
116
+ const head = table.find((r) => r.pid === root);
117
+ if (!head || root <= 1) return false;
118
+ if (!SHELLS.has(baseOf(head.comm))) return false;
119
+ const parent = table.find((r) => r.pid === head.ppid);
120
+ if (!parent || !AGENTS.has(baseOf(parent.comm))) return false;
121
+ return !jobTree(root, table).some(
122
+ (r) =>
123
+ protectedPids.includes(r.pid) ||
124
+ r.pid === parent.pid ||
125
+ NEVER.has(baseOf(r.comm))
126
+ );
127
+ }
128
+
129
+ // The PIDs to signal: the root first, so a loop can't respawn children, then
130
+ // everything under it.
131
+ export const killPlan = (root: number, table: ProcRow[]): number[] =>
132
+ jobTree(root, table).map((r) => r.pid);
133
+
134
+ // ps's TIME column: [[D-]H:]M:SS[.cc] to seconds.
135
+ export function parseCpuTime(text: string): number | null {
136
+ const m = /^\s*(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)\s*$/.exec(text);
137
+ if (!m) return null;
138
+ return (
139
+ Number(m[1] ?? 0) * 86400 +
140
+ Number(m[2] ?? 0) * 3600 +
141
+ Number(m[3]) * 60 +
142
+ Number(m[4])
143
+ );
144
+ }
145
+
146
+ export function spinNote(description: string, pid: number): string {
147
+ const mins = Math.round(SPIN_WINDOW_MS / MIN);
148
+ return `AgentOS stopped your background job "${description}" (PID ${pid}): its shell used about a core for ${mins}+ minutes with no output, which is a wait loop that can't exit. ${WAIT_ADVICE}`;
149
+ }
150
+
151
+ export function spinEventLine(
152
+ sessionName: string,
153
+ description: string,
154
+ pid: number
155
+ ): string {
156
+ const mins = Math.round(SPIN_WINDOW_MS / MIN);
157
+ return `Stopped background job "${description}" (PID ${pid}) in ${sessionName}: a shell wait loop holding a core for ${mins}+ minutes with no output. The task was told.`;
158
+ }
@@ -11,6 +11,7 @@ import { healthConfig, type HealthConfig } from "./config";
11
11
  import { runDetectors } from "./detectors";
12
12
  import { gatherFacts, liveBackgroundJobs } from "./facts";
13
13
  import { envTick, realEnv, type EnvDeps } from "./env/envTick";
14
+ import { spinTick } from "./spin-tick";
14
15
  import { addNote } from "./notes";
15
16
  import { closeResolvedPieces } from "./pieces";
16
17
  import { deliver, deliverClosed, type RouteDeps, realRoute } from "./route";
@@ -43,9 +44,10 @@ export const MAX_WAKES = 2;
43
44
  // A wake that never came back (a restart cut it off) is given up on.
44
45
  export const WAKE_GIVE_UP_MS = 3 * MIN;
45
46
 
46
- // The checks the tick evaluates: piece-dup is raised where a piece starts.
47
+ // The checks the tick evaluates: piece-dup is raised where a piece starts,
48
+ // bg-spin by the process step (spin-tick.ts).
47
49
  export const TICK_CHECKS: readonly CheckId[] = CHECKS.filter(
48
- (c) => c !== "piece-dup"
50
+ (c) => c !== "piece-dup" && c !== "bg-spin"
49
51
  );
50
52
 
51
53
  export interface TickDeps {
@@ -54,6 +56,8 @@ export interface TickDeps {
54
56
  watch: WatchDeps;
55
57
  route: RouteDeps;
56
58
  env: EnvDeps;
59
+ // Stops background jobs that spin (spin-tick.ts).
60
+ spin: (now: number) => Promise<void>;
57
61
  wake: typeof wake;
58
62
  }
59
63
 
@@ -63,6 +67,7 @@ export const realTick: TickDeps = {
63
67
  watch: { run: runClaude, usagePct: currentUsagePct },
64
68
  route: realRoute,
65
69
  env: realEnv,
70
+ spin: spinTick,
66
71
  wake,
67
72
  };
68
73
 
@@ -145,6 +150,9 @@ export function healthTick(
145
150
  closeResolvedPieces(now);
146
151
  leaveHeartbeat(now, liveBackgroundJobs());
147
152
  giveUpStale(now, deps);
153
+ void deps
154
+ .spin(now)
155
+ .catch((error) => console.error("[health] spin check:", error));
148
156
  void envTick(now, c, deps.env).catch((error) =>
149
157
  console.error("[health] environment failures:", error)
150
158
  );
@@ -14,6 +14,7 @@ export const CHECKS = [
14
14
  "load-red",
15
15
  "queue-held",
16
16
  "bg-quiet",
17
+ "bg-spin",
17
18
  "piece-dup",
18
19
  "restarts",
19
20
  "ci-slow",
@@ -26,6 +27,7 @@ export const AREA_OF: Record<CheckId, Area> = {
26
27
  "load-red": "scheduler",
27
28
  "queue-held": "scheduler",
28
29
  "bg-quiet": "tasks",
30
+ "bg-spin": "tasks",
29
31
  "piece-dup": "pipeline",
30
32
  restarts: "pipeline",
31
33
  "ci-slow": "pipeline",
@@ -10,6 +10,7 @@ import { hostLink } from "../hosts/remote-api";
10
10
  import { peerPane } from "../hosts/peer-sessions";
11
11
  import { workspaceSessions } from "./facts";
12
12
  import { untrusted } from "./untrusted";
13
+ import { grepText } from "../bus/condense";
13
14
  import { heldChatTail, isHeldRemote } from "../tasks/contract/held-calls";
14
15
  import {
15
16
  findExternalSession,
@@ -55,7 +56,10 @@ export function findWorkspaceSession(
55
56
  // A session AgentOS didn't start: its screen, read the way the sidebar
56
57
  // reads one (a linked machine's only through its AgentOS, which doesn't
57
58
  // serve a screen it didn't start).
58
- async function readExternal(s: ExternalSession, n: number): Promise<string> {
59
+ export async function readExternal(
60
+ s: ExternalSession,
61
+ n: number
62
+ ): Promise<string> {
59
63
  const head = `${s.ref} (external terminal on ${s.host}, not started by AgentOS)`;
60
64
  if (hostLink(s.hostId))
61
65
  return `${head}: its screen is on that machine and isn't served to this one.`;
@@ -127,6 +131,16 @@ export async function readSession(
127
131
  if (!external) throw error;
128
132
  return readExternal(external, n);
129
133
  }
134
+ return readResolved(s, n);
135
+ }
136
+
137
+ // One session's recent conversation or screen, whichever it has.
138
+ export async function readResolved(
139
+ s: Session,
140
+ n: number,
141
+ grep?: string
142
+ ): Promise<string> {
143
+ const cut = (text: string) => tail(grep ? grepText(text, grep) : text, n);
130
144
  const link = hostLink(s.host_id);
131
145
  if (s.view === "chat" && link) {
132
146
  // Its orchestrator's task is read through the machine that holds it.
@@ -137,12 +151,12 @@ export async function readSession(
137
151
  }
138
152
  if (s.view === "chat") {
139
153
  const text = chatText(lastItems(s.id, n * 2));
140
- const body = text ? untrusted(s.name, tail(text, n)) : "(no messages yet)";
154
+ const body = text ? untrusted(s.name, cut(text)) : "(no messages yet)";
141
155
  return `${s.name} (chat), last ${n} lines:\n${body}`;
142
156
  }
143
157
  if (link) {
144
158
  const lines = await peerPane(link, s.id);
145
- const body = lines ? tail(lines.join("\n"), n) : "";
159
+ const body = lines ? cut(lines.join("\n")) : "";
146
160
  return `${s.name} (terminal on ${link.hostName}), last ${n} lines:\n${body ? untrusted(s.name, body) : "(couldn't be read)"}`;
147
161
  }
148
162
  await statusDetector.refreshCache();
@@ -152,6 +166,6 @@ export async function readSession(
152
166
  s.host_id || "local",
153
167
  `tmux capture-pane -t ${shellQuote(`=${s.tmux_name}:`)} -p -J -S -${n}`
154
168
  );
155
- const body = tail(pane, n);
169
+ const body = cut(pane);
156
170
  return `${s.name} (terminal), last ${n} lines:\n${body ? untrusted(s.name, body) : "(empty)"}`;
157
171
  }
@@ -12,6 +12,10 @@ export interface PaletteCommand {
12
12
  keywords?: string[];
13
13
  // Shown beside it, e.g. ⌘K.
14
14
  hint?: string;
15
+ // A line under the title (what a conversation hit said).
16
+ detail?: string;
17
+ // Running it leaves the palette open (it changes the palette itself).
18
+ keepOpen?: boolean;
15
19
  icon?: unknown;
16
20
  run: () => void;
17
21
  }
@@ -46,6 +46,7 @@ import { planFromTranscript } from "./plan";
46
46
  import { dropSnapshot, restoreWorktree, type RestoredAs } from "./snapshot";
47
47
  import { restoreSwept, sweptNote } from "./swept";
48
48
  import { forgetSwept } from "../worktree-sweep/store";
49
+ import { isRestorableTask } from "../tasks/restorable";
49
50
  import { setPr, setTask } from "../db/session-parts";
50
51
 
51
52
  export interface ResurrectOptions {
@@ -66,8 +67,6 @@ export interface ResurrectOutcome {
66
67
  const row = (id: string) =>
67
68
  queries.getSession(db).get(id) as Session | undefined;
68
69
 
69
- const ENDED_TASK = new Set(["dropped", "done"]);
70
-
71
70
  // Restores in flight here, so two clicks can't both build a worktree.
72
71
  const g = globalThis as unknown as { __agentosResurrecting?: Set<string> };
73
72
  const inFlight = (g.__agentosResurrecting ??= new Set());
@@ -113,7 +112,7 @@ export async function resurrectSession(
113
112
  throw new Error(`${s.name} runs on ${peer.hostName}: restore it there.`);
114
113
  if (isMirror(s)) return onItsMachine(s);
115
114
 
116
- const endedTask = !!s.task_status && ENDED_TASK.has(s.task_status);
115
+ const endedTask = isRestorableTask(s.task_status);
117
116
  if (s.task_status === "merged")
118
117
  throw new Error(
119
118
  `${s.name} was merged: its work is in the base branch. Start a new task to continue it.`
@@ -0,0 +1,117 @@
1
+ import type Database from "better-sqlite3";
2
+ import { getDb } from "@/lib/db";
3
+ import { SCHEMA_VERSION, SEARCHED_KINDS } from "./schema";
4
+
5
+ const BATCH = 500;
6
+
7
+ interface State {
8
+ version: number;
9
+ cursor: number;
10
+ backfill_to: number;
11
+ done: number;
12
+ }
13
+
14
+ const stateOf = (db: Database.Database): State | undefined =>
15
+ db.prepare(`SELECT * FROM chat_search_state WHERE id = 1`).get() as
16
+ | State
17
+ | undefined;
18
+
19
+ // Starts the index over: the stored version is not this build's.
20
+ function reset(db: Database.Database): void {
21
+ db.transaction(() => {
22
+ db.exec(`DELETE FROM chat_search; DELETE FROM chat_search_keys`);
23
+ db.prepare(
24
+ `UPDATE chat_search_state SET version = ?, cursor = 0, done = 0,
25
+ backfill_to = (SELECT COALESCE(MAX(rowid), 0) FROM chat_items)
26
+ WHERE id = 1`
27
+ ).run(SCHEMA_VERSION);
28
+ })();
29
+ }
30
+
31
+ // Indexes one batch of the chats that were already there when the index was
32
+ // made, in rowid order, and records where it got to in the same transaction:
33
+ // stopped at any point, the next call carries on. Returns true when finished.
34
+ export function backfillStep(db: Database.Database, batch = BATCH): boolean {
35
+ const state = stateOf(db);
36
+ if (!state) return true;
37
+ if (state.version !== SCHEMA_VERSION) reset(db);
38
+ else if (state.done) return true;
39
+ const { cursor, backfill_to } = stateOf(db)!;
40
+ const kinds = SEARCHED_KINDS.map(() => "?").join(", ");
41
+ return db.transaction(() => {
42
+ // Rowids are not dense (tool calls are most rows), so the batch is
43
+ // counted in rows of the kinds searched and the cursor moves to the last.
44
+ const rows = db
45
+ .prepare(
46
+ `SELECT rowid AS rid, session_id, item_id,
47
+ json_extract(data, '$.text') AS text
48
+ FROM chat_items
49
+ WHERE rowid > ? AND rowid <= ? AND kind IN (${kinds})
50
+ ORDER BY rowid LIMIT ?`
51
+ )
52
+ .all(cursor, backfill_to, ...SEARCHED_KINDS, batch) as {
53
+ rid: number;
54
+ session_id: string;
55
+ item_id: string;
56
+ text: string | null;
57
+ }[];
58
+ const key = db.prepare(
59
+ `SELECT id FROM chat_search_keys WHERE session_id = ? AND item_id = ?`
60
+ );
61
+ const del = db.prepare(`DELETE FROM chat_search WHERE rowid = ?`);
62
+ const mk = db.prepare(
63
+ `INSERT OR IGNORE INTO chat_search_keys (session_id, item_id) VALUES (?, ?)`
64
+ );
65
+ const put = db.prepare(
66
+ `INSERT INTO chat_search (rowid, body) VALUES (?, ?)`
67
+ );
68
+ for (const r of rows) {
69
+ if (!r.text) continue;
70
+ const have = key.get(r.session_id, r.item_id) as
71
+ | { id: number }
72
+ | undefined;
73
+ if (have) del.run(have.id);
74
+ else mk.run(r.session_id, r.item_id);
75
+ const id = (key.get(r.session_id, r.item_id) as { id: number }).id;
76
+ put.run(id, r.text);
77
+ }
78
+ const finished = rows.length < batch;
79
+ const next = finished ? backfill_to : rows[rows.length - 1].rid;
80
+ db.prepare(
81
+ `UPDATE chat_search_state SET cursor = ?, done = ? WHERE id = 1`
82
+ ).run(next, finished ? 1 : 0);
83
+ return finished;
84
+ })();
85
+ }
86
+
87
+ // 0..1 while the first index is being built; null once it is complete.
88
+ export function indexProgress(): number | null {
89
+ const s = stateOf(getDb());
90
+ if (!s || (s.done && s.version === SCHEMA_VERSION)) return null;
91
+ return s.backfill_to ? Math.min(1, s.cursor / s.backfill_to) : 0;
92
+ }
93
+
94
+ let running = false;
95
+
96
+ // Runs batches until done, giving the event loop back between them, so a
97
+ // large history never holds up a request or the boot.
98
+ export function startConversationBackfill(): void {
99
+ if (running) return;
100
+ running = true;
101
+ const t0 = Date.now();
102
+ const tick = () => {
103
+ try {
104
+ if (backfillStep(getDb())) {
105
+ running = false;
106
+ console.log(`Conversation search index ready (${Date.now() - t0} ms)`);
107
+ return;
108
+ }
109
+ } catch (error) {
110
+ running = false;
111
+ console.error("Indexing conversations for search failed:", error);
112
+ return;
113
+ }
114
+ setImmediate(tick);
115
+ };
116
+ setImmediate(tick);
117
+ }
@@ -0,0 +1,31 @@
1
+ export interface MatchQuery {
2
+ // An FTS5 MATCH expression.
3
+ match: string;
4
+ // The words typed, lowercased, for marking a preview.
5
+ terms: string[];
6
+ }
7
+
8
+ const HAS_WORD = /[\p{L}\p{N}]/u;
9
+ const WORDS = /[\p{L}\p{N}_]+/gu;
10
+ // A quoted phrase (possibly still being typed) or a bare word.
11
+ const TOKEN = /"([^"]*)(")?|[^\s"]+/g;
12
+
13
+ // What was typed as an FTS5 expression. Every piece goes in as a quoted
14
+ // string, so FTS syntax typed by hand (AND, NEAR, *, :, -, parentheses) is
15
+ // plain text. Bare words match as prefixes, so a word being typed finds its
16
+ // completions; a closed "phrase" is exact, an open one is a prefix on its last
17
+ // word. Null when nothing searchable was typed.
18
+ export function buildMatchQuery(input: string): MatchQuery | null {
19
+ const parts: string[] = [];
20
+ const terms: string[] = [];
21
+ for (const m of input.matchAll(TOKEN)) {
22
+ const quoted = m[1] !== undefined;
23
+ const text = (quoted ? m[1] : m[0]).trim();
24
+ if (!HAS_WORD.test(text)) continue;
25
+ const prefix = !quoted || !m[2];
26
+ parts.push(`"${text}"${prefix ? "*" : ""}`);
27
+ for (const w of text.toLowerCase().match(WORDS) ?? []) terms.push(w);
28
+ }
29
+ if (!parts.length || terms.join("").length < 2) return null;
30
+ return { match: parts.join(" "), terms };
31
+ }
@@ -0,0 +1,9 @@
1
+ export { buildMatchQuery } from "./buildMatchQuery";
2
+ export { searchConversations } from "./searchConversations";
3
+ export { getMessageContext } from "./messageContext";
4
+ export {
5
+ startConversationBackfill,
6
+ backfillStep,
7
+ indexProgress,
8
+ } from "./backfill";
9
+ export type * from "./types";
@@ -0,0 +1,45 @@
1
+ import { getDb } from "@/lib/db";
2
+ import type { ContextItem } from "./types";
3
+
4
+ interface Row {
5
+ item_id: string;
6
+ kind: "user" | "assistant";
7
+ text: string | null;
8
+ created_at: number | null;
9
+ seq: number;
10
+ }
11
+
12
+ const COLS = `item_id, kind, seq, json_extract(data, '$.text') AS text,
13
+ json_extract(data, '$.createdAt') AS created_at`;
14
+
15
+ // A message and the ones said just before and after it, in order. Null when
16
+ // the message is gone. Only text: images and files stay out of a preview.
17
+ export function getMessageContext(
18
+ sessionId: string,
19
+ itemId: string,
20
+ around = 3
21
+ ): ContextItem[] | null {
22
+ const db = getDb();
23
+ const hit = db
24
+ .prepare(
25
+ `SELECT ${COLS} FROM chat_items WHERE session_id = ? AND item_id = ?`
26
+ )
27
+ .get(sessionId, itemId) as Row | undefined;
28
+ if (!hit) return null;
29
+ const side = (op: "<" | ">", order: "ASC" | "DESC") =>
30
+ db
31
+ .prepare(
32
+ `SELECT ${COLS} FROM chat_items
33
+ WHERE session_id = ? AND kind IN ('user', 'assistant') AND seq ${op} ?
34
+ ORDER BY seq ${order} LIMIT ?`
35
+ )
36
+ .all(sessionId, hit.seq, around) as Row[];
37
+ const rows = [...side("<", "DESC").reverse(), hit, ...side(">", "ASC")];
38
+ return rows.map((r) => ({
39
+ id: r.item_id,
40
+ role: r.kind,
41
+ text: r.text ?? "",
42
+ createdAt: r.created_at,
43
+ hit: r.item_id === itemId,
44
+ }));
45
+ }
@@ -0,0 +1,71 @@
1
+ import type Database from "better-sqlite3";
2
+
3
+ // Bump to rebuild the index from scratch (a different set of kinds, a new
4
+ // tokenizer); the next start's backfill does it.
5
+ export const SCHEMA_VERSION = 1;
6
+
7
+ // The item kinds whose text is searched. Tool calls and their output are
8
+ // most of the rows and mostly noise.
9
+ export const SEARCHED_KINDS = ["user", "assistant"] as const;
10
+
11
+ const KINDS_SQL = SEARCHED_KINDS.map((k) => `'${k}'`).join(", ");
12
+
13
+ // An item worth indexing: a searched kind with some text.
14
+ const eligible = `new.kind IN (${KINDS_SQL}) AND COALESCE(json_extract(new.data, '$.text'), '') <> ''`;
15
+
16
+ // The item's own key: the FTS rowid. chat_items has no integer primary key,
17
+ // so its rowids can be renumbered by a VACUUM.
18
+ const keyOf = (row: "new" | "old") =>
19
+ `(SELECT id FROM chat_search_keys WHERE session_id = ${row}.session_id AND item_id = ${row}.item_id)`;
20
+
21
+ const reindexItem = `
22
+ DELETE FROM chat_search WHERE rowid = ${keyOf("old")};
23
+ DELETE FROM chat_search_keys WHERE session_id = old.session_id AND item_id = old.item_id;
24
+ INSERT INTO chat_search_keys (session_id, item_id)
25
+ SELECT new.session_id, new.item_id WHERE ${eligible};
26
+ INSERT INTO chat_search (rowid, body)
27
+ SELECT ${keyOf("new")}, json_extract(new.data, '$.text') WHERE ${eligible};`;
28
+
29
+ // Every writer of chat_items (the server, the chat worker, an import) is
30
+ // indexed by these, so no code path can forget.
31
+ export function createConversationSearch(db: Database.Database): void {
32
+ db.exec(`
33
+ CREATE TABLE IF NOT EXISTS chat_search_keys (
34
+ id INTEGER PRIMARY KEY,
35
+ session_id TEXT NOT NULL,
36
+ item_id TEXT NOT NULL,
37
+ UNIQUE (session_id, item_id)
38
+ );
39
+ CREATE VIRTUAL TABLE IF NOT EXISTS chat_search USING fts5(
40
+ body,
41
+ tokenize = "unicode61 remove_diacritics 2 tokenchars '_'",
42
+ prefix = '2 3'
43
+ );
44
+ CREATE TABLE IF NOT EXISTS chat_search_state (
45
+ id INTEGER PRIMARY KEY CHECK (id = 1),
46
+ version INTEGER NOT NULL,
47
+ cursor INTEGER NOT NULL DEFAULT 0,
48
+ backfill_to INTEGER NOT NULL DEFAULT 0,
49
+ done INTEGER NOT NULL DEFAULT 0
50
+ );
51
+ INSERT OR IGNORE INTO chat_search_state (id, version, backfill_to, done)
52
+ SELECT 1, ${SCHEMA_VERSION}, COALESCE(MAX(rowid), 0), MAX(rowid) IS NULL
53
+ FROM chat_items;
54
+ CREATE TRIGGER IF NOT EXISTS chat_search_ai AFTER INSERT ON chat_items
55
+ BEGIN
56
+ DELETE FROM chat_search WHERE rowid = ${keyOf("new")};
57
+ INSERT OR IGNORE INTO chat_search_keys (session_id, item_id)
58
+ SELECT new.session_id, new.item_id WHERE ${eligible};
59
+ INSERT INTO chat_search (rowid, body)
60
+ SELECT ${keyOf("new")}, json_extract(new.data, '$.text') WHERE ${eligible};
61
+ END;
62
+ CREATE TRIGGER IF NOT EXISTS chat_search_au AFTER UPDATE OF data ON chat_items
63
+ BEGIN${reindexItem}
64
+ END;
65
+ CREATE TRIGGER IF NOT EXISTS chat_search_ad AFTER DELETE ON chat_items
66
+ BEGIN
67
+ DELETE FROM chat_search WHERE rowid = ${keyOf("old")};
68
+ DELETE FROM chat_search_keys WHERE session_id = old.session_id AND item_id = old.item_id;
69
+ END;
70
+ `);
71
+ }