@phren/cli 0.3.67 → 0.3.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/agent-hook-callback.js +11 -2
- package/dist/bridge/agent-hooks.d.ts +11 -1
- package/dist/bridge/agent-hooks.js +259 -40
- package/dist/bridge/changes.d.ts +2 -2
- package/dist/bridge/changes.js +11 -9
- package/dist/bridge/checkpoints.d.ts +24 -0
- package/dist/bridge/checkpoints.js +423 -0
- package/dist/bridge/claude-input-state.d.ts +14 -0
- package/dist/bridge/claude-input-state.js +58 -0
- package/dist/bridge/claude-queue.js +5 -2
- package/dist/bridge/claude-suggestion.d.ts +3 -0
- package/dist/bridge/claude-suggestion.js +10 -0
- package/dist/bridge/codex-threads.d.ts +7 -1
- package/dist/bridge/codex-threads.js +185 -56
- package/dist/bridge/conversation-search.d.ts +20 -0
- package/dist/bridge/conversation-search.js +928 -0
- package/dist/bridge/git-watch.d.ts +46 -0
- package/dist/bridge/git-watch.js +351 -0
- package/dist/bridge/git-worktree-remove.d.ts +44 -0
- package/dist/bridge/git-worktree-remove.js +140 -0
- package/dist/bridge/git-worktrees.d.ts +14 -0
- package/dist/bridge/git-worktrees.js +7 -3
- package/dist/bridge/git-write-policy.js +1 -1
- package/dist/bridge/gitboy-webhooks.d.ts +3 -1
- package/dist/bridge/gitboy-webhooks.js +26 -2
- package/dist/bridge/herdr.d.ts +6 -1
- package/dist/bridge/herdr.js +56 -10
- package/dist/bridge/hook-fast.js +13 -1
- package/dist/bridge/input-classification.d.ts +33 -0
- package/dist/bridge/input-classification.js +84 -0
- package/dist/bridge/pane-status.d.ts +3 -3
- package/dist/bridge/pane-status.js +16 -8
- package/dist/bridge/prompt-deliveries.d.ts +30 -4
- package/dist/bridge/prompt-deliveries.js +365 -13
- package/dist/bridge/prompt-identity.d.ts +8 -0
- package/dist/bridge/prompt-identity.js +15 -0
- package/dist/bridge/request-origin.js +2 -1
- package/dist/bridge/schedule-events.d.ts +17 -0
- package/dist/bridge/schedule-events.js +53 -0
- package/dist/bridge/schedule-format.d.ts +22 -2
- package/dist/bridge/schedule-format.js +40 -5
- package/dist/bridge/schedule-update.js +2 -1
- package/dist/bridge/schedule-watch.js +15 -5
- package/dist/bridge/schedules.d.ts +9 -0
- package/dist/bridge/schedules.js +120 -7
- package/dist/bridge/server-pane-routes.d.ts +7 -8
- package/dist/bridge/server-pane-routes.js +186 -82
- package/dist/bridge/server-routes.d.ts +26 -2
- package/dist/bridge/server-routes.js +180 -56
- package/dist/bridge/server-stream.js +66 -15
- package/dist/bridge/server.js +25 -7
- package/dist/bridge/session-activity.d.ts +49 -4
- package/dist/bridge/session-activity.js +47 -0
- package/dist/bridge/session-evidence.d.ts +18 -0
- package/dist/bridge/session-evidence.js +72 -0
- package/dist/bridge/session-seen.d.ts +23 -0
- package/dist/bridge/session-seen.js +69 -0
- package/dist/bridge/tab-activity.d.ts +26 -1
- package/dist/bridge/tab-activity.js +64 -9
- package/dist/bridge/terminal-resolve.d.ts +20 -0
- package/dist/bridge/terminal-resolve.js +145 -0
- package/dist/bridge/terminal-tmux.d.ts +8 -3
- package/dist/bridge/terminal-tmux.js +59 -7
- package/dist/bridge/transcript-claude.d.ts +12 -2
- package/dist/bridge/transcript-claude.js +91 -13
- package/dist/bridge/transcript-index.d.ts +4 -2
- package/dist/bridge/transcript-index.js +6 -5
- package/dist/bridge/transcript-preview.d.ts +16 -1
- package/dist/bridge/transcript-preview.js +171 -27
- package/dist/bridge/transcripts.d.ts +41 -1
- package/dist/bridge/transcripts.js +181 -23
- package/dist/bridge/turn-queue-pane.d.ts +6 -0
- package/dist/bridge/turn-queue-pane.js +30 -0
- package/dist/bridge/turn-queue.d.ts +86 -0
- package/dist/bridge/turn-queue.js +280 -0
- package/dist/bridge/turn-records.d.ts +18 -0
- package/dist/bridge/turn-records.js +77 -5
- package/dist/bridge-hook.mjs +7074 -3173
- package/dist/cli/config-shared.js +2 -1
- package/dist/cli/schedules.js +6 -1
- package/dist/content/secrets.js +1 -1
- package/dist/modules/registry.js +12 -11
- package/package.json +1 -1
|
@@ -0,0 +1,928 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { appendFile, mkdir, open, opendir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { claudeHomes } from "./claude-accounts.js";
|
|
5
|
+
import { codexHome, copilotHome } from "../home-paths.js";
|
|
6
|
+
import { redactSecretsForLog } from "../content/secrets.js";
|
|
7
|
+
import { codexThreadSessions, codexTurnSettled, materializeCodexThread, materializedPath, materializedRoot } from "./codex-threads.js";
|
|
8
|
+
import { paneIdentity, snapshot, recentServers } from "./herdr.js";
|
|
9
|
+
import { BridgeError, bridgeRoot, object, objects, sessionId } from "./protocol.js";
|
|
10
|
+
import { phrenStoreRoot, selectTranscriptMatch, visibleEvent } from "./transcripts.js";
|
|
11
|
+
const INDEX_FILE = () => path.join(bridgeRoot(), "conversation-search.sqlite");
|
|
12
|
+
const SIDECAR_DIR = () => path.join(bridgeRoot(), "conversation-search-rows");
|
|
13
|
+
const DEFAULT_LIMIT = 20;
|
|
14
|
+
const MAX_LIMIT = 50;
|
|
15
|
+
const QUERY_MIN = 2;
|
|
16
|
+
const QUERY_MAX = 200;
|
|
17
|
+
const MAX_REQUEST_MS = 1_500;
|
|
18
|
+
const MAX_DIRS_PER_REQUEST = 16;
|
|
19
|
+
const MAX_STAT_FILES = 64;
|
|
20
|
+
const MAX_SCAN_FILES = 24;
|
|
21
|
+
const MAX_BYTES_PER_REQUEST = 8 * 1024 * 1024;
|
|
22
|
+
const MAX_MATERIALIZE_PER_REQUEST = 8;
|
|
23
|
+
const READ_BLOCK = 65_536;
|
|
24
|
+
const MAX_ROW_BYTES = 64 * 1024 * 1024;
|
|
25
|
+
const DISCOVERY_RESCAN_MS = 5_000;
|
|
26
|
+
const MAX_DB_BYTES = 512 * 1024 * 1024;
|
|
27
|
+
const SNIPPET_CHARS = 200;
|
|
28
|
+
const SNIPPET_LEAD = 80;
|
|
29
|
+
const MAX_TITLE_CHARS = 200;
|
|
30
|
+
const SCHEMA = `
|
|
31
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
32
|
+
path TEXT PRIMARY KEY, root TEXT NOT NULL, source TEXT NOT NULL, session TEXT NOT NULL,
|
|
33
|
+
size INTEGER, mtime_ms REAL, dev TEXT, ino TEXT,
|
|
34
|
+
offset INTEGER NOT NULL DEFAULT 0, scan_offset INTEGER, scanning_row INTEGER NOT NULL DEFAULT 0, ready_until INTEGER NOT NULL DEFAULT -1, publishing INTEGER NOT NULL DEFAULT 0, publication_seq INTEGER NOT NULL DEFAULT -1, unresolved INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
canonical INTEGER NOT NULL DEFAULT 0, projected INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
lines INTEGER NOT NULL DEFAULT 0, title TEXT, cwd TEXT, last_at TEXT,
|
|
37
|
+
refresh INTEGER NOT NULL DEFAULT 0, copilot_final INTEGER NOT NULL DEFAULT 0, checked_at REAL, materialized_at INTEGER NOT NULL DEFAULT 0
|
|
38
|
+
);
|
|
39
|
+
CREATE INDEX IF NOT EXISTS files_identity ON files(source, session, projected, canonical);
|
|
40
|
+
CREATE INDEX IF NOT EXISTS files_fair ON files(COALESCE(checked_at, 0), path);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS files_materialize ON files(refresh, canonical, materialized_at, path);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS files_incomplete ON files(canonical, (size IS NULL OR offset < size OR scanning_row != 0));
|
|
43
|
+
CREATE INDEX IF NOT EXISTS files_publishing ON files(canonical, publishing);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS files_unresolved ON files(canonical, unresolved);
|
|
45
|
+
CREATE TABLE IF NOT EXISTS pending (
|
|
46
|
+
file TEXT NOT NULL, seq INTEGER NOT NULL, entry INTEGER NOT NULL, at TEXT NOT NULL,
|
|
47
|
+
role TEXT NOT NULL, text TEXT NOT NULL, message_id INTEGER, token_offset INTEGER NOT NULL DEFAULT 0, turn_id TEXT, queue_key TEXT, queued INTEGER NOT NULL DEFAULT 0, settle_after INTEGER NOT NULL, PRIMARY KEY(file, seq)
|
|
48
|
+
);
|
|
49
|
+
CREATE INDEX IF NOT EXISTS pending_queue ON pending(file, queue_key);
|
|
50
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
51
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, file TEXT NOT NULL, conversation TEXT NOT NULL,
|
|
52
|
+
source TEXT NOT NULL, entry INTEGER NOT NULL, at TEXT NOT NULL, role TEXT NOT NULL,
|
|
53
|
+
text TEXT NOT NULL, published INTEGER NOT NULL
|
|
54
|
+
);
|
|
55
|
+
CREATE INDEX IF NOT EXISTS messages_snapshot ON messages(published, at, id);
|
|
56
|
+
CREATE INDEX IF NOT EXISTS messages_file ON messages(file, at DESC, id DESC);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS messages_at ON messages(at, id);
|
|
58
|
+
CREATE TABLE IF NOT EXISTS tokens (
|
|
59
|
+
message_id INTEGER NOT NULL, token TEXT NOT NULL, PRIMARY KEY(message_id, token)
|
|
60
|
+
);
|
|
61
|
+
CREATE INDEX IF NOT EXISTS tokens_token ON tokens(token);
|
|
62
|
+
CREATE TABLE IF NOT EXISTS activity (
|
|
63
|
+
revision INTEGER PRIMARY KEY, file TEXT NOT NULL, at TEXT NOT NULL
|
|
64
|
+
);
|
|
65
|
+
CREATE INDEX IF NOT EXISTS activity_order ON activity(at DESC, revision DESC);
|
|
66
|
+
CREATE INDEX IF NOT EXISTS activity_file ON activity(file, revision DESC);
|
|
67
|
+
CREATE TABLE IF NOT EXISTS dirs (
|
|
68
|
+
root TEXT NOT NULL, path TEXT NOT NULL, source TEXT NOT NULL, depth INTEGER NOT NULL,
|
|
69
|
+
PRIMARY KEY(root, path)
|
|
70
|
+
);
|
|
71
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
72
|
+
`;
|
|
73
|
+
const safe = (text) => redactSecretsForLog(text);
|
|
74
|
+
const sidecarPath = (file) => path.join(SIDECAR_DIR(), createHash("sha256").update(file).digest("hex") + ".part");
|
|
75
|
+
const textTokens = (text) => [...new Set((text.match(/[\p{L}\p{N}_]+/gu) ?? []).map(token => token.toLowerCase()))];
|
|
76
|
+
function absolutePath(value) {
|
|
77
|
+
return typeof value === "string" && value.length > 0 && value.length <= 4_096 && path.isAbsolute(value) ? value : undefined;
|
|
78
|
+
}
|
|
79
|
+
function textBlock(block) {
|
|
80
|
+
if (!["text", "input_text", "output_text"].includes(String(block.type)))
|
|
81
|
+
return undefined;
|
|
82
|
+
return typeof block.text === "string" ? block.text : undefined;
|
|
83
|
+
}
|
|
84
|
+
function messageText(content) {
|
|
85
|
+
if (typeof content === "string")
|
|
86
|
+
return content;
|
|
87
|
+
return objects(content).map(textBlock).filter((text) => text !== undefined).join("\n");
|
|
88
|
+
}
|
|
89
|
+
/** The user or assistant text of one projected transcript row. Tool results,
|
|
90
|
+
* reasoning, deltas and system rows carry no message text. */
|
|
91
|
+
function messageOf(raw, source) {
|
|
92
|
+
let role, content;
|
|
93
|
+
if (source === "claude") {
|
|
94
|
+
if (raw.isCompactSummary === true)
|
|
95
|
+
return undefined;
|
|
96
|
+
const message = object(raw.message);
|
|
97
|
+
role = message.role;
|
|
98
|
+
content = message.content;
|
|
99
|
+
}
|
|
100
|
+
else if (source === "codex") {
|
|
101
|
+
const payload = object(raw.payload);
|
|
102
|
+
if (payload.type !== "message")
|
|
103
|
+
return undefined;
|
|
104
|
+
role = payload.role;
|
|
105
|
+
content = payload.content;
|
|
106
|
+
}
|
|
107
|
+
else if (source === "copilot") {
|
|
108
|
+
if (raw.type === "user.message")
|
|
109
|
+
role = "user";
|
|
110
|
+
else if (raw.type === "assistant.message")
|
|
111
|
+
role = "assistant";
|
|
112
|
+
else
|
|
113
|
+
return undefined;
|
|
114
|
+
content = object(raw.data).content;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
if (raw.type === "user/message")
|
|
118
|
+
role = "user";
|
|
119
|
+
else if (raw.type === "assistant/message")
|
|
120
|
+
role = "assistant";
|
|
121
|
+
else
|
|
122
|
+
return undefined;
|
|
123
|
+
const message = object(object(raw.data).message);
|
|
124
|
+
role = message.role ?? role;
|
|
125
|
+
content = message.content;
|
|
126
|
+
}
|
|
127
|
+
if (role !== "user" && role !== "assistant")
|
|
128
|
+
return undefined;
|
|
129
|
+
const text = messageText(content).trim();
|
|
130
|
+
return text ? { role, text } : undefined;
|
|
131
|
+
}
|
|
132
|
+
function timestampOf(raw, source) {
|
|
133
|
+
const value = source === "opencode" ? raw.time : raw.timestamp;
|
|
134
|
+
if (typeof value !== "string")
|
|
135
|
+
return undefined;
|
|
136
|
+
const parsed = Date.parse(value);
|
|
137
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : undefined;
|
|
138
|
+
}
|
|
139
|
+
function cwdOf(raw, source) {
|
|
140
|
+
const direct = absolutePath(raw.cwd);
|
|
141
|
+
if (direct)
|
|
142
|
+
return direct;
|
|
143
|
+
const payload = object(raw.payload), data = object(raw.data);
|
|
144
|
+
if (source === "copilot")
|
|
145
|
+
return absolutePath(object(data.context).cwd);
|
|
146
|
+
return absolutePath(payload.cwd) ?? absolutePath(data.cwd) ?? absolutePath(object(data.context).cwd);
|
|
147
|
+
}
|
|
148
|
+
/** A turn's end as the harness writes it; the same markers the returned
|
|
149
|
+
* conversation's final reply is read from (schedule-watch.ts). */
|
|
150
|
+
function turnEnded(raw, source, copilotFinal) {
|
|
151
|
+
const payload = object(raw.payload), message = object(raw.message), data = object(raw.data);
|
|
152
|
+
if (source === "claude")
|
|
153
|
+
return (raw.type === "assistant" && message.stop_reason === "end_turn")
|
|
154
|
+
|| (raw.type === "system" && raw.subtype === "turn_duration");
|
|
155
|
+
if (source === "codex")
|
|
156
|
+
return raw.type === "event_msg"
|
|
157
|
+
&& ["task_complete", "task_completed", "turn_aborted", "task_aborted"].includes(String(payload.type));
|
|
158
|
+
if (source === "copilot")
|
|
159
|
+
return ["session.error", "session.idle", "abort"].includes(String(raw.type))
|
|
160
|
+
|| (raw.type === "assistant.turn_end" && !raw.agentId && copilotFinal);
|
|
161
|
+
return data.stop_reason === "end_turn";
|
|
162
|
+
}
|
|
163
|
+
function queuedMessage(projected, source, raw) {
|
|
164
|
+
if (projected.phrenQueued === true)
|
|
165
|
+
return true;
|
|
166
|
+
return source === "copilot" && object(raw.data).delivery === "queued";
|
|
167
|
+
}
|
|
168
|
+
function materializedFile(file) {
|
|
169
|
+
return file.startsWith(materializedRoot() + path.sep);
|
|
170
|
+
}
|
|
171
|
+
async function openIndex() {
|
|
172
|
+
await mkdir(bridgeRoot(), { recursive: true, mode: 0o700 });
|
|
173
|
+
const sqlite = await import("node:sqlite");
|
|
174
|
+
let db = new sqlite.DatabaseSync(INDEX_FILE());
|
|
175
|
+
const version = Number(object(db.prepare("PRAGMA user_version").get()).user_version ?? 0);
|
|
176
|
+
if (version !== 2 && db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'files'").get()) {
|
|
177
|
+
// This is a derived cache. Rebuild older layouts rather than retaining
|
|
178
|
+
// completed scan offsets with incompatible token/publication state.
|
|
179
|
+
db.close();
|
|
180
|
+
await rm(INDEX_FILE(), { force: true });
|
|
181
|
+
await rm(INDEX_FILE() + "-wal", { force: true });
|
|
182
|
+
await rm(INDEX_FILE() + "-shm", { force: true });
|
|
183
|
+
db = new sqlite.DatabaseSync(INDEX_FILE());
|
|
184
|
+
}
|
|
185
|
+
db.exec("PRAGMA busy_timeout = 50");
|
|
186
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
187
|
+
db.exec("PRAGMA synchronous = NORMAL");
|
|
188
|
+
db.exec(SCHEMA);
|
|
189
|
+
db.exec("PRAGMA user_version = 2");
|
|
190
|
+
return db;
|
|
191
|
+
}
|
|
192
|
+
function metaValue(db, key) {
|
|
193
|
+
const row = object(db.prepare("SELECT value FROM meta WHERE key = ?").get(key));
|
|
194
|
+
return typeof row.value === "string" ? row.value : undefined;
|
|
195
|
+
}
|
|
196
|
+
function setMeta(db, key, value) {
|
|
197
|
+
db.prepare("INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
|
|
198
|
+
}
|
|
199
|
+
function rowFor(db, file) {
|
|
200
|
+
const row = db.prepare("SELECT * FROM files WHERE path = ?").get(file);
|
|
201
|
+
return row ? row : undefined;
|
|
202
|
+
}
|
|
203
|
+
async function clearFile(db, file, deadline) {
|
|
204
|
+
if (db.prepare("SELECT 1 FROM messages WHERE file = ? AND published > 0 LIMIT 1").get(file)) {
|
|
205
|
+
setMeta(db, "generation", String(Number(metaValue(db, "generation") ?? "0") + 1));
|
|
206
|
+
}
|
|
207
|
+
db.prepare("UPDATE files SET size = NULL, ready_until = -1, last_at = NULL, publishing = 0, publication_seq = -1, unresolved = 0 WHERE path = ?").run(file);
|
|
208
|
+
db.exec("BEGIN");
|
|
209
|
+
try {
|
|
210
|
+
while (Date.now() < deadline) {
|
|
211
|
+
const message = object(db.prepare("SELECT id FROM messages WHERE file = ? LIMIT 1").get(file));
|
|
212
|
+
if (message.id !== undefined) {
|
|
213
|
+
const token = object(db.prepare("SELECT token FROM tokens WHERE message_id = ? LIMIT 1").get(message.id));
|
|
214
|
+
if (token.token !== undefined)
|
|
215
|
+
db.prepare("DELETE FROM tokens WHERE message_id = ? AND token = ?").run(message.id, token.token);
|
|
216
|
+
else
|
|
217
|
+
db.prepare("DELETE FROM messages WHERE id = ?").run(message.id);
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
const pending = object(db.prepare("SELECT seq FROM pending WHERE file = ? LIMIT 1").get(file));
|
|
221
|
+
if (pending.seq !== undefined) {
|
|
222
|
+
db.prepare("DELETE FROM pending WHERE file = ? AND seq = ?").run(file, pending.seq);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
const activity = object(db.prepare("SELECT revision FROM activity WHERE file = ? LIMIT 1").get(file));
|
|
226
|
+
if (activity.revision !== undefined) {
|
|
227
|
+
db.prepare("DELETE FROM activity WHERE revision = ?").run(activity.revision);
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
await rm(sidecarPath(file), { force: true });
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
finally {
|
|
236
|
+
db.exec("COMMIT");
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function selectCanonical(db, source, session) {
|
|
240
|
+
const copies = db.prepare("SELECT path FROM files WHERE source = ? AND session = ? AND projected = 0 LIMIT 2").all(source, session)
|
|
241
|
+
.map(row => ({ id: undefined, path: row.path }));
|
|
242
|
+
const unique = selectTranscriptMatch(source, copies);
|
|
243
|
+
const projected = source === "codex" && !unique
|
|
244
|
+
? object(db.prepare("SELECT path FROM files WHERE source = ? AND session = ? AND projected = 1 LIMIT 1").get(source, session)).path : undefined;
|
|
245
|
+
const selected = unique?.path ?? (typeof projected === "string" ? projected : undefined);
|
|
246
|
+
const previous = object(db.prepare("SELECT path FROM files WHERE source = ? AND session = ? AND canonical = 1 LIMIT 1").get(source, session)).path;
|
|
247
|
+
if (previous === selected)
|
|
248
|
+
return;
|
|
249
|
+
if (typeof previous === "string" && db.prepare("SELECT 1 FROM messages WHERE file = ? AND published > 0 LIMIT 1").get(previous)) {
|
|
250
|
+
setMeta(db, "generation", String(Number(metaValue(db, "generation") ?? "0") + 1));
|
|
251
|
+
}
|
|
252
|
+
db.prepare("UPDATE files SET canonical = 0 WHERE source = ? AND session = ? AND canonical = 1").run(source, session);
|
|
253
|
+
if (selected)
|
|
254
|
+
db.prepare("UPDATE files SET canonical = 1 WHERE path = ?").run(selected);
|
|
255
|
+
}
|
|
256
|
+
async function deleteFile(db, file, deadline) {
|
|
257
|
+
const row = rowFor(db, file);
|
|
258
|
+
if (await clearFile(db, file, deadline)) {
|
|
259
|
+
db.prepare("DELETE FROM files WHERE path = ?").run(file);
|
|
260
|
+
if (row)
|
|
261
|
+
selectCanonical(db, row.source, row.session);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/** Publication is a durable job: neither a large turn nor a large vocabulary
|
|
265
|
+
* must finish in the request that discovers its end marker. */
|
|
266
|
+
async function commitPending(db, row, state, deadline) {
|
|
267
|
+
db.exec("BEGIN");
|
|
268
|
+
try {
|
|
269
|
+
db.prepare("UPDATE files SET publishing = 1 WHERE path = ?").run(row.path);
|
|
270
|
+
let after = rowFor(db, row.path)?.publication_seq ?? -1;
|
|
271
|
+
if (after === -1)
|
|
272
|
+
db.prepare("UPDATE files SET unresolved = 0 WHERE path = ?").run(row.path);
|
|
273
|
+
while (Date.now() < deadline) {
|
|
274
|
+
const item = db.prepare("SELECT * FROM pending WHERE file = ? AND seq > ? ORDER BY seq LIMIT 1").get(row.path, after);
|
|
275
|
+
if (!item) {
|
|
276
|
+
db.prepare("UPDATE files SET publishing = 0, publication_seq = -1 WHERE path = ?").run(row.path);
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
const settled = materializedFile(row.path) && !item.queued ? await codexTurnSettled(row.session, item.turn_id) : false;
|
|
280
|
+
if (item.queued || (materializedFile(row.path) ? settled !== true : item.settle_after > (rowFor(db, row.path)?.ready_until ?? -1))) {
|
|
281
|
+
if (materializedFile(row.path) && !item.queued && settled === undefined)
|
|
282
|
+
db.prepare("UPDATE files SET unresolved = 1 WHERE path = ?").run(row.path);
|
|
283
|
+
after = item.seq;
|
|
284
|
+
db.prepare("UPDATE files SET publication_seq = ? WHERE path = ?").run(after, row.path);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
let id = item.message_id;
|
|
288
|
+
if (id === null) {
|
|
289
|
+
id = Number(db.prepare(`INSERT INTO messages(file, conversation, source, entry, at, role, text, published)
|
|
290
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, 0)`).run(row.path, row.session, row.source, item.entry, item.at, item.role, item.text).lastInsertRowid);
|
|
291
|
+
db.prepare("UPDATE pending SET message_id = ? WHERE file = ? AND seq = ?").run(id, row.path, item.seq);
|
|
292
|
+
}
|
|
293
|
+
const words = /[\p{L}\p{N}_]+/gu;
|
|
294
|
+
words.lastIndex = item.token_offset;
|
|
295
|
+
const insert = db.prepare("INSERT OR IGNORE INTO tokens(message_id, token) VALUES(?, ?)");
|
|
296
|
+
let match, offset = item.token_offset;
|
|
297
|
+
while (Date.now() < deadline && (match = words.exec(item.text)) !== null) {
|
|
298
|
+
insert.run(id, match[0].toLowerCase());
|
|
299
|
+
offset = words.lastIndex;
|
|
300
|
+
}
|
|
301
|
+
db.prepare("UPDATE pending SET token_offset = ? WHERE file = ? AND seq = ?").run(offset, row.path, item.seq);
|
|
302
|
+
if (Date.now() >= deadline)
|
|
303
|
+
return;
|
|
304
|
+
const published = Number(metaValue(db, "published") ?? "0") + 1;
|
|
305
|
+
const newest = state.lastAt && state.lastAt > item.at ? state.lastAt : item.at;
|
|
306
|
+
db.prepare("UPDATE messages SET published = ? WHERE id = ?").run(published, id);
|
|
307
|
+
db.prepare("DELETE FROM pending WHERE file = ? AND seq = ?").run(row.path, item.seq);
|
|
308
|
+
db.prepare("UPDATE files SET last_at = ? WHERE path = ?").run(newest, row.path);
|
|
309
|
+
db.prepare("INSERT INTO activity(revision, file, at) VALUES(?, ?, ?)").run(published, row.path, newest);
|
|
310
|
+
setMeta(db, "published", String(published));
|
|
311
|
+
state.lastAt = newest;
|
|
312
|
+
after = item.seq;
|
|
313
|
+
db.prepare("UPDATE files SET publication_seq = ? WHERE path = ?").run(after, row.path);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
finally {
|
|
317
|
+
db.exec("COMMIT");
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
function addPending(db, row, message, entry, at, state, turn, queued, key) {
|
|
321
|
+
if (message.role === "user" && !state.title)
|
|
322
|
+
state.title = message.text;
|
|
323
|
+
db.prepare("INSERT OR IGNORE INTO pending(file, seq, entry, at, role, text, turn_id, queued, queue_key, settle_after) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
|
324
|
+
.run(row.path, entry, entry, at, message.role, message.text, typeof turn === "string" ? turn : null, queued ? 1 : 0, typeof key === "string" ? key : null, entry);
|
|
325
|
+
}
|
|
326
|
+
/** Eviction first invalidates coverage, then deletes one message at a time.
|
|
327
|
+
* Reset offsets ensure readable history is revisited; partial stays true until
|
|
328
|
+
* that happens. Never leave a completed offset pointing at discarded rows. */
|
|
329
|
+
async function pruneIndex(db, deadline) {
|
|
330
|
+
let victim = metaValue(db, "evicting");
|
|
331
|
+
if (!victim) {
|
|
332
|
+
const pages = Number(object(db.prepare("PRAGMA page_count").get()).page_count);
|
|
333
|
+
const free = Number(object(db.prepare("PRAGMA freelist_count").get()).freelist_count ?? 0);
|
|
334
|
+
const size = Number(object(db.prepare("PRAGMA page_size").get()).page_size);
|
|
335
|
+
if ((pages - free) * size <= MAX_DB_BYTES)
|
|
336
|
+
return false;
|
|
337
|
+
victim = String(object(db.prepare("SELECT file AS path FROM messages ORDER BY at, id LIMIT 1").get()).path ?? "");
|
|
338
|
+
if (!victim)
|
|
339
|
+
return false;
|
|
340
|
+
setMeta(db, "evicting", victim);
|
|
341
|
+
db.prepare("UPDATE files SET size = NULL, offset = 0, scan_offset = 0, lines = 0, scanning_row = 0, ready_until = -1, last_at = NULL, publishing = 0 WHERE path = ?").run(victim);
|
|
342
|
+
await rm(sidecarPath(victim), { force: true });
|
|
343
|
+
}
|
|
344
|
+
if (await clearFile(db, victim, deadline))
|
|
345
|
+
db.prepare("DELETE FROM meta WHERE key = 'evicting'").run();
|
|
346
|
+
return true;
|
|
347
|
+
}
|
|
348
|
+
function transcriptRoots() {
|
|
349
|
+
const roots = [];
|
|
350
|
+
for (const home of claudeHomes())
|
|
351
|
+
roots.push({ key: `claude:${home.id}`, root: path.join(home.dir, "projects"), source: "claude", maxDepth: 1 });
|
|
352
|
+
roots.push({ key: "codex", root: path.join(codexHome(), "sessions"), source: "codex", maxDepth: 3 });
|
|
353
|
+
roots.push({ key: "copilot", root: path.join(copilotHome(), "session-state"), source: "copilot", maxDepth: 1 });
|
|
354
|
+
roots.push({ key: "opencode", root: path.join(phrenStoreRoot(), ".runtime", "sessions"), source: "opencode", maxDepth: 0 });
|
|
355
|
+
roots.push({ key: "codex-materialized", root: materializedRoot(), source: "codex", maxDepth: 0 });
|
|
356
|
+
return roots;
|
|
357
|
+
}
|
|
358
|
+
function fileIdentity(source, file) {
|
|
359
|
+
const name = path.basename(file);
|
|
360
|
+
if (source === "claude") {
|
|
361
|
+
const session = name.endsWith(".jsonl") ? name.slice(0, -".jsonl".length) : "";
|
|
362
|
+
return sessionId.safeParse(session).success ? { session } : undefined;
|
|
363
|
+
}
|
|
364
|
+
if (source === "copilot") {
|
|
365
|
+
if (name !== "events.jsonl")
|
|
366
|
+
return undefined;
|
|
367
|
+
const session = path.basename(path.dirname(file));
|
|
368
|
+
return sessionId.safeParse(session).success ? { session } : undefined;
|
|
369
|
+
}
|
|
370
|
+
if (source === "opencode") {
|
|
371
|
+
const match = /^opencode-(.+)\.events\.jsonl$/.exec(name);
|
|
372
|
+
return match && sessionId.safeParse(match[1]).success ? { session: match[1] } : undefined;
|
|
373
|
+
}
|
|
374
|
+
const materialized = /^([0-9a-f-]{36}|ses_[0-9A-Za-z]{1,64})\.jsonl$/i.exec(name);
|
|
375
|
+
if (materialized && materializedFile(file) && sessionId.safeParse(materialized[1]).success)
|
|
376
|
+
return { session: materialized[1] };
|
|
377
|
+
if (path.relative(path.join(codexHome(), "sessions"), file).split(path.sep).length !== 4)
|
|
378
|
+
return undefined;
|
|
379
|
+
const rollout = /^rollout-.*-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i.exec(name);
|
|
380
|
+
return rollout && sessionId.safeParse(rollout[1]).success ? { session: rollout[1] } : undefined;
|
|
381
|
+
}
|
|
382
|
+
function insertFile(db, root, source, file, refresh = 0) {
|
|
383
|
+
const identity = fileIdentity(source, file);
|
|
384
|
+
if (!identity)
|
|
385
|
+
return;
|
|
386
|
+
db.prepare(`INSERT INTO files(path, root, source, session, refresh, projected) VALUES(?, ?, ?, ?, ?, ?)
|
|
387
|
+
ON CONFLICT(path) DO UPDATE SET root = excluded.root, source = excluded.source, session = excluded.session,
|
|
388
|
+
refresh = MAX(files.refresh, excluded.refresh)`).run(file, root, source, identity.session, refresh, materializedFile(file) ? 1 : 0);
|
|
389
|
+
selectCanonical(db, source, identity.session);
|
|
390
|
+
}
|
|
391
|
+
// Open directory handles are resumable within the Hook. After a restart the
|
|
392
|
+
// durable directory queue restarts the unfinished directory (idempotently).
|
|
393
|
+
const directoryScans = new Map();
|
|
394
|
+
async function discoverDirectories(db, deadline) {
|
|
395
|
+
const roots = transcriptRoots(), specs = new Map(roots.map(spec => [spec.key, spec]));
|
|
396
|
+
const pending = db.prepare("SELECT 1 FROM dirs LIMIT 1").get();
|
|
397
|
+
if (!pending && Date.now() - Number(metaValue(db, "discovery_cycle") ?? "0") >= DISCOVERY_RESCAN_MS) {
|
|
398
|
+
for (const spec of roots) {
|
|
399
|
+
if (Date.now() >= deadline)
|
|
400
|
+
return true;
|
|
401
|
+
db.prepare("INSERT OR IGNORE INTO dirs(root, path, source, depth) VALUES(?, ?, ?, 0)").run(spec.key, spec.root, spec.source);
|
|
402
|
+
}
|
|
403
|
+
setMeta(db, "discovery_cycle", String(Date.now()));
|
|
404
|
+
}
|
|
405
|
+
let processed = 0, entries = 0;
|
|
406
|
+
while (processed < MAX_DIRS_PER_REQUEST && entries < 512 && Date.now() < deadline) {
|
|
407
|
+
const row = object(db.prepare("SELECT root, path, source, depth FROM dirs ORDER BY rowid LIMIT 1").get());
|
|
408
|
+
if (typeof row.path !== "string" || typeof row.root !== "string")
|
|
409
|
+
break;
|
|
410
|
+
const key = INDEX_FILE() + "\0" + row.path;
|
|
411
|
+
let directory = directoryScans.get(key);
|
|
412
|
+
if (!directory) {
|
|
413
|
+
directory = await opendir(row.path).catch(() => undefined);
|
|
414
|
+
if (directory)
|
|
415
|
+
directoryScans.set(key, directory);
|
|
416
|
+
}
|
|
417
|
+
const entry = directory ? await directory.read().catch(() => null) : null;
|
|
418
|
+
if (!entry) {
|
|
419
|
+
await directory?.close().catch(() => undefined);
|
|
420
|
+
directoryScans.delete(key);
|
|
421
|
+
db.prepare("DELETE FROM dirs WHERE root = ? AND path = ?").run(row.root, row.path);
|
|
422
|
+
processed++;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
entries++;
|
|
426
|
+
if (entry.isSymbolicLink())
|
|
427
|
+
continue;
|
|
428
|
+
const spec = specs.get(row.root), child = path.join(row.path, entry.name);
|
|
429
|
+
if (entry.isDirectory() && Number(row.depth) < (spec?.maxDepth ?? 0)) {
|
|
430
|
+
db.prepare("INSERT OR IGNORE INTO dirs(root, path, source, depth) VALUES(?, ?, ?, ?)").run(row.root, child, row.source, Number(row.depth) + 1);
|
|
431
|
+
}
|
|
432
|
+
else if (entry.isFile())
|
|
433
|
+
insertFile(db, row.path, row.source, child);
|
|
434
|
+
}
|
|
435
|
+
return !!db.prepare("SELECT 1 FROM dirs LIMIT 1").get();
|
|
436
|
+
}
|
|
437
|
+
async function discoverSqliteThreads(db, deadline) {
|
|
438
|
+
let after = metaValue(db, "sqlite_after") ?? "";
|
|
439
|
+
if (!after && Date.now() - Number(metaValue(db, "sqlite_cycle") ?? "0") < DISCOVERY_RESCAN_MS)
|
|
440
|
+
return false;
|
|
441
|
+
const threads = await codexThreadSessions(128, after).catch(() => []);
|
|
442
|
+
for (const thread of threads) {
|
|
443
|
+
if (Date.now() >= deadline)
|
|
444
|
+
return true;
|
|
445
|
+
if (sessionId.safeParse(thread).success)
|
|
446
|
+
insertFile(db, materializedRoot(), "codex", materializedPath(thread), 1);
|
|
447
|
+
after = thread;
|
|
448
|
+
setMeta(db, "sqlite_after", after);
|
|
449
|
+
}
|
|
450
|
+
if (threads.length === 128)
|
|
451
|
+
return true;
|
|
452
|
+
setMeta(db, "sqlite_after", "");
|
|
453
|
+
setMeta(db, "sqlite_cycle", String(Date.now()));
|
|
454
|
+
return false;
|
|
455
|
+
}
|
|
456
|
+
function updateChecked(db, file) {
|
|
457
|
+
const tick = Number(metaValue(db, "check_clock") ?? "0") + 1;
|
|
458
|
+
setMeta(db, "check_clock", String(tick));
|
|
459
|
+
db.prepare("UPDATE files SET checked_at = ? WHERE path = ?").run(tick, file);
|
|
460
|
+
}
|
|
461
|
+
async function processRow(db, row, line, lineNumber, state, fallbackAt, deadline) {
|
|
462
|
+
if (!line.trim())
|
|
463
|
+
return;
|
|
464
|
+
let raw;
|
|
465
|
+
try {
|
|
466
|
+
raw = object(JSON.parse(line));
|
|
467
|
+
}
|
|
468
|
+
catch {
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
state.cwd ??= cwdOf(raw, row.source) ?? null;
|
|
472
|
+
if (row.source === "copilot" && raw.type === "assistant.message" && !raw.agentId) {
|
|
473
|
+
state.copilotFinal = object(raw.data).phase === "final_answer";
|
|
474
|
+
}
|
|
475
|
+
const projected = visibleEvent(raw, row.source);
|
|
476
|
+
const message = projected ? messageOf(projected, row.source) : undefined;
|
|
477
|
+
if (projected?.type === "phren_queue_consumed" && typeof projected.key === "string") {
|
|
478
|
+
db.prepare("UPDATE pending SET queued = 0, turn_id = COALESCE(?, turn_id), settle_after = ? WHERE file = ? AND queue_key = ?")
|
|
479
|
+
.run(typeof raw.phrenTurnId === "string" ? raw.phrenTurnId : null, lineNumber, row.path, projected.key);
|
|
480
|
+
}
|
|
481
|
+
if (message && projected) {
|
|
482
|
+
const at = timestampOf(projected, row.source) ?? fallbackAt;
|
|
483
|
+
addPending(db, row, message, lineNumber, at, state, raw.phrenTurnId, queuedMessage(projected, row.source, raw), projected.phrenQueueKey);
|
|
484
|
+
}
|
|
485
|
+
if (turnEnded(raw, row.source, state.copilotFinal)) {
|
|
486
|
+
db.prepare("UPDATE files SET ready_until = ? WHERE path = ?").run(lineNumber, row.path);
|
|
487
|
+
await commitPending(db, row, state, deadline);
|
|
488
|
+
state.copilotFinal = false;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
async function scanFile(db, row, info, budget, deadline) {
|
|
492
|
+
const sidecar = sidecarPath(row.path), sidecarInfo = await stat(sidecar).catch(() => undefined);
|
|
493
|
+
let replacement = row.size === null || row.dev !== String(info.dev) || row.ino !== String(info.ino)
|
|
494
|
+
|| info.size < Math.max(row.offset, row.scan_offset ?? 0)
|
|
495
|
+
|| (info.size === row.size && row.mtime_ms !== info.mtimeMs);
|
|
496
|
+
if (row.scanning_row === 1 && !sidecarInfo?.isFile())
|
|
497
|
+
replacement = true;
|
|
498
|
+
let offset = replacement ? 0 : row.offset;
|
|
499
|
+
let lines = replacement ? 0 : row.lines;
|
|
500
|
+
let scanningRow = replacement ? 0 : row.scanning_row;
|
|
501
|
+
let sidecarBytes = replacement ? 0 : sidecarInfo?.size ?? 0;
|
|
502
|
+
let start = replacement ? 0 : (scanningRow ? row.scan_offset ?? row.offset : row.offset);
|
|
503
|
+
let title = replacement ? null : row.title;
|
|
504
|
+
let cwd = replacement ? null : row.cwd;
|
|
505
|
+
let lastAt = replacement ? null : row.last_at;
|
|
506
|
+
let copilotFinal = replacement ? false : row.copilot_final === 1;
|
|
507
|
+
if (replacement && !await clearFile(db, row.path, deadline))
|
|
508
|
+
return { bytes: 0, complete: false };
|
|
509
|
+
const state = { title, cwd, lastAt, copilotFinal };
|
|
510
|
+
const fallbackAt = new Date(info.mtimeMs).toISOString();
|
|
511
|
+
let bytesRead = 0, position = start, chunkStart = start, pending = Buffer.alloc(0);
|
|
512
|
+
const handle = await open(row.path, "r");
|
|
513
|
+
try {
|
|
514
|
+
while (bytesRead < budget && position < info.size && Date.now() < deadline) {
|
|
515
|
+
const length = Math.min(READ_BLOCK, info.size - position, budget - bytesRead);
|
|
516
|
+
if (length <= 0)
|
|
517
|
+
break;
|
|
518
|
+
const buffer = Buffer.alloc(length);
|
|
519
|
+
const { bytesRead: read } = await handle.read(buffer, 0, length, position);
|
|
520
|
+
if (!read)
|
|
521
|
+
break;
|
|
522
|
+
bytesRead += read;
|
|
523
|
+
const chunk = pending.length ? Buffer.concat([pending, buffer.subarray(0, read)]) : buffer.subarray(0, read);
|
|
524
|
+
let begin = 0;
|
|
525
|
+
if (scanningRow === 2) {
|
|
526
|
+
const newline = chunk.indexOf(10, begin);
|
|
527
|
+
if (newline < 0) {
|
|
528
|
+
position += read;
|
|
529
|
+
pending = Buffer.alloc(0);
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
lines++;
|
|
533
|
+
chunkStart = position + newline + 1;
|
|
534
|
+
offset = chunkStart;
|
|
535
|
+
begin = newline + 1;
|
|
536
|
+
scanningRow = 0;
|
|
537
|
+
}
|
|
538
|
+
else if (scanningRow === 1) {
|
|
539
|
+
const newline = chunk.indexOf(10, begin);
|
|
540
|
+
if (newline < 0) {
|
|
541
|
+
if (sidecarBytes + chunk.length > MAX_ROW_BYTES) {
|
|
542
|
+
await rm(sidecar, { force: true });
|
|
543
|
+
sidecarBytes = 0;
|
|
544
|
+
scanningRow = 2;
|
|
545
|
+
}
|
|
546
|
+
else {
|
|
547
|
+
await appendFile(sidecar, chunk, { mode: 0o600 });
|
|
548
|
+
sidecarBytes += chunk.length;
|
|
549
|
+
}
|
|
550
|
+
position += read;
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
const rowBytes = chunk.subarray(0, newline);
|
|
554
|
+
if (sidecarBytes + rowBytes.length <= MAX_ROW_BYTES) {
|
|
555
|
+
if (rowBytes.length)
|
|
556
|
+
await appendFile(sidecar, rowBytes, { mode: 0o600 });
|
|
557
|
+
const line = await readFile(sidecar, "utf8").catch(() => "");
|
|
558
|
+
if (Date.now() >= deadline) {
|
|
559
|
+
position += newline;
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
562
|
+
await processRow(db, row, line, lines, state, fallbackAt, deadline);
|
|
563
|
+
}
|
|
564
|
+
lines++;
|
|
565
|
+
chunkStart = position + newline + 1;
|
|
566
|
+
offset = chunkStart;
|
|
567
|
+
begin = newline + 1;
|
|
568
|
+
scanningRow = 0;
|
|
569
|
+
sidecarBytes = 0;
|
|
570
|
+
await rm(sidecar, { force: true });
|
|
571
|
+
}
|
|
572
|
+
let newline;
|
|
573
|
+
while (Date.now() < deadline && (newline = chunk.indexOf(10, begin)) >= 0) {
|
|
574
|
+
await processRow(db, row, chunk.subarray(begin, newline).toString("utf8"), lines, state, fallbackAt, deadline);
|
|
575
|
+
lines++;
|
|
576
|
+
chunkStart += newline + 1 - begin;
|
|
577
|
+
offset = chunkStart;
|
|
578
|
+
begin = newline + 1;
|
|
579
|
+
}
|
|
580
|
+
if (Date.now() >= deadline) {
|
|
581
|
+
position = offset;
|
|
582
|
+
pending = Buffer.alloc(0);
|
|
583
|
+
break;
|
|
584
|
+
}
|
|
585
|
+
pending = Buffer.from(chunk.subarray(begin));
|
|
586
|
+
if (pending.length > MAX_ROW_BYTES) {
|
|
587
|
+
pending = Buffer.alloc(0);
|
|
588
|
+
chunkStart = position + read;
|
|
589
|
+
scanningRow = 2;
|
|
590
|
+
}
|
|
591
|
+
position += read;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
finally {
|
|
595
|
+
await handle.close();
|
|
596
|
+
}
|
|
597
|
+
if (position < info.size && scanningRow === 0 && pending.length > 0) {
|
|
598
|
+
await mkdir(SIDECAR_DIR(), { recursive: true, mode: 0o700 });
|
|
599
|
+
await writeFile(sidecar, pending, { mode: 0o600 });
|
|
600
|
+
sidecarBytes = pending.length;
|
|
601
|
+
scanningRow = 1;
|
|
602
|
+
}
|
|
603
|
+
await commitPending(db, row, state, deadline);
|
|
604
|
+
const complete = offset >= info.size && scanningRow === 0;
|
|
605
|
+
db.prepare(`UPDATE files SET size = ?, mtime_ms = ?, dev = ?, ino = ?, offset = ?, scan_offset = ?,
|
|
606
|
+
scanning_row = ?, lines = ?, title = ?, cwd = ?, last_at = ?, copilot_final = ? WHERE path = ?`)
|
|
607
|
+
.run(info.size, info.mtimeMs, String(info.dev), String(info.ino), offset, scanningRow ? position : offset, scanningRow, lines, state.title ?? title, state.cwd ?? cwd, state.lastAt ?? lastAt, state.copilotFinal ? 1 : 0, row.path);
|
|
608
|
+
return { bytes: bytesRead, complete };
|
|
609
|
+
}
|
|
610
|
+
async function refreshFile(db, row, budget, deadline) {
|
|
611
|
+
if (Date.now() >= deadline || budget.checked >= MAX_STAT_FILES || budget.scanned >= MAX_SCAN_FILES || budget.bytes >= MAX_BYTES_PER_REQUEST) {
|
|
612
|
+
budget.partial = true;
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
budget.checked++;
|
|
616
|
+
updateChecked(db, row.path);
|
|
617
|
+
if (row.path === metaValue(db, "evicting")) {
|
|
618
|
+
budget.partial = true;
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
const info = await stat(row.path).catch(() => undefined);
|
|
622
|
+
if (!info?.isFile()) {
|
|
623
|
+
if (materializedFile(row.path)) {
|
|
624
|
+
db.prepare("UPDATE files SET refresh = 1 WHERE path = ?").run(row.path);
|
|
625
|
+
if (row.canonical)
|
|
626
|
+
budget.partial = true;
|
|
627
|
+
}
|
|
628
|
+
else
|
|
629
|
+
await deleteFile(db, row.path, deadline);
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
632
|
+
if (!row.canonical)
|
|
633
|
+
return;
|
|
634
|
+
if (materializedFile(row.path))
|
|
635
|
+
db.prepare("UPDATE files SET refresh = 1 WHERE path = ?").run(row.path);
|
|
636
|
+
const unchanged = row.size === info.size && row.mtime_ms === info.mtimeMs && row.dev === String(info.dev)
|
|
637
|
+
&& row.ino === String(info.ino) && row.offset >= info.size && row.scanning_row === 0;
|
|
638
|
+
try {
|
|
639
|
+
if (!unchanged) {
|
|
640
|
+
const result = await scanFile(db, row, info, MAX_BYTES_PER_REQUEST - budget.bytes, deadline);
|
|
641
|
+
budget.bytes += result.bytes;
|
|
642
|
+
budget.scanned++;
|
|
643
|
+
if (!result.complete)
|
|
644
|
+
budget.partial = true;
|
|
645
|
+
}
|
|
646
|
+
else
|
|
647
|
+
await commitPending(db, row, { title: row.title, cwd: row.cwd, lastAt: row.last_at, copilotFinal: false }, deadline);
|
|
648
|
+
}
|
|
649
|
+
catch {
|
|
650
|
+
budget.partial = true;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
async function refreshFiles(db, deadline) {
|
|
654
|
+
const budget = { partial: false, checked: 0, scanned: 0, bytes: 0 };
|
|
655
|
+
const first = metaValue(db, "refresh_first") ?? "materialize";
|
|
656
|
+
setMeta(db, "refresh_first", first === "materialize" ? "files" : "materialize");
|
|
657
|
+
// Alternate the first queue too: even an unusually expensive work unit in
|
|
658
|
+
// one queue cannot consume the other queue's budget on every request.
|
|
659
|
+
for (const phase of first === "materialize" ? ["materialize", "files"] : ["files", "materialize"]) {
|
|
660
|
+
if (Date.now() >= deadline) {
|
|
661
|
+
budget.partial = true;
|
|
662
|
+
break;
|
|
663
|
+
}
|
|
664
|
+
if (phase === "materialize") {
|
|
665
|
+
const rows = db.prepare("SELECT * FROM files WHERE refresh = 1 AND canonical = 1 ORDER BY materialized_at, path LIMIT ?")
|
|
666
|
+
.all(MAX_MATERIALIZE_PER_REQUEST);
|
|
667
|
+
for (const row of rows) {
|
|
668
|
+
if (Date.now() >= deadline) {
|
|
669
|
+
budget.partial = true;
|
|
670
|
+
break;
|
|
671
|
+
}
|
|
672
|
+
const file = await materializeCodexThread(row.session, deadline, 128).catch(() => undefined);
|
|
673
|
+
const tick = Number(metaValue(db, "materialize_clock") ?? "0") + 1;
|
|
674
|
+
setMeta(db, "materialize_clock", String(tick));
|
|
675
|
+
db.prepare("UPDATE files SET materialized_at = ? WHERE path = ?").run(tick, row.path);
|
|
676
|
+
if (!file) {
|
|
677
|
+
budget.partial = true;
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
const state = await readFile(file + ".state.json", "utf8").then(value => object(JSON.parse(value))).catch(() => ({}));
|
|
681
|
+
if (state.forwardDone !== true || state.updateTarget !== undefined || state.upgrading === true)
|
|
682
|
+
budget.partial = true;
|
|
683
|
+
// If materialization ran second, index its new tail in this request
|
|
684
|
+
// too, using the same total file/byte allowance as ordinary refresh.
|
|
685
|
+
await refreshFile(db, rowFor(db, row.path) ?? row, budget, deadline);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
// Leave room for all eight materialized tails when that queue runs
|
|
690
|
+
// second; otherwise a newly discovered thread waits a full stat rotation.
|
|
691
|
+
const rows = db.prepare("SELECT * FROM files ORDER BY COALESCE(checked_at, 0), path LIMIT ?")
|
|
692
|
+
.all(Math.max(0, MAX_STAT_FILES - budget.checked - (first === "files" ? MAX_MATERIALIZE_PER_REQUEST : 0)));
|
|
693
|
+
for (const row of rows) {
|
|
694
|
+
if (Date.now() >= deadline || budget.scanned >= MAX_SCAN_FILES || budget.bytes >= MAX_BYTES_PER_REQUEST) {
|
|
695
|
+
budget.partial = true;
|
|
696
|
+
break;
|
|
697
|
+
}
|
|
698
|
+
await refreshFile(db, row, budget, deadline);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
return budget.partial || !!db.prepare("SELECT 1 FROM files WHERE canonical = 1 AND (size IS NULL OR offset < size OR scanning_row != 0) = 1 LIMIT 1").get()
|
|
703
|
+
|| !!db.prepare("SELECT 1 FROM files WHERE canonical = 1 AND publishing = 1 LIMIT 1").get()
|
|
704
|
+
|| !!db.prepare("SELECT 1 FROM files WHERE canonical = 1 AND unresolved = 1 LIMIT 1").get();
|
|
705
|
+
}
|
|
706
|
+
const encodeCursor = (cursor) => Buffer.from(JSON.stringify(cursor)).toString("base64url");
|
|
707
|
+
function decodeCursor(cursor) {
|
|
708
|
+
if (!cursor)
|
|
709
|
+
return undefined;
|
|
710
|
+
try {
|
|
711
|
+
const value = object(JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")));
|
|
712
|
+
if (typeof value.q !== "string" || !Number.isSafeInteger(value.w) || !Number.isSafeInteger(value.g) || typeof value.ca !== "string"
|
|
713
|
+
|| typeof value.at !== "string" || !Number.isSafeInteger(value.id)
|
|
714
|
+
|| typeof value.file !== "string" || !Number.isSafeInteger(value.revision))
|
|
715
|
+
throw new Error();
|
|
716
|
+
return { q: value.q, w: value.w, g: value.g, ca: value.ca, at: value.at, id: value.id, file: value.file, revision: value.revision };
|
|
717
|
+
}
|
|
718
|
+
catch {
|
|
719
|
+
throw new BridgeError(400, "Invalid cursor.");
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
function snippetFor(text, tokens, deadline) {
|
|
723
|
+
const lower = text.toLowerCase();
|
|
724
|
+
let first = -1;
|
|
725
|
+
for (const token of tokens) {
|
|
726
|
+
if (Date.now() >= deadline)
|
|
727
|
+
return undefined;
|
|
728
|
+
const at = lower.indexOf(token);
|
|
729
|
+
if (at >= 0 && (first < 0 || at < first))
|
|
730
|
+
first = at;
|
|
731
|
+
}
|
|
732
|
+
if (first < 0)
|
|
733
|
+
first = 0;
|
|
734
|
+
const start = Math.max(0, Math.min(first - SNIPPET_LEAD, text.length - SNIPPET_CHARS));
|
|
735
|
+
const snippet = text.slice(start, start + SNIPPET_CHARS);
|
|
736
|
+
const redacted = safe(text);
|
|
737
|
+
return redacted === text ? snippet : redacted.slice(0, SNIPPET_CHARS);
|
|
738
|
+
}
|
|
739
|
+
function titleFor(title, session) {
|
|
740
|
+
const text = title ?? session;
|
|
741
|
+
const redacted = safe(text);
|
|
742
|
+
return redacted.slice(0, MAX_TITLE_CHARS);
|
|
743
|
+
}
|
|
744
|
+
async function resolvePanes(keys, deadline) {
|
|
745
|
+
const panes = new Map();
|
|
746
|
+
// Live snapshots are primary evidence, including OpenCode, whose lifecycle
|
|
747
|
+
// lives in per-PID status files rather than the persisted turn directory.
|
|
748
|
+
const serverNames = new Set(["default"]);
|
|
749
|
+
const candidates = new Map();
|
|
750
|
+
for (const server of serverNames) {
|
|
751
|
+
if (Date.now() >= deadline)
|
|
752
|
+
break;
|
|
753
|
+
const view = await withinDeadline(snapshot(server).catch(() => undefined), deadline, undefined);
|
|
754
|
+
for (const pane of objects(view?.panes)) {
|
|
755
|
+
if (Date.now() >= deadline)
|
|
756
|
+
break;
|
|
757
|
+
const source = String(pane.agent);
|
|
758
|
+
if (!["claude", "codex", "copilot", "opencode"].includes(source))
|
|
759
|
+
continue;
|
|
760
|
+
const identity = await withinDeadline(paneIdentity(server, pane, true).catch(() => undefined), deadline, undefined);
|
|
761
|
+
const key = `${source}\0${identity}`;
|
|
762
|
+
if (!identity || !keys.has(key))
|
|
763
|
+
continue;
|
|
764
|
+
const name = String(pane.pane_id), terminal = String(pane.terminal_id);
|
|
765
|
+
const record = await readFile(path.join(bridgeRoot(), "turns", encodeURIComponent(server), encodeURIComponent(name) + ".json"), "utf8")
|
|
766
|
+
.then(value => object(JSON.parse(value))).catch(() => undefined);
|
|
767
|
+
if (record && record.session === identity && record.terminal !== terminal)
|
|
768
|
+
continue;
|
|
769
|
+
const at = record?.session === identity && typeof record.at === "string" ? record.at : "";
|
|
770
|
+
const previous = candidates.get(key);
|
|
771
|
+
if (!previous || at > previous.at) {
|
|
772
|
+
candidates.set(key, { server, pane: name, terminal, source, session: identity, at });
|
|
773
|
+
panes.set(key, name);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
if (server === "default" && candidates.size < keys.size && Date.now() < deadline) {
|
|
777
|
+
const listed = await withinDeadline(recentServers().catch(() => []), deadline, []);
|
|
778
|
+
for (const item of listed) {
|
|
779
|
+
if (Date.now() >= deadline)
|
|
780
|
+
break;
|
|
781
|
+
serverNames.add(String(item.session));
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
return panes;
|
|
786
|
+
}
|
|
787
|
+
/** Bound external terminal probes too; the underlying read-only probe may
|
|
788
|
+
* finish later, but cannot delay the response beyond this request's budget. */
|
|
789
|
+
async function withinDeadline(work, deadline, fallback) {
|
|
790
|
+
let timer;
|
|
791
|
+
try {
|
|
792
|
+
return await Promise.race([work, new Promise(resolve => { timer = setTimeout(() => resolve(fallback), Math.max(0, deadline - Date.now())); })]);
|
|
793
|
+
}
|
|
794
|
+
finally {
|
|
795
|
+
clearTimeout(timer);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
async function search(q, limit, cursor) {
|
|
799
|
+
const query = (q ?? "").trim();
|
|
800
|
+
if (query.length < QUERY_MIN || query.length > QUERY_MAX)
|
|
801
|
+
throw new BridgeError(400, `q must be ${QUERY_MIN} to ${QUERY_MAX} characters.`);
|
|
802
|
+
let count = DEFAULT_LIMIT;
|
|
803
|
+
if (limit !== null && limit !== undefined && String(limit).trim() !== "") {
|
|
804
|
+
const parsed = Number(limit);
|
|
805
|
+
if (!Number.isInteger(parsed) || parsed < 1 || parsed > MAX_LIMIT)
|
|
806
|
+
throw new BridgeError(400, `limit must be 1 to ${MAX_LIMIT}.`);
|
|
807
|
+
count = parsed;
|
|
808
|
+
}
|
|
809
|
+
const decoded = decodeCursor(cursor);
|
|
810
|
+
if (decoded && decoded.q !== query)
|
|
811
|
+
throw new BridgeError(400, "The cursor belongs to another query.");
|
|
812
|
+
const tokens = textTokens(query);
|
|
813
|
+
const deadline = Date.now() + MAX_REQUEST_MS;
|
|
814
|
+
const db = await openIndex();
|
|
815
|
+
try {
|
|
816
|
+
const discoveryPartial = await discoverDirectories(db, deadline);
|
|
817
|
+
const sqlitePartial = Date.now() < deadline ? await discoverSqliteThreads(db, deadline) : true;
|
|
818
|
+
const pruningPartial = Date.now() < deadline ? await pruneIndex(db, deadline) : true;
|
|
819
|
+
const indexingPartial = Date.now() < deadline ? await refreshFiles(db, deadline) : true;
|
|
820
|
+
let partial = discoveryPartial || sqlitePartial || indexingPartial || pruningPartial;
|
|
821
|
+
if (!tokens.length)
|
|
822
|
+
return { results: [], cursor: null, ...(partial ? { partial: true } : {}) };
|
|
823
|
+
const generation = Number(metaValue(db, "generation") ?? "0");
|
|
824
|
+
if (decoded && decoded.g !== generation)
|
|
825
|
+
throw new BridgeError(400, "Indexed history changed. Start a new search.");
|
|
826
|
+
const watermark = decoded?.w ?? Number(metaValue(db, "published") ?? "0");
|
|
827
|
+
// Activity is versioned at publication. Indexed keyset reads walk the
|
|
828
|
+
// newest conversation versions in the cursor's watermark, independently
|
|
829
|
+
// of matching text. No window or sort over the set of matching messages.
|
|
830
|
+
const position = decoded ?? { q: query, w: watermark, g: generation, ca: "\uffff", revision: Number.MAX_SAFE_INTEGER, file: "", at: "\uffff", id: Number.MAX_SAFE_INTEGER };
|
|
831
|
+
const page = [];
|
|
832
|
+
const titles = new Map();
|
|
833
|
+
let exhausted = false;
|
|
834
|
+
while (page.length < count && Date.now() < deadline) {
|
|
835
|
+
if (!position.file) {
|
|
836
|
+
const activity = object(db.prepare(`SELECT * FROM activity WHERE
|
|
837
|
+
(at, revision) < (?, ?) ORDER BY at DESC, revision DESC LIMIT 1`)
|
|
838
|
+
.get(position.ca, position.revision));
|
|
839
|
+
if (typeof activity.file !== "string") {
|
|
840
|
+
exhausted = true;
|
|
841
|
+
break;
|
|
842
|
+
}
|
|
843
|
+
position.ca = String(activity.at);
|
|
844
|
+
position.revision = Number(activity.revision);
|
|
845
|
+
if (position.revision > watermark)
|
|
846
|
+
continue;
|
|
847
|
+
const latest = object(db.prepare("SELECT revision FROM activity WHERE file = ? AND revision <= ? ORDER BY revision DESC LIMIT 1").get(activity.file, watermark));
|
|
848
|
+
if (latest.revision !== activity.revision)
|
|
849
|
+
continue;
|
|
850
|
+
const file = rowFor(db, activity.file);
|
|
851
|
+
if (!file?.canonical)
|
|
852
|
+
continue;
|
|
853
|
+
position.file = activity.file;
|
|
854
|
+
position.at = "\uffff";
|
|
855
|
+
position.id = Number.MAX_SAFE_INTEGER;
|
|
856
|
+
}
|
|
857
|
+
// Rediscovery can reveal another account copy or a preferred rollout.
|
|
858
|
+
// Pause with this file and message position intact until it completes;
|
|
859
|
+
// advancing past it would silently discard entries from the snapshot.
|
|
860
|
+
if (discoveryPartial) {
|
|
861
|
+
const source = rowFor(db, position.file)?.source;
|
|
862
|
+
if (source === "claude" || source === "codex")
|
|
863
|
+
break;
|
|
864
|
+
}
|
|
865
|
+
const row = object(db.prepare(`SELECT m.*, f.title, f.cwd FROM messages m JOIN files f ON f.path = m.file
|
|
866
|
+
WHERE f.canonical = 1 AND m.file = ? AND (m.at, m.id) < (?, ?)
|
|
867
|
+
ORDER BY m.at DESC, m.id DESC LIMIT 1`).get(position.file, position.at, position.id));
|
|
868
|
+
if (row.id === undefined) {
|
|
869
|
+
position.file = "";
|
|
870
|
+
continue;
|
|
871
|
+
}
|
|
872
|
+
let matches = Number(row.published) > 0 && Number(row.published) <= watermark;
|
|
873
|
+
for (const token of tokens) {
|
|
874
|
+
if (!matches)
|
|
875
|
+
break;
|
|
876
|
+
if (Date.now() >= deadline)
|
|
877
|
+
break;
|
|
878
|
+
if (!db.prepare("SELECT 1 FROM tokens WHERE message_id = ? AND token = ?").get(row.id, token)) {
|
|
879
|
+
matches = false;
|
|
880
|
+
break;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
// Keep the message position unchanged if a word check was interrupted.
|
|
884
|
+
if (Date.now() >= deadline)
|
|
885
|
+
break;
|
|
886
|
+
if (matches) {
|
|
887
|
+
if (!titles.has(position.file))
|
|
888
|
+
titles.set(position.file, titleFor(typeof row.title === "string" ? row.title : null, String(row.conversation)));
|
|
889
|
+
const snippet = snippetFor(String(row.text), tokens, deadline);
|
|
890
|
+
if (snippet === undefined)
|
|
891
|
+
break;
|
|
892
|
+
page.push({ ...row, text: undefined, title: titles.get(position.file), snippet });
|
|
893
|
+
}
|
|
894
|
+
position.at = String(row.at);
|
|
895
|
+
position.id = Number(row.id);
|
|
896
|
+
}
|
|
897
|
+
if (!exhausted && page.length < count)
|
|
898
|
+
partial = true;
|
|
899
|
+
const next = exhausted ? null : encodeCursor(position);
|
|
900
|
+
const wanted = new Set(page.map(row => `${String(row.source)}\0${String(row.conversation)}`));
|
|
901
|
+
const panes = wanted.size && Date.now() < deadline ? await resolvePanes(wanted, deadline) : new Map();
|
|
902
|
+
if (Date.now() >= deadline)
|
|
903
|
+
partial = true;
|
|
904
|
+
return {
|
|
905
|
+
results: page.map(row => ({
|
|
906
|
+
conversation: String(row.conversation), source: String(row.source),
|
|
907
|
+
title: String(row.title),
|
|
908
|
+
cwd: typeof row.cwd === "string" ? row.cwd : null,
|
|
909
|
+
pane: panes.get(`${String(row.source)}\0${String(row.conversation)}`) ?? null,
|
|
910
|
+
entry: String(row.entry), at: String(row.at), role: String(row.role),
|
|
911
|
+
snippet: String(row.snippet),
|
|
912
|
+
})),
|
|
913
|
+
cursor: next,
|
|
914
|
+
...(partial ? { partial: true } : {}),
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
finally {
|
|
918
|
+
db.close();
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
let tail = Promise.resolve();
|
|
922
|
+
/** Serialize index updates in this Hook process; reads stay consistent with the
|
|
923
|
+
* last write and two searches cannot rebuild the same file twice at once. */
|
|
924
|
+
export function conversationSearch(q, limit, cursor) {
|
|
925
|
+
const run = tail.then(() => search(q, limit, cursor));
|
|
926
|
+
tail = run.catch(() => undefined);
|
|
927
|
+
return run;
|
|
928
|
+
}
|