conductor-remote 1.92.0 → 1.92.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.html CHANGED
@@ -25,7 +25,7 @@
25
25
  <title>Conductor Remote</title>
26
26
  <!-- Runs before the module bundle so it can catch a stale shell that fails to boot. -->
27
27
  <script src="/self-heal.js"></script>
28
- <script type="module" crossorigin src="/assets/index-fo7654RB.js"></script>
28
+ <script type="module" crossorigin src="/assets/index-DnnHEVDt.js"></script>
29
29
  <link rel="stylesheet" crossorigin href="/assets/index-DFclyO59.css">
30
30
  <link rel="manifest" href="/manifest.webmanifest"></head>
31
31
  <body>
package/dist/sw.js CHANGED
@@ -1 +1 @@
1
- if(!self.define){let e,s={};const i=(i,n)=>(i=new URL(i+".js",n).href,s[i]||new Promise(s=>{if("document"in self){const e=document.createElement("script");e.src=i,e.onload=s,document.head.appendChild(e)}else e=i,importScripts(i),s()}).then(()=>{let e=s[i];if(!e)throw new Error(`Module ${i} didn’t register its module`);return e}));self.define=(n,r)=>{const o=e||("document"in self?document.currentScript.src:"")||location.href;if(s[o])return;let l={};const c=e=>i(e,o),t={module:{uri:o},exports:l,require:c};s[o]=Promise.all(n.map(e=>t[e]||c(e))).then(e=>(r(...e),l))}}define(["./workbox-9c191d2f"],function(e){"use strict";importScripts("/push-sw.js"),self.addEventListener("message",e=>{e.data&&"SKIP_WAITING"===e.data.type&&self.skipWaiting()}),e.clientsClaim(),e.precacheAndRoute([{url:"self-heal.js",revision:"49bd63adb25a09341f8d2610e8bd3c76"},{url:"push-sw.js",revision:"e7ef44deca46c0539e6ff7bba5eb815e"},{url:"index.html",revision:"175c7da73f7e7fd24ab30b2a00788436"},{url:"assets/workbox-window.prod.es5-BBnX5xw4.js",revision:null},{url:"assets/index-fo7654RB.js",revision:null},{url:"assets/index-DFclyO59.css",revision:null},{url:"apple-touch-icon.png",revision:"1127bb396b4648add53dce3f22c92aee"},{url:"icon-192.png",revision:"c5e01ac58768627e18ee7b8b6a9239ef"},{url:"icon-512.png",revision:"a40638c55e310312457a621c9a0002c8"},{url:"icon-maskable-512.png",revision:"a9b0d962686287452216492cd2247499"},{url:"icon.svg",revision:"c1aee186821798733dd477e69a0ef243"},{url:"manifest.webmanifest",revision:"cf88fbc5755108a7fe0616fa160a8a15"}],{}),e.cleanupOutdatedCaches(),e.registerRoute(new e.NavigationRoute(e.createHandlerBoundToURL("/index.html"),{denylist:[/^\/api\//]}))});
1
+ if(!self.define){let e,s={};const i=(i,n)=>(i=new URL(i+".js",n).href,s[i]||new Promise(s=>{if("document"in self){const e=document.createElement("script");e.src=i,e.onload=s,document.head.appendChild(e)}else e=i,importScripts(i),s()}).then(()=>{let e=s[i];if(!e)throw new Error(`Module ${i} didn’t register its module`);return e}));self.define=(n,r)=>{const o=e||("document"in self?document.currentScript.src:"")||location.href;if(s[o])return;let l={};const c=e=>i(e,o),t={module:{uri:o},exports:l,require:c};s[o]=Promise.all(n.map(e=>t[e]||c(e))).then(e=>(r(...e),l))}}define(["./workbox-9c191d2f"],function(e){"use strict";importScripts("/push-sw.js"),self.addEventListener("message",e=>{e.data&&"SKIP_WAITING"===e.data.type&&self.skipWaiting()}),e.clientsClaim(),e.precacheAndRoute([{url:"self-heal.js",revision:"49bd63adb25a09341f8d2610e8bd3c76"},{url:"push-sw.js",revision:"e7ef44deca46c0539e6ff7bba5eb815e"},{url:"index.html",revision:"3b37955213b676ec4a6a2b69275ddb07"},{url:"assets/workbox-window.prod.es5-BBnX5xw4.js",revision:null},{url:"assets/index-DnnHEVDt.js",revision:null},{url:"assets/index-DFclyO59.css",revision:null},{url:"apple-touch-icon.png",revision:"1127bb396b4648add53dce3f22c92aee"},{url:"icon-192.png",revision:"c5e01ac58768627e18ee7b8b6a9239ef"},{url:"icon-512.png",revision:"a40638c55e310312457a621c9a0002c8"},{url:"icon-maskable-512.png",revision:"a9b0d962686287452216492cd2247499"},{url:"icon.svg",revision:"c1aee186821798733dd477e69a0ef243"},{url:"manifest.webmanifest",revision:"cf88fbc5755108a7fe0616fa160a8a15"}],{}),e.cleanupOutdatedCaches(),e.registerRoute(new e.NavigationRoute(e.createHandlerBoundToURL("/index.html"),{denylist:[/^\/api\//]}))});
@@ -1,4 +1,6 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
+ const SLOW_QUERY_MS = 100;
3
+ const SLOW_QUERY_LOG_INTERVAL_MS = 60_000;
2
4
  /**
3
5
  * Read-only handle to Conductor's SQLite DB.
4
6
  *
@@ -8,9 +10,14 @@ import { DatabaseSync } from 'node:sqlite';
8
10
  */
9
11
  export class ConductorDb {
10
12
  dbPath;
13
+ onSlowQuery;
14
+ slowQueryMs;
15
+ slowQueryLogs = new Map();
11
16
  db;
12
- constructor(dbPath) {
17
+ constructor(dbPath, options = {}) {
13
18
  this.dbPath = dbPath;
19
+ this.onSlowQuery = options.onSlowQuery ?? (message => console.warn(message));
20
+ this.slowQueryMs = options.slowQueryMs ?? SLOW_QUERY_MS;
14
21
  this.db = this.open();
15
22
  }
16
23
  open() {
@@ -24,13 +31,41 @@ export class ConductorDb {
24
31
  return db;
25
32
  }
26
33
  query(sql, params = []) {
34
+ const started = performance.now();
35
+ let reopened = false;
27
36
  try {
28
- return this.db.prepare(sql).all(...params);
37
+ try {
38
+ return this.db.prepare(sql).all(...params);
39
+ }
40
+ catch {
41
+ // If the DB file was swapped underneath us (app update), reopen once.
42
+ reopened = true;
43
+ this.db = this.open();
44
+ return this.db.prepare(sql).all(...params);
45
+ }
46
+ }
47
+ finally {
48
+ this.reportSlowQuery(sql, performance.now() - started, reopened);
49
+ }
50
+ }
51
+ reportSlowQuery(sql, elapsedMs, reopened) {
52
+ if (elapsedMs < this.slowQueryMs)
53
+ return;
54
+ const summary = sql.replace(/\s+/g, ' ').trim();
55
+ const now = Date.now();
56
+ const previous = this.slowQueryLogs.get(summary);
57
+ if (previous && now - previous.at < SLOW_QUERY_LOG_INTERVAL_MS) {
58
+ previous.suppressed++;
59
+ return;
60
+ }
61
+ const suppressed = previous?.suppressed ? `; ${previous.suppressed} similar calls suppressed` : '';
62
+ const retried = reopened ? '; connection reopened' : '';
63
+ try {
64
+ this.onSlowQuery(`⚠ slow Conductor DB query (${Math.round(elapsedMs)}ms${retried}${suppressed}): ${summary.slice(0, 240)}`);
29
65
  }
30
66
  catch {
31
- // If the DB file was swapped underneath us (app update), reopen once.
32
- this.db = this.open();
33
- return this.db.prepare(sql).all(...params);
67
+ // Instrumentation must never turn a successful read into a failed API call.
34
68
  }
69
+ this.slowQueryLogs.set(summary, { at: now, suppressed: 0 });
35
70
  }
36
71
  }
@@ -0,0 +1,269 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { DatabaseSync } from 'node:sqlite';
4
+ import { parentPort, workerData } from 'node:worker_threads';
5
+ import { ConductorDb } from "./db.js";
6
+ import { HIT_CLOSE, HIT_OPEN, matchQuery } from "./search.js";
7
+ import { parseMessage } from "./transcript.js";
8
+ /** Bump when extraction or an invariant such as one-chunk-per-source changes stored rows. */
9
+ const SCHEMA_VERSION = 2;
10
+ /**
11
+ * Source rows advanced per tick. The cursor moves by *scanned* rowid rather than
12
+ * matched rowid, so a caught-up index re-scans nothing.
13
+ */
14
+ const WINDOW_ROWS = 4000;
15
+ const BACKFILL_PAUSE_MS = 5;
16
+ const IDLE_POLL_MS = 15_000;
17
+ const MAX_CHUNK_CHARS = 64_000;
18
+ const CHUNK_LIMIT = 300;
19
+ const SLOW_OPERATION_MS = 100;
20
+ const SLOW_LOG_INTERVAL_MS = 60_000;
21
+ const INDEXED_ROLES = new Set(['user', 'assistant', 'thinking']);
22
+ const port = parentPort;
23
+ if (!port)
24
+ throw new Error('search worker requires a parent port');
25
+ const post = (message) => port.postMessage(message);
26
+ const log = (level, message) => post({ type: 'log', level, message });
27
+ const errorText = (error) => (error instanceof Error ? error.message : String(error));
28
+ class SearchIndexWorker {
29
+ source;
30
+ file;
31
+ db = null;
32
+ cursor = 0;
33
+ chunks = 0;
34
+ caughtUp = false;
35
+ timer = null;
36
+ sourceMax = 0;
37
+ slowLogs = new Map();
38
+ constructor(sourceDbPath, file) {
39
+ this.source = new ConductorDb(sourceDbPath, { onSlowQuery: message => log('warn', message) });
40
+ this.file = file;
41
+ }
42
+ open() {
43
+ fs.mkdirSync(path.dirname(this.file), { recursive: true });
44
+ const db = new DatabaseSync(this.file);
45
+ // Set the wait before any pragma or schema operation that may need the writer.
46
+ // A second dev relay can own it for a batch; this wait is why the connection must
47
+ // live off the HTTP thread.
48
+ db.exec('PRAGMA busy_timeout = 5000');
49
+ db.exec('PRAGMA journal_mode = WAL');
50
+ db.exec('PRAGMA synchronous = NORMAL');
51
+ db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)');
52
+ const version = Number(this.readMeta(db, 'version') ?? 0);
53
+ if (version !== SCHEMA_VERSION) {
54
+ db.exec('DROP TABLE IF EXISTS chunks');
55
+ db.exec(`
56
+ CREATE VIRTUAL TABLE chunks USING fts5(
57
+ body,
58
+ session_id UNINDEXED,
59
+ src_rowid UNINDEXED,
60
+ role UNINDEXED,
61
+ at UNINDEXED,
62
+ tokenize='porter unicode61'
63
+ )
64
+ `);
65
+ db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('version', String(SCHEMA_VERSION));
66
+ db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', '0');
67
+ if (version)
68
+ log('log', `search index schema ${version} → ${SCHEMA_VERSION}, rebuilding`);
69
+ }
70
+ this.db = db;
71
+ this.cursor = Number(this.readMeta(db, 'cursor') ?? 0);
72
+ this.chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
73
+ }
74
+ start() {
75
+ this.schedule(0);
76
+ }
77
+ stop() {
78
+ if (this.timer)
79
+ clearTimeout(this.timer);
80
+ this.timer = null;
81
+ }
82
+ status() {
83
+ if (!this.db)
84
+ return { chunks: 0, ready: false, progress: 0 };
85
+ if (this.caughtUp)
86
+ return { chunks: this.chunks, ready: true, progress: 1 };
87
+ if (!this.sourceMax) {
88
+ const max = this.source.query('SELECT MAX(rowid) m FROM session_messages')[0]?.m;
89
+ this.sourceMax = max ?? 0;
90
+ }
91
+ const progress = this.sourceMax ? Math.min(1, this.cursor / this.sourceMax) : 0;
92
+ return { chunks: this.chunks, ready: false, progress };
93
+ }
94
+ search(raw, { limit = CHUNK_LIMIT, sessionIds } = {}) {
95
+ const started = performance.now();
96
+ try {
97
+ const db = this.db;
98
+ if (!db)
99
+ return [];
100
+ const match = matchQuery(raw);
101
+ if (!match || (sessionIds && !sessionIds.length))
102
+ return [];
103
+ const scope = sessionIds ? 'AND session_id IN (SELECT value FROM json_each(?))' : '';
104
+ const params = sessionIds
105
+ ? [HIT_OPEN, HIT_CLOSE, match, JSON.stringify(sessionIds), limit]
106
+ : [HIT_OPEN, HIT_CLOSE, match, limit];
107
+ let rows;
108
+ try {
109
+ rows = db
110
+ .prepare(`SELECT session_id, src_rowid, role, at, -bm25(chunks) AS score,
111
+ snippet(chunks, 0, ?, ?, '…', 24) AS snippet
112
+ FROM chunks WHERE chunks MATCH ? ${scope} ORDER BY bm25(chunks) LIMIT ?`)
113
+ .all(...params);
114
+ }
115
+ catch (error) {
116
+ throw new Error(`search failed for ${JSON.stringify(match)}: ${errorText(error)}`);
117
+ }
118
+ return rows.map(row => ({
119
+ sessionId: row.session_id,
120
+ srcRowid: Number(row.src_rowid),
121
+ role: INDEXED_ROLES.has(row.role) ? row.role : 'assistant',
122
+ at: row.at,
123
+ score: Number(row.score),
124
+ snippet: row.snippet
125
+ }));
126
+ }
127
+ finally {
128
+ this.reportSlow('query', performance.now() - started);
129
+ }
130
+ }
131
+ readMeta(db, key) {
132
+ const row = db.prepare('SELECT v FROM meta WHERE k = ?').get(key);
133
+ return row?.v ?? null;
134
+ }
135
+ schedule(ms) {
136
+ if (this.timer)
137
+ clearTimeout(this.timer);
138
+ this.timer = setTimeout(() => {
139
+ const started = performance.now();
140
+ let more = false;
141
+ try {
142
+ more = this.tick();
143
+ }
144
+ catch (error) {
145
+ log('warn', `⚠ search index tick failed: ${errorText(error)}`);
146
+ }
147
+ finally {
148
+ this.reportSlow('backfill batch', performance.now() - started);
149
+ }
150
+ try {
151
+ post({ type: 'status', status: this.status() });
152
+ }
153
+ catch (error) {
154
+ log('warn', `⚠ search index status failed: ${errorText(error)}`);
155
+ }
156
+ this.schedule(more ? BACKFILL_PAUSE_MS : IDLE_POLL_MS);
157
+ }, ms);
158
+ this.timer.unref?.();
159
+ }
160
+ /**
161
+ * Index one source-row window. Filtering happens after the rowid window is chosen,
162
+ * so the cursor advances across tool-only rows instead of scanning them forever.
163
+ */
164
+ tick() {
165
+ const db = this.db;
166
+ if (!db)
167
+ return false;
168
+ const startingCursor = this.cursor;
169
+ const window = this.source.query('SELECT rowid FROM session_messages WHERE rowid > ? ORDER BY rowid LIMIT ?', [startingCursor, WINDOW_ROWS]);
170
+ if (!window.length) {
171
+ this.caughtUp = true;
172
+ return false;
173
+ }
174
+ this.caughtUp = false;
175
+ const end = window[window.length - 1].rowid;
176
+ const rows = this.source.query(`SELECT rowid, id, session_id, role, content, full_message, created_at, sent_at, queue_order
177
+ FROM session_messages
178
+ WHERE rowid > ? AND rowid <= ? AND session_id IS NOT NULL
179
+ AND (role = 'user' OR content LIKE '%"type":"text"%' OR content LIKE '%"type":"thinking"%')
180
+ ORDER BY rowid`, [startingCursor, end]);
181
+ // Parse outside the sidecar write transaction. A second relay may be waiting for
182
+ // that writer, and parsing transcript JSON does not need to hold it.
183
+ const chunks = [];
184
+ for (const row of rows) {
185
+ for (const entry of parseMessage(row, null)) {
186
+ if (!INDEXED_ROLES.has(entry.role))
187
+ continue;
188
+ const body = entry.text.trim();
189
+ if (!body)
190
+ continue;
191
+ chunks.push({
192
+ body: body.slice(0, MAX_CHUNK_CHARS),
193
+ sessionId: row.session_id,
194
+ srcRowid: row.rowid,
195
+ role: entry.role,
196
+ at: entry.ts
197
+ });
198
+ }
199
+ }
200
+ // BEGIN IMMEDIATE serialises the cursor check with the batch write. Without the
201
+ // check, a dev relay and the service can both read cursor N, wait for each other,
202
+ // and then insert the same N→M chunks twice.
203
+ db.exec('BEGIN IMMEDIATE');
204
+ try {
205
+ const durableCursor = Number(this.readMeta(db, 'cursor') ?? 0);
206
+ if (durableCursor !== startingCursor) {
207
+ db.exec('ROLLBACK');
208
+ this.cursor = durableCursor;
209
+ this.chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
210
+ return true;
211
+ }
212
+ const insert = db.prepare('INSERT INTO chunks(body, session_id, src_rowid, role, at) VALUES (?, ?, ?, ?, ?)');
213
+ for (const chunk of chunks) {
214
+ insert.run(chunk.body, chunk.sessionId, chunk.srcRowid, chunk.role, chunk.at);
215
+ }
216
+ db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', String(end));
217
+ db.exec('COMMIT');
218
+ }
219
+ catch (error) {
220
+ try {
221
+ db.exec('ROLLBACK');
222
+ }
223
+ catch {
224
+ // Preserve the original SQLite error.
225
+ }
226
+ throw error;
227
+ }
228
+ this.cursor = end;
229
+ this.chunks += chunks.length;
230
+ return true;
231
+ }
232
+ reportSlow(operation, elapsedMs) {
233
+ if (elapsedMs < SLOW_OPERATION_MS)
234
+ return;
235
+ const now = Date.now();
236
+ const previous = this.slowLogs.get(operation);
237
+ if (previous && now - previous.at < SLOW_LOG_INTERVAL_MS) {
238
+ previous.suppressed++;
239
+ return;
240
+ }
241
+ const suppressed = previous?.suppressed ? `; ${previous.suppressed} similar calls suppressed` : '';
242
+ log('warn', `⚠ slow search index ${operation} (${Math.round(elapsedMs)}ms${suppressed})`);
243
+ this.slowLogs.set(operation, { at: now, suppressed: 0 });
244
+ }
245
+ }
246
+ const config = workerData;
247
+ let index = null;
248
+ try {
249
+ index = new SearchIndexWorker(config.sourceDbPath, config.file);
250
+ index.open();
251
+ post({ type: 'status', status: index.status() });
252
+ index.start();
253
+ }
254
+ catch (error) {
255
+ const message = errorText(error);
256
+ log('warn', `⚠ search index unavailable (${message}) — /api/search will report it`);
257
+ post({ type: 'status', status: { chunks: 0, ready: false, progress: 0, error: message } });
258
+ }
259
+ port.on('message', (request) => {
260
+ if (request.type !== 'search')
261
+ return;
262
+ try {
263
+ post({ type: 'result', id: request.id, hits: index?.search(request.raw, request.options) ?? [] });
264
+ }
265
+ catch (error) {
266
+ post({ type: 'error', id: request.id, error: errorText(error) });
267
+ }
268
+ });
269
+ process.once('exit', () => index?.stop());