conductor-remote 1.92.1 → 1.93.0

package/dist-node/src/run-configs.js

@@ -0,0 +1,337 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+const fileCache = new Map();
+/** Remove a TOML comment without treating a `#` inside a quoted string as one. */
+function withoutComment(line) {
+    let quote = null;
+    let escaped = false;
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+        if (quote === '"' && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (quote === '"' && char === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (quote) {
+            if (char === quote)
+                quote = null;
+            continue;
+        }
+        if (char === '"' || char === "'")
+            quote = char;
+        else if (char === '#')
+            return line.slice(0, i);
+    }
+    return line;
+}
+/** Strip comments from a value that can span several physical TOML lines. */
+function withoutComments(value) {
+    return value
+        .split(/\r?\n/)
+        .map(line => withoutComment(line))
+        .join('\n');
+}
+function tomlString(raw) {
+    const value = withoutComment(raw).trim();
+    if (value.startsWith("'") && value.endsWith("'") && !value.startsWith("'''"))
+        return value.slice(1, -1);
+    if (!(value.startsWith('"') && value.endsWith('"')) || value.startsWith('"""'))
+        return null;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return null;
+    }
+}
+/** Split a TOML dotted key, retaining dots inside quoted components. */
+function dottedKey(raw) {
+    const parts = [];
+    let start = 0;
+    let quote = null;
+    let escaped = false;
+    for (let i = 0; i < raw.length; i++) {
+        const char = raw[i];
+        if (quote === '"' && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (quote === '"' && char === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (quote) {
+            if (char === quote)
+                quote = null;
+            continue;
+        }
+        if (char === '"' || char === "'")
+            quote = char;
+        else if (char === '.') {
+            parts.push(raw.slice(start, i));
+            start = i + 1;
+        }
+    }
+    if (quote)
+        return null;
+    parts.push(raw.slice(start));
+    const decoded = parts.map(part => {
+        const value = part.trim();
+        if (!value)
+            return null;
+        if (value.startsWith('"') || value.startsWith("'"))
+            return tomlString(value);
+        return /^[A-Za-z0-9_-]+$/.test(value) ? value : null;
+    });
+    return decoded.every((part) => part !== null) ? decoded : null;
+}
+function tablePath(line) {
+    const clean = withoutComment(line).trim();
+    if (clean.startsWith('[['))
+        return null;
+    const match = clean.match(/^\[([^\]]+)]$/);
+    return match ? dottedKey(match[1]) : null;
+}
+function assignment(line) {
+    const clean = withoutComment(line);
+    let quote = null;
+    let escaped = false;
+    for (let i = 0; i < clean.length; i++) {
+        const char = clean[i];
+        if (quote === '"' && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (quote === '"' && char === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (quote) {
+            if (char === quote)
+                quote = null;
+            continue;
+        }
+        if (char === '"' || char === "'")
+            quote = char;
+        else if (char === '=') {
+            const key = dottedKey(clean.slice(0, i));
+            if (key?.length !== 1)
+                return null;
+            return { key: key[0], value: clean.slice(i + 1).trim() };
+        }
+    }
+    return null;
+}
+function multilineDelimiter(value) {
+    const trimmed = value.trimStart();
+    for (const delimiter of ['"""', "'''"]) {
+        if (!trimmed.startsWith(delimiter))
+            continue;
+        return trimmed.indexOf(delimiter, delimiter.length) < 0 ? delimiter : null;
+    }
+    return null;
+}
+function stringValue(value) {
+    if (tomlString(value) !== null)
+        return true;
+    const trimmed = withoutComments(value).trim();
+    return ((trimmed.startsWith('"""') && trimmed.indexOf('"""', 3) >= 3) ||
+        (trimmed.startsWith("'''") && trimmed.indexOf("'''", 3) >= 3));
+}
+function arrayOpen(value) {
+    let depth = 0;
+    let quote = null;
+    let escaped = false;
+    for (const char of withoutComments(value)) {
+        if (quote === '"' && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (quote === '"' && char === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (quote) {
+            if (char === quote)
+                quote = null;
+            continue;
+        }
+        if (char === '"' || char === "'")
+            quote = char;
+        else if (char === '[')
+            depth++;
+        else if (char === ']')
+            depth--;
+    }
+    return depth > 0;
+}
+function stringList(raw) {
+    const scalar = tomlString(raw);
+    if (scalar !== null)
+        return [scalar];
+    const value = withoutComments(raw).trim();
+    if (!value.startsWith('[') || !value.endsWith(']'))
+        return null;
+    const values = [];
+    let start = 1;
+    let quote = null;
+    let escaped = false;
+    for (let i = 1; i < value.length - 1; i++) {
+        const char = value[i];
+        if (quote === '"' && escaped) {
+            escaped = false;
+            continue;
+        }
+        if (quote === '"' && char === '\\') {
+            escaped = true;
+            continue;
+        }
+        if (quote) {
+            if (char === quote)
+                quote = null;
+            continue;
+        }
+        if (char === '"' || char === "'")
+            quote = char;
+        else if (char === ',') {
+            const item = tomlString(value.slice(start, i));
+            if (item === null)
+                return null;
+            values.push(item);
+            start = i + 1;
+        }
+    }
+    const tail = value.slice(start, -1).trim();
+    if (tail) {
+        const item = tomlString(tail);
+        if (item === null)
+            return null;
+        values.push(item);
+    }
+    return values;
+}
+function displayName(id) {
+    return id
+        .replace(/[-\s]+/g, ' ')
+        .trim()
+        .replace(/(^| )([a-z])/g, (_whole, prefix, letter) => `${prefix}${letter.toUpperCase()}`);
+}
+function parseLayer(text) {
+    const configs = new Map();
+    let kind = null;
+    let section = [];
+    let multiline = null;
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (multiline) {
+            if (line.includes(multiline))
+                multiline = null;
+            continue;
+        }
+        const header = tablePath(line);
+        if (header) {
+            section = header;
+            if (section.length === 3 && section[0] === 'scripts' && section[1] === 'run') {
+                kind = 'named';
+                const id = section[2];
+                if (!configs.has(id))
+                    configs.set(id, { id });
+            }
+            continue;
+        }
+        const found = assignment(line);
+        if (!found)
+            continue;
+        multiline = multilineDelimiter(found.value);
+        if (section.length === 1 && section[0] === 'scripts' && found.key === 'run') {
+            if (stringValue(found.value) || multiline)
+                kind = 'legacy';
+            continue;
+        }
+        if (section.length !== 3 || section[0] !== 'scripts' || section[1] !== 'run')
+            continue;
+        const config = configs.get(section[2]) ?? { id: section[2] };
+        configs.set(config.id, config);
+        if (found.key === 'command' && (stringValue(found.value) || multiline))
+            config.command = true;
+        else if (found.key === 'hide' && /^(?:true|false)$/.test(found.value))
+            config.hide = found.value === 'true';
+        else if (found.key === 'available_in') {
+            let value = found.value;
+            while (arrayOpen(value) && i + 1 < lines.length)
+                value += `\n${lines[++i]}`;
+            const available = stringList(value);
+            if (available)
+                config.availableIn = available;
+        }
+    }
+    return { kind, configs: [...configs.values()] };
+}
+function resolveLayers(layers) {
+    let kind = null;
+    const resolved = new Map();
+    for (const layer of layers) {
+        if (layer.kind === 'legacy') {
+            kind = 'legacy';
+            resolved.clear();
+            continue;
+        }
+        if (layer.kind !== 'named')
+            continue;
+        if (kind === 'legacy')
+            resolved.clear();
+        kind = 'named';
+        for (const patch of layer.configs) {
+            const previous = resolved.get(patch.id) ?? { id: patch.id };
+            resolved.set(patch.id, { ...previous, ...patch });
+        }
+    }
+    if (kind !== 'named')
+        return [];
+    return [...resolved.values()].flatMap(config => {
+        if (!config.command || config.hide || (config.availableIn && !config.availableIn.includes('local')))
+            return [];
+        return [{ id: config.id, name: displayName(config.id) }];
+    });
+}
+/** Resolve lower-to-higher-priority TOML layers using Conductor's per-ID merge. */
+export function resolveRunConfigs(layers) {
+    return resolveLayers(layers.map(parseLayer));
+}
+function readLayer(file) {
+    try {
+        const stat = fs.statSync(file);
+        const cached = fileCache.get(file);
+        if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size)
+            return cached.value;
+        const source = fs.readFileSync(file, 'utf8');
+        const value = parseLayer(source);
+        fileCache.set(file, { mtimeMs: stat.mtimeMs, size: stat.size, value });
+        return value;
+    }
+    catch {
+        fileCache.delete(file);
+        return null;
+    }
+}
+/** Read the same user -> shared -> local -> managed settings layers Conductor resolves. */
+export function runConfigsFor(workspace) {
+    const shared = workspace.worktree
+        ? path.join(workspace.worktree, '.conductor', 'settings.toml')
+        : workspace.repo_root
+            ? path.join(workspace.repo_root, '.conductor', 'settings.toml')
+            : null;
+    const files = [
+        path.join(os.homedir(), '.conductor', 'settings.toml'),
+        shared,
+        workspace.repo_root ? path.join(workspace.repo_root, '.conductor', 'settings.local.toml') : null,
+        workspace.worktree ? path.join(workspace.worktree, '.conductor', 'settings.local.toml') : null,
+        path.join(os.homedir(), '.conductor', 'settings.managed.toml')
+    ];
+    return resolveLayers(files.flatMap(file => (file ? [readLayer(file)].filter((layer) => layer !== null) : [])));
+}

package/dist-node/src/search-worker.js

@@ -0,0 +1,269 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { parentPort, workerData } from 'node:worker_threads';
+import { ConductorDb } from "./db.js";
+import { HIT_CLOSE, HIT_OPEN, matchQuery } from "./search.js";
+import { parseMessage } from "./transcript.js";
+/** Bump when extraction or an invariant such as one-chunk-per-source changes stored rows. */
+const SCHEMA_VERSION = 2;
+/**
+ * Source rows advanced per tick. The cursor moves by *scanned* rowid rather than
+ * matched rowid, so a caught-up index re-scans nothing.
+ */
+const WINDOW_ROWS = 4000;
+const BACKFILL_PAUSE_MS = 5;
+const IDLE_POLL_MS = 15_000;
+const MAX_CHUNK_CHARS = 64_000;
+const CHUNK_LIMIT = 300;
+const SLOW_OPERATION_MS = 100;
+const SLOW_LOG_INTERVAL_MS = 60_000;
+const INDEXED_ROLES = new Set(['user', 'assistant', 'thinking']);
+const port = parentPort;
+if (!port)
+    throw new Error('search worker requires a parent port');
+const post = (message) => port.postMessage(message);
+const log = (level, message) => post({ type: 'log', level, message });
+const errorText = (error) => (error instanceof Error ? error.message : String(error));
+class SearchIndexWorker {
+    source;
+    file;
+    db = null;
+    cursor = 0;
+    chunks = 0;
+    caughtUp = false;
+    timer = null;
+    sourceMax = 0;
+    slowLogs = new Map();
+    constructor(sourceDbPath, file) {
+        this.source = new ConductorDb(sourceDbPath, { onSlowQuery: message => log('warn', message) });
+        this.file = file;
+    }
+    open() {
+        fs.mkdirSync(path.dirname(this.file), { recursive: true });
+        const db = new DatabaseSync(this.file);
+        // Set the wait before any pragma or schema operation that may need the writer.
+        // A second dev relay can own it for a batch; this wait is why the connection must
+        // live off the HTTP thread.
+        db.exec('PRAGMA busy_timeout = 5000');
+        db.exec('PRAGMA journal_mode = WAL');
+        db.exec('PRAGMA synchronous = NORMAL');
+        db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)');
+        const version = Number(this.readMeta(db, 'version') ?? 0);
+        if (version !== SCHEMA_VERSION) {
+            db.exec('DROP TABLE IF EXISTS chunks');
+            db.exec(`
+				CREATE VIRTUAL TABLE chunks USING fts5(
+					body,
+					session_id UNINDEXED,
+					src_rowid UNINDEXED,
+					role UNINDEXED,
+					at UNINDEXED,
+					tokenize='porter unicode61'
+				)
+			`);
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('version', String(SCHEMA_VERSION));
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', '0');
+            if (version)
+                log('log', `search index schema ${version} → ${SCHEMA_VERSION}, rebuilding`);
+        }
+        this.db = db;
+        this.cursor = Number(this.readMeta(db, 'cursor') ?? 0);
+        this.chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
+    }
+    start() {
+        this.schedule(0);
+    }
+    stop() {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = null;
+    }
+    status() {
+        if (!this.db)
+            return { chunks: 0, ready: false, progress: 0 };
+        if (this.caughtUp)
+            return { chunks: this.chunks, ready: true, progress: 1 };
+        if (!this.sourceMax) {
+            const max = this.source.query('SELECT MAX(rowid) m FROM session_messages')[0]?.m;
+            this.sourceMax = max ?? 0;
+        }
+        const progress = this.sourceMax ? Math.min(1, this.cursor / this.sourceMax) : 0;
+        return { chunks: this.chunks, ready: false, progress };
+    }
+    search(raw, { limit = CHUNK_LIMIT, sessionIds } = {}) {
+        const started = performance.now();
+        try {
+            const db = this.db;
+            if (!db)
+                return [];
+            const match = matchQuery(raw);
+            if (!match || (sessionIds && !sessionIds.length))
+                return [];
+            const scope = sessionIds ? 'AND session_id IN (SELECT value FROM json_each(?))' : '';
+            const params = sessionIds
+                ? [HIT_OPEN, HIT_CLOSE, match, JSON.stringify(sessionIds), limit]
+                : [HIT_OPEN, HIT_CLOSE, match, limit];
+            let rows;
+            try {
+                rows = db
+                    .prepare(`SELECT session_id, src_rowid, role, at, -bm25(chunks) AS score,
+						        snippet(chunks, 0, ?, ?, '…', 24) AS snippet
+						 FROM chunks WHERE chunks MATCH ? ${scope} ORDER BY bm25(chunks) LIMIT ?`)
+                    .all(...params);
+            }
+            catch (error) {
+                throw new Error(`search failed for ${JSON.stringify(match)}: ${errorText(error)}`);
+            }
+            return rows.map(row => ({
+                sessionId: row.session_id,
+                srcRowid: Number(row.src_rowid),
+                role: INDEXED_ROLES.has(row.role) ? row.role : 'assistant',
+                at: row.at,
+                score: Number(row.score),
+                snippet: row.snippet
+            }));
+        }
+        finally {
+            this.reportSlow('query', performance.now() - started);
+        }
+    }
+    readMeta(db, key) {
+        const row = db.prepare('SELECT v FROM meta WHERE k = ?').get(key);
+        return row?.v ?? null;
+    }
+    schedule(ms) {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+            const started = performance.now();
+            let more = false;
+            try {
+                more = this.tick();
+            }
+            catch (error) {
+                log('warn', `⚠ search index tick failed: ${errorText(error)}`);
+            }
+            finally {
+                this.reportSlow('backfill batch', performance.now() - started);
+            }
+            try {
+                post({ type: 'status', status: this.status() });
+            }
+            catch (error) {
+                log('warn', `⚠ search index status failed: ${errorText(error)}`);
+            }
+            this.schedule(more ? BACKFILL_PAUSE_MS : IDLE_POLL_MS);
+        }, ms);
+        this.timer.unref?.();
+    }
+    /**
+     * Index one source-row window. Filtering happens after the rowid window is chosen,
+     * so the cursor advances across tool-only rows instead of scanning them forever.
+     */
+    tick() {
+        const db = this.db;
+        if (!db)
+            return false;
+        const startingCursor = this.cursor;
+        const window = this.source.query('SELECT rowid FROM session_messages WHERE rowid > ? ORDER BY rowid LIMIT ?', [startingCursor, WINDOW_ROWS]);
+        if (!window.length) {
+            this.caughtUp = true;
+            return false;
+        }
+        this.caughtUp = false;
+        const end = window[window.length - 1].rowid;
+        const rows = this.source.query(`SELECT rowid, id, session_id, role, content, full_message, created_at, sent_at, queue_order
+			 FROM session_messages
+			 WHERE rowid > ? AND rowid <= ? AND session_id IS NOT NULL
+			   AND (role = 'user' OR content LIKE '%"type":"text"%' OR content LIKE '%"type":"thinking"%')
+			 ORDER BY rowid`, [startingCursor, end]);
+        // Parse outside the sidecar write transaction. A second relay may be waiting for
+        // that writer, and parsing transcript JSON does not need to hold it.
+        const chunks = [];
+        for (const row of rows) {
+            for (const entry of parseMessage(row, null)) {
+                if (!INDEXED_ROLES.has(entry.role))
+                    continue;
+                const body = entry.text.trim();
+                if (!body)
+                    continue;
+                chunks.push({
+                    body: body.slice(0, MAX_CHUNK_CHARS),
+                    sessionId: row.session_id,
+                    srcRowid: row.rowid,
+                    role: entry.role,
+                    at: entry.ts
+                });
+            }
+        }
+        // BEGIN IMMEDIATE serialises the cursor check with the batch write. Without the
+        // check, a dev relay and the service can both read cursor N, wait for each other,
+        // and then insert the same N→M chunks twice.
+        db.exec('BEGIN IMMEDIATE');
+        try {
+            const durableCursor = Number(this.readMeta(db, 'cursor') ?? 0);
+            if (durableCursor !== startingCursor) {
+                db.exec('ROLLBACK');
+                this.cursor = durableCursor;
+                this.chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
+                return true;
+            }
+            const insert = db.prepare('INSERT INTO chunks(body, session_id, src_rowid, role, at) VALUES (?, ?, ?, ?, ?)');
+            for (const chunk of chunks) {
+                insert.run(chunk.body, chunk.sessionId, chunk.srcRowid, chunk.role, chunk.at);
+            }
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', String(end));
+            db.exec('COMMIT');
+        }
+        catch (error) {
+            try {
+                db.exec('ROLLBACK');
+            }
+            catch {
+                // Preserve the original SQLite error.
+            }
+            throw error;
+        }
+        this.cursor = end;
+        this.chunks += chunks.length;
+        return true;
+    }
+    reportSlow(operation, elapsedMs) {
+        if (elapsedMs < SLOW_OPERATION_MS)
+            return;
+        const now = Date.now();
+        const previous = this.slowLogs.get(operation);
+        if (previous && now - previous.at < SLOW_LOG_INTERVAL_MS) {
+            previous.suppressed++;
+            return;
+        }
+        const suppressed = previous?.suppressed ? `; ${previous.suppressed} similar calls suppressed` : '';
+        log('warn', `⚠ slow search index ${operation} (${Math.round(elapsedMs)}ms${suppressed})`);
+        this.slowLogs.set(operation, { at: now, suppressed: 0 });
+    }
+}
+const config = workerData;
+let index = null;
+try {
+    index = new SearchIndexWorker(config.sourceDbPath, config.file);
+    index.open();
+    post({ type: 'status', status: index.status() });
+    index.start();
+}
+catch (error) {
+    const message = errorText(error);
+    log('warn', `⚠ search index unavailable (${message}) — /api/search will report it`);
+    post({ type: 'status', status: { chunks: 0, ready: false, progress: 0, error: message } });
+}
+port.on('message', (request) => {
+    if (request.type !== 'search')
+        return;
+    try {
+        post({ type: 'result', id: request.id, hits: index?.search(request.raw, request.options) ?? [] });
+    }
+    catch (error) {
+        post({ type: 'error', id: request.id, error: errorText(error) });
+    }
+});
+process.once('exit', () => index?.stop());