fraim-hub 2.0.253 → 2.0.255

package/dist/src/ai-hub/atomic-json-file.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renameWithRetry = renameWithRetry;
+exports.writeJsonAtomic = writeJsonAtomic;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+// Atomic JSON file write for the conversation store and its derived caches.
+//
+// Extracted in issue #1065. Both the conversation store and the searchable projection need the
+// same write-temp-then-rename dance, and the same Windows-specific retry around it. Keeping two
+// copies meant a future fix to the retry behaviour would land in one and silently not the other,
+// which is exactly the drift the store's #820 durability work exists to prevent.
+const RENAME_ATTEMPTS = 25;
+const RENAME_RETRY_DELAY_MS = 10;
+let tempSeq = 0;
+/**
+ * Rename, retrying the Windows contention codes.
+ *
+ * Windows can throw EPERM/EBUSY/EACCES when a reader briefly holds the destination open. Callers
+ * already hold the per-bucket advisory lock against other writers, so a short bounded retry is
+ * enough; a genuine error still propagates.
+ */
+function renameWithRetry(from, to) {
+    for (let attempt = 0; attempt < RENAME_ATTEMPTS; attempt += 1) {
+        try {
+            fs_1.default.renameSync(from, to);
+            return;
+        }
+        catch (error) {
+            const code = error.code;
+            if ((code === 'EPERM' || code === 'EBUSY' || code === 'EACCES') && attempt < RENAME_ATTEMPTS - 1) {
+                // Synchronous sleep without a busy spin: the store's write path is synchronous.
+                Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RENAME_RETRY_DELAY_MS);
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+/**
+ * Serialize `value` to `filePath` atomically: write a uniquely-named temp file beside it, then
+ * rename over the destination. A reader therefore sees either the old file or the new one, never a
+ * half-written one. The temp name carries the pid and a per-process sequence so two processes
+ * writing the same destination cannot collide on the temp file itself.
+ */
+function writeJsonAtomic(filePath, value) {
+    fs_1.default.mkdirSync(path_1.default.dirname(filePath), { recursive: true });
+    const tempPath = `${filePath}.${process.pid}.${tempSeq++}.tmp`;
+    fs_1.default.writeFileSync(tempPath, JSON.stringify(value), 'utf8');
+    try {
+        renameWithRetry(tempPath, filePath);
+    }
+    catch (error) {
+        // Do not leave the temp file behind for a scanner to trip over.
+        try {
+            fs_1.default.rmSync(tempPath, { force: true });
+        }
+        catch { /* best effort */ }
+        throw error;
+    }
+}

package/dist/src/ai-hub/conversation-search-index.js

@@ -0,0 +1,232 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConversationSearchIndex = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const conversation_search_projection_1 = require("./conversation-search-projection");
+const conversation_search_state_1 = require("./conversation-search-state");
+function timestampValue(value) {
+    if (typeof value === 'number')
+        return Number.isFinite(value) ? value : 0;
+    if (typeof value === 'string') {
+        const parsed = Date.parse(value);
+        if (Number.isFinite(parsed))
+            return parsed;
+        const numeric = Number(value);
+        return Number.isFinite(numeric) ? numeric : 0;
+    }
+    return 0;
+}
+function text(value) {
+    return typeof value === 'string' ? value : '';
+}
+class ConversationSearchIndex {
+    constructor(store) {
+        this.store = store;
+        this.fileCache = new Map();
+        this.pendingByBucket = new Map();
+        this.building = false;
+        this.disposed = false;
+    }
+    /** Stop the background indexer. Called from server shutdown so it cannot outlive the Hub. */
+    dispose() {
+        this.disposed = true;
+        this.pendingByBucket.clear();
+    }
+    collect(targets) {
+        const items = [];
+        const headerOnlyScopes = [];
+        let conversationsSearched = 0;
+        let pendingConversations = 0;
+        for (const target of targets) {
+            const headers = this.safeHeaders(target.bucketKey);
+            if (headers.length === 0)
+                continue;
+            const projection = this.readProjection(target.bucketKey);
+            if (projection.unreadable > 0 && !headerOnlyScopes.includes(target.projectLabel)) {
+                headerOnlyScopes.push(target.projectLabel);
+            }
+            // Two different conditions, deliberately counted differently. A conversation with NO entry
+            // has had no thread text searched, so it is genuinely pending and R30 must say so. A
+            // conversation whose entry is merely stamped older than its header did have its thread
+            // searched; rebuilding it is a refresh, not an incomplete result, so it must not make the UI
+            // claim it is still indexing.
+            const rebuildIds = [];
+            for (const header of headers) {
+                conversationsSearched += 1;
+                const entry = projection.entries.get(header.id);
+                if (!entry) {
+                    pendingConversations += 1;
+                    rebuildIds.push(header.id);
+                }
+                else if (timestampValue(header.lastUpdatedAt) > timestampValue(entry.lastUpdatedAt)) {
+                    rebuildIds.push(header.id);
+                }
+                items.push(this.toSearchable(target, header, entry));
+            }
+            if (rebuildIds.length > 0)
+                this.queueBuild(target.bucketKey, rebuildIds);
+        }
+        return { items, conversationsSearched, pendingConversations, headerOnlyScopes };
+    }
+    // ---- internals ----
+    safeHeaders(bucketKey) {
+        try {
+            return this.store.loadProjectHeaders(bucketKey);
+        }
+        catch {
+            return [];
+        }
+    }
+    toSearchable(target, header, entry) {
+        return {
+            id: header.id,
+            projectPath: text(header.projectPath),
+            bucketKey: target.bucketKey,
+            scope: target.scope,
+            projectLabel: target.projectLabel,
+            title: text(header.title),
+            jobId: text(header.jobId),
+            jobTitle: text(header.jobTitle),
+            employeeLabel: (0, conversation_search_projection_1.conversationSpeakerLabel)(header),
+            personaKey: text(header.personaKey),
+            issueNumber: text(header.issueNumber),
+            state: (0, conversation_search_state_1.conversationSearchState)(header),
+            lastUpdatedAt: timestampValue(header.lastUpdatedAt),
+            artifacts: entry ? entry.artifacts : [],
+            messages: entry ? entry.messages : [],
+            threadTruncated: entry ? entry.threadTruncated : false,
+            threadIndexed: Boolean(entry),
+        };
+    }
+    /**
+     * Read a bucket's projection, reusing already-parsed sidecars.
+     *
+     * The memo is keyed per FILE on mtime plus byte size, not per directory. A directory-level memo
+     * (dir mtime + file count) looked cheaper but is unsafe: replacing an existing sidecar leaves the
+     * file count unchanged, so a coarse mtime clock could serve stale thread text for a conversation
+     * that was just patched, and R28 says a run that just finished must be findable. Per-file keys
+     * mean an overwritten sidecar is re-parsed unless its mtime AND its length are both identical, and
+     * `collect` additionally treats any entry older than its header as pending so even that residual
+     * case self-heals rather than going unnoticed.
+     */
+    readProjection(bucketKey) {
+        const dir = (0, conversation_search_projection_1.searchProjectionDirPath)(this.store.bucketDirectory(bucketKey));
+        let files;
+        try {
+            files = fs_1.default.readdirSync(dir);
+        }
+        catch {
+            return { entries: new Map(), unreadable: 0, missing: true };
+        }
+        const entries = new Map();
+        let unreadable = 0;
+        const seen = new Set();
+        for (const file of files) {
+            if (!file.endsWith('.json'))
+                continue;
+            const filePath = path_1.default.join(dir, file);
+            seen.add(filePath);
+            let mtimeMs = -1;
+            let size = -1;
+            try {
+                const stat = fs_1.default.statSync(filePath);
+                mtimeMs = stat.mtimeMs;
+                size = stat.size;
+            }
+            catch {
+                unreadable += 1;
+                continue;
+            }
+            const cached = this.fileCache.get(filePath);
+            if (cached && cached.mtimeMs === mtimeMs && cached.size === size) {
+                if (cached.entry)
+                    entries.set(cached.entry.id, cached.entry);
+                else
+                    unreadable += 1;
+                continue;
+            }
+            let parsed = null;
+            try {
+                const candidate = JSON.parse(fs_1.default.readFileSync(filePath, 'utf8'));
+                if (candidate && typeof candidate.id === 'string' && Array.isArray(candidate.messages))
+                    parsed = candidate;
+            }
+            catch {
+                parsed = null;
+            }
+            this.fileCache.set(filePath, { mtimeMs, size, entry: parsed });
+            if (parsed)
+                entries.set(parsed.id, parsed);
+            else
+                unreadable += 1;
+        }
+        // Drop memo entries for sidecars that are gone, so a pruned conversation cannot linger.
+        for (const filePath of Array.from(this.fileCache.keys())) {
+            if (filePath.startsWith(dir) && !seen.has(filePath))
+                this.fileCache.delete(filePath);
+        }
+        return { entries, unreadable, missing: false };
+    }
+    queueBuild(bucketKey, ids) {
+        if (this.disposed)
+            return;
+        let set = this.pendingByBucket.get(bucketKey);
+        if (!set) {
+            set = new Set();
+            this.pendingByBucket.set(bucketKey, set);
+        }
+        for (const id of ids)
+            set.add(id);
+        this.startBuildLoop();
+    }
+    /**
+     * One conversation per event-loop turn. A self-rescheduling `setImmediate` defers the next
+     * conversation to the following loop iteration, so the poll phase runs in between and ordinary
+     * Hub requests are answered while a large store indexes for the first time (R31).
+     */
+    startBuildLoop() {
+        if (this.building || this.disposed)
+            return;
+        this.building = true;
+        const step = () => {
+            if (this.disposed) {
+                this.building = false;
+                return;
+            }
+            const next = this.takeNextPending();
+            if (!next) {
+                this.building = false;
+                return;
+            }
+            try {
+                this.store.indexConversationForSearch(next.bucketKey, next.id);
+            }
+            catch {
+                /* a single unreadable conversation must not stall the whole index */
+            }
+            const handle = setImmediate(step);
+            if (typeof handle.unref === 'function')
+                handle.unref();
+        };
+        const handle = setImmediate(step);
+        if (typeof handle.unref === 'function')
+            handle.unref();
+    }
+    takeNextPending() {
+        for (const [bucketKey, ids] of this.pendingByBucket) {
+            for (const id of ids) {
+                ids.delete(id);
+                if (ids.size === 0)
+                    this.pendingByBucket.delete(bucketKey);
+                return { bucketKey, id };
+            }
+            this.pendingByBucket.delete(bucketKey);
+        }
+        return null;
+    }
+}
+exports.ConversationSearchIndex = ConversationSearchIndex;

package/dist/src/ai-hub/conversation-search-projection.js

@@ -0,0 +1,229 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_RETAINED_THREAD_CHARS = exports.MAX_RETAINED_MESSAGE_CHARS = exports.SEARCH_PROJECTION_VERSION = exports.SEARCH_PROJECTION_DIR_NAME = void 0;
+exports.conversationSpeakerLabel = conversationSpeakerLabel;
+exports.artifactSearchNames = artifactSearchNames;
+exports.buildConversationSearchEntry = buildConversationSearchEntry;
+exports.searchProjectionDirPath = searchProjectionDirPath;
+exports.searchProjectionFileName = searchProjectionFileName;
+exports.searchProjectionEntryPath = searchProjectionEntryPath;
+exports.writeConversationSearchEntry = writeConversationSearchEntry;
+exports.readConversationSearchEntries = readConversationSearchEntries;
+exports.pruneConversationSearchEntries = pruneConversationSearchEntries;
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const atomic_json_file_1 = require("./atomic-json-file");
+// Issue #1065: the searchable projection.
+//
+// Search reads the manager-visible thread and the header facts; it never reads the raw host
+// event log, which is 91% of the stored bytes and is tool-call noise (measured: one ordinary
+// term matched 25 conversations against titles + thread text, and 181 of 224 against the raw
+// JSON including events). Header facts are already in the bucket's `index.json`, so this
+// projection is purely additive: it carries the bounded thread text plus artifact names.
+//
+// Layout follows the established `index.json` discipline (see conversation-store.ts): a derived,
+// rebuildable cache that is never the source of truth. It is stored one file per conversation
+// under `<bucketDir>/search/`, so a write costs one conversation rather than a whole-bucket
+// rewrite, and so project removal (which recursively deletes the bucket directory) removes the
+// projection by construction rather than through a second code path.
+exports.SEARCH_PROJECTION_DIR_NAME = 'search';
+exports.SEARCH_PROJECTION_VERSION = 1;
+/** Per-message cap. A single run message can be megabytes; only its head is searchable. */
+exports.MAX_RETAINED_MESSAGE_CHARS = 8000;
+/**
+ * Per-conversation cap. Conversation size is extremely skewed (p50 1.9 KB, p99 4.1 MB), so a
+ * per-conversation bound is what stops one pathological run from dominating the corpus.
+ */
+exports.MAX_RETAINED_THREAD_CHARS = 64 * 1024;
+const MANAGER_SPEAKER_LABEL = 'You';
+function cleanText(value) {
+    return typeof value === 'string' ? value : '';
+}
+/** The employee's display name, preferring the persona snapshot the conversation carries. */
+function conversationSpeakerLabel(conv) {
+    const snapshot = conv.personaSnapshot;
+    const display = cleanText(snapshot?.displayName).trim();
+    if (display)
+        return display;
+    const agentLabel = cleanText(conv.configuredAgentLabel).trim();
+    if (agentLabel)
+        return agentLabel;
+    const personaKey = cleanText(conv.personaKey).trim();
+    if (personaKey)
+        return personaKey.replace(/^custom:/, '');
+    return cleanText(conv.agentName).trim() || 'Employee';
+}
+function normalizeRole(raw) {
+    const role = cleanText(raw).trim().toLowerCase();
+    if (role === 'manager' || role === 'user' || role === 'human')
+        return 'manager';
+    if (role === 'employee' || role === 'assistant' || role === 'agent')
+        return 'employee';
+    return 'system';
+}
+function artifactSearchNames(conv) {
+    const raw = Array.isArray(conv.artifacts) ? conv.artifacts : [];
+    const out = [];
+    for (const artifact of raw) {
+        if (!artifact || typeof artifact !== 'object')
+            continue;
+        const value = artifact;
+        for (const field of ['label', 'path', 'url', 'localPath', 'name']) {
+            const text = cleanText(value[field]).trim();
+            if (text && !out.includes(text))
+                out.push(text);
+        }
+    }
+    return out;
+}
+/**
+ * Build the searchable projection for one conversation.
+ *
+ * Retention rule (R29): the first message is always kept, because the kickoff instruction is
+ * what a manager is most likely to remember. The remaining budget is filled newest-first (a
+ * run's conclusion is the next most memorable thing) and the retained set is then re-ordered
+ * chronologically so snippets read in thread order.
+ */
+function buildConversationSearchEntry(conv) {
+    const rawMessages = Array.isArray(conv.messages) ? conv.messages : [];
+    const employeeLabel = conversationSpeakerLabel(conv);
+    const candidates = [];
+    for (let index = 0; index < rawMessages.length; index += 1) {
+        const raw = rawMessages[index];
+        if (!raw || typeof raw !== 'object')
+            continue;
+        const text = cleanText(raw.text);
+        if (!text.trim())
+            continue;
+        const role = normalizeRole(raw.role);
+        const clipped = text.length > exports.MAX_RETAINED_MESSAGE_CHARS;
+        candidates.push({
+            index,
+            clipped,
+            message: {
+                role,
+                who: role === 'manager' ? MANAGER_SPEAKER_LABEL : (role === 'employee' ? employeeLabel : 'System'),
+                text: clipped ? text.slice(0, exports.MAX_RETAINED_MESSAGE_CHARS) : text,
+            },
+        });
+    }
+    const keptIndexes = new Set();
+    let budget = exports.MAX_RETAINED_THREAD_CHARS;
+    let clippedAny = false;
+    const take = (candidate) => {
+        if (keptIndexes.has(candidate.index))
+            return true;
+        if (candidate.message.text.length > budget)
+            return false;
+        keptIndexes.add(candidate.index);
+        budget -= candidate.message.text.length;
+        if (candidate.clipped)
+            clippedAny = true;
+        return true;
+    };
+    if (candidates.length > 0) {
+        // The kickoff always wins the first slice of the budget.
+        take(candidates[0]);
+        for (let cursor = candidates.length - 1; cursor >= 1; cursor -= 1) {
+            if (!take(candidates[cursor]))
+                break;
+        }
+    }
+    const retained = candidates
+        .filter((candidate) => keptIndexes.has(candidate.index))
+        .sort((a, b) => a.index - b.index);
+    return {
+        version: exports.SEARCH_PROJECTION_VERSION,
+        id: cleanText(conv.id),
+        lastUpdatedAt: conv.lastUpdatedAt ?? conv.createdAt ?? 0,
+        messages: retained.map((candidate) => candidate.message),
+        artifacts: artifactSearchNames(conv),
+        threadTruncated: clippedAny || retained.length < candidates.length,
+        threadMessageCount: candidates.length,
+    };
+}
+// ---- disk layout -----------------------------------------------------------
+function searchProjectionDirPath(bucketDir) {
+    return path_1.default.join(bucketDir, exports.SEARCH_PROJECTION_DIR_NAME);
+}
+/** Same hashing as the conversation shard so the two files are trivially correlated. */
+function searchProjectionFileName(conversationId) {
+    return `${crypto_1.default.createHash('sha256').update(conversationId).digest('hex').slice(0, 40)}.json`;
+}
+function searchProjectionEntryPath(bucketDir, conversationId) {
+    return path_1.default.join(searchProjectionDirPath(bucketDir), searchProjectionFileName(conversationId));
+}
+/** Best-effort: the projection is a cache, so a failed write must never fail the conversation write. */
+function writeConversationSearchEntry(bucketDir, conv) {
+    const id = cleanText(conv.id);
+    if (!id)
+        return;
+    try {
+        (0, atomic_json_file_1.writeJsonAtomic)(searchProjectionEntryPath(bucketDir, id), buildConversationSearchEntry(conv));
+    }
+    catch {
+        /* the projection self-heals on the next read */
+    }
+}
+function readConversationSearchEntries(bucketDir) {
+    const dir = searchProjectionDirPath(bucketDir);
+    let files;
+    try {
+        files = fs_1.default.readdirSync(dir);
+    }
+    catch {
+        return { entries: new Map(), unreadable: 0, missing: true };
+    }
+    const entries = new Map();
+    let unreadable = 0;
+    for (const file of files) {
+        if (!file.endsWith('.json') || file.endsWith('.tmp'))
+            continue;
+        try {
+            const parsed = JSON.parse(fs_1.default.readFileSync(path_1.default.join(dir, file), 'utf8'));
+            if (parsed && typeof parsed.id === 'string' && Array.isArray(parsed.messages)) {
+                entries.set(parsed.id, parsed);
+            }
+            else {
+                unreadable += 1;
+            }
+        }
+        catch {
+            unreadable += 1;
+        }
+    }
+    return { entries, unreadable, missing: false };
+}
+/**
+ * Drop projection entries for conversations that no longer exist (C1). Called from the same
+ * whole-bucket rewrite that prunes conversation shards, so deletion reaches every copy.
+ */
+function pruneConversationSearchEntries(bucketDir, desiredIds) {
+    const dir = searchProjectionDirPath(bucketDir);
+    let files;
+    try {
+        files = fs_1.default.readdirSync(dir);
+    }
+    catch {
+        return;
+    }
+    const keep = new Set();
+    for (const id of desiredIds)
+        keep.add(searchProjectionFileName(id));
+    for (const file of files) {
+        if (!file.endsWith('.json'))
+            continue;
+        if (keep.has(file))
+            continue;
+        try {
+            fs_1.default.rmSync(path_1.default.join(dir, file), { force: true });
+        }
+        catch {
+            /* best-effort */
+        }
+    }
+}

package/dist/src/ai-hub/conversation-search-state.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.conversationSearchState = conversationSearchState;
+exports.conversationSearchStateLabel = conversationSearchStateLabel;
+exports.conversationSearchStateDot = conversationSearchStateDot;
+exports.resolveSearchState = resolveSearchState;
+exports.isKnownSearchState = isKnownSearchState;
+const SEARCH_STATE_ALIASES = {
+    working: 'working',
+    running: 'working',
+    waiting: 'waiting',
+    complete: 'complete',
+    completed: 'complete',
+    done: 'complete',
+    stopped: 'stopped',
+    failed: 'failed',
+    error: 'failed',
+    idle: 'idle',
+};
+/**
+ * Server-side mirror of the Hub client's `conversationUiState`, so `is:` filters and the result
+ * row's state agree with what the employee tree shows. The client's `error` state is named
+ * `failed` here because that is the vocabulary the spec's `is:` facet uses.
+ */
+function conversationSearchState(conv) {
+    if (!conv)
+        return 'idle';
+    if (conv.status === 'running')
+        return 'working';
+    switch (conv.pauseReason) {
+        case 'working': return 'working';
+        case 'done': return 'complete';
+        case 'error': return 'failed';
+        case 'awaiting_review': return 'waiting';
+        case 'awaiting_user': return 'waiting';
+        case 'stopped': return 'stopped';
+        default: break;
+    }
+    if (conv.stopped && conv.status === 'failed')
+        return 'stopped';
+    if (conv.status === 'failed')
+        return 'waiting';
+    if (conv.status === 'completed' && conv.reviewApproved && conv.pauseReason === undefined)
+        return 'complete';
+    if (conv.status === 'completed')
+        return 'waiting';
+    return 'idle';
+}
+function conversationSearchStateLabel(state) {
+    switch (state) {
+        case 'working': return 'Working';
+        case 'waiting': return 'Waiting on you';
+        case 'complete': return 'Done';
+        case 'stopped': return 'Stopped';
+        case 'failed': return 'Failed';
+        default: return 'Idle';
+    }
+}
+function conversationSearchStateDot(state) {
+    switch (state) {
+        case 'working': return 'amber';
+        case 'waiting': return 'red';
+        case 'complete': return 'green';
+        case 'stopped': return 'amber-static';
+        case 'failed': return 'red';
+        default: return 'grey';
+    }
+}
+/** Resolve an `is:` token to a state, or null when the token names no known state. */
+function resolveSearchState(token) {
+    return SEARCH_STATE_ALIASES[token] ?? null;
+}
+/** Whether a value is one of the states `is:` accepts. Drives the unrecognised-filter notice. */
+function isKnownSearchState(value) {
+    return Object.prototype.hasOwnProperty.call(SEARCH_STATE_ALIASES, value);
+}