fraim-hub 2.0.317 → 2.0.319

package/dist/src/ai-hub/atomic-json-file.js

@@ -14,8 +14,8 @@ const path_1 = __importDefault(require("path"));
 // same write-temp-then-rename dance, and the same Windows-specific retry around it. Keeping two
 // copies meant a future fix to the retry behaviour would land in one and silently not the other,
 // which is exactly the drift the store's #820 durability work exists to prevent.
-const RENAME_ATTEMPTS = 25;
-const RENAME_RETRY_DELAY_MS = 10;
+const RENAME_ATTEMPTS = process.platform === 'win32' ? 250 : 25;
+const RENAME_RETRY_DELAY_MS = process.platform === 'win32' ? 20 : 10;
 let tempSeq = 0;
 /**
  * Rename, retrying the Windows contention codes.
@@ -66,7 +66,25 @@ function writeFileAtomic(filePath, data, encoding) {
         fs_1.default.writeFileSync(tempPath, data);
     }
     try {
+        const fd = fs_1.default.openSync(tempPath, 'r+');
+        try {
+            fs_1.default.fsyncSync(fd);
+        }
+        finally {
+            fs_1.default.closeSync(fd);
+        }
         renameWithRetry(tempPath, filePath);
+        // POSIX supports directory fsync for rename durability. Windows does not expose
+        // a directory flush through Node; file data is flushed, rename is atomic.
+        if (process.platform !== 'win32') {
+            const directory = fs_1.default.openSync(path_1.default.dirname(filePath), 'r');
+            try {
+                fs_1.default.fsyncSync(directory);
+            }
+            finally {
+                fs_1.default.closeSync(directory);
+            }
+        }
     }
     catch (error) {
         // Do not leave the temp file behind for a scanner to trip over.

package/dist/src/ai-hub/catalog.js

@@ -492,7 +492,11 @@ function loadJobPhases(jobId, projectPath, discriminant = 'feature') {
         const phaseDef = fm.phases[cursor];
         if (!phaseDef)
             break;
-        cursor = (0, resolve_phase_edge_1.resolvePhaseEdge)(fm.phases[cursor]?.onSuccess, discriminant);
+        if ((0, resolve_phase_edge_1.isRecurrencePhase)(phaseDef)) {
+            ordered.splice(ordered.length - 1, 1, phaseDef.setup.phase, phaseDef.recur.phase, phaseDef.done.phase);
+            break;
+        }
+        cursor = (0, resolve_phase_edge_1.resolvePhaseEdge)(phaseDef.onSuccess, discriminant);
     }
     const labels = fm.phaseLabels || {};
     return ordered.map((id) => ({ id, label: friendlyPhaseLabel(id, labels[id]) }));
@@ -513,6 +517,8 @@ function resolveJobPhaseTransition(jobId, projectPath, phaseId, outcome, discrim
     const phaseDef = fm.phases[phaseId];
     if (!phaseDef)
         return null;
+    if ((0, resolve_phase_edge_1.isRecurrencePhase)(phaseDef))
+        return outcome === 'complete' ? phaseDef.setup.phase : null;
     const edge = outcome === 'complete' ? phaseDef.onSuccess : phaseDef.onFailure;
     return (0, resolve_phase_edge_1.resolvePhaseEdge)(edge, discriminant);
 }
@@ -525,7 +531,18 @@ function loadAllJobPhaseIds(jobId, projectPath) {
         const phases = loadJobPhasesFromSteps(declaredPath);
         return new Set(phases.map((p) => p.id));
     }
-    return new Set(Object.keys(fm.phases));
+    const ids = new Set();
+    for (const [phaseId, phase] of Object.entries(fm.phases)) {
+        if ((0, resolve_phase_edge_1.isRecurrencePhase)(phase)) {
+            ids.add(phase.setup.phase);
+            ids.add(phase.recur.phase);
+            ids.add(phase.done.phase);
+        }
+        else {
+            ids.add(phaseId);
+        }
+    }
+    return ids;
 }
 // Issue #347 — public exposure of the friendly-label rule so callers
 // outside catalog.ts can render labels for phases that exist in the

package/dist/src/ai-hub/conversation-history.js

@@ -0,0 +1,363 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.historyDirectory = exports.historyHash = void 0;
+exports.historySequenceRange = historySequenceRange;
+exports.historyOf = historyOf;
+exports.jsonBytes = jsonBytes;
+exports.historyMessagesNewest = historyMessagesNewest;
+exports.prepareConversationHead = prepareConversationHead;
+exports.messagePage = messagePage;
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const atomic_json_file_1 = require("./atomic-json-file");
+const history_policy_1 = require("./history-policy");
+const conversation_ui_state_1 = require("./conversation-ui-state");
+const REFERENCE_WINDOW = 100;
+function referenceDirectory(bucket, id) { return path_1.default.join((0, exports.historyDirectory)(bucket, id), 'pages'); }
+function readReferencePage(bucket, id, window, revision) {
+    const directory = referenceDirectory(bucket, id);
+    try {
+        let hash = JSON.parse(fs_1.default.readFileSync(path_1.default.join(directory, `${window}.head.json`), 'utf8')).hash;
+        for (let hops = 0; hash && hops < 10000; hops++) {
+            if (!/^[a-f0-9]{64}$/.test(hash))
+                return null;
+            const file = path_1.default.join(directory, `${hash}.json`);
+            if (fs_1.default.statSync(file).size > 32 * 1024)
+                return null;
+            const text = fs_1.default.readFileSync(file, 'utf8');
+            if ((0, exports.historyHash)(text) !== hash)
+                return null;
+            const page = JSON.parse(text);
+            if (page.revision <= revision) {
+                const commit = JSON.parse(fs_1.default.readFileSync(path_1.default.join((0, exports.historyDirectory)(bucket, id), 'commits', `${page.revision}.json`), 'utf8'));
+                if (commit.id === page.commitId)
+                    return { hash, page };
+            }
+            hash = page.previous;
+        }
+    }
+    catch { /* derived page missing/corrupt: caller reads the authoritative chunks */ }
+    return null;
+}
+function indexChunk(bucket, id, hash, messages, revision, commitId) {
+    const windows = new Map();
+    messages.forEach((message, index) => {
+        const window = Math.floor((message.sequence - 1) / REFERENCE_WINDOW);
+        if (!windows.has(window))
+            windows.set(window, []);
+        windows.get(window).push([message.sequence, { chunk: hash, index }]);
+    });
+    for (const [window, additions] of windows) {
+        const previous = readReferencePage(bucket, id, window, revision);
+        const entries = { ...previous?.page.entries };
+        for (const [sequence, ref] of additions)
+            entries[sequence] = ref;
+        const page = { revision, commitId, previous: previous?.hash ?? null, entries };
+        const text = JSON.stringify(page), pageHash = (0, exports.historyHash)(text), directory = referenceDirectory(bucket, id);
+        writeImmutable(path_1.default.join(directory, `${pageHash}.json`), text);
+        (0, atomic_json_file_1.writeFileAtomic)(path_1.default.join(directory, `${window}.head.json`), JSON.stringify({ hash: pageHash }), 'utf8');
+    }
+}
+/** Resolve a sequence range with one small reference page and one chunk cache at a time. */
+function historySequenceRange(bucket, record, first, last) {
+    const resident = new Map((record.messages || []).map(message => [message.sequence, message]));
+    const found = new Map();
+    let window = -1, references, chunkHash = '', chunk, missing = false;
+    for (let sequence = first; sequence <= last; sequence++) {
+        const inline = resident.get(sequence);
+        if (inline) {
+            found.set(sequence, inline);
+            continue;
+        }
+        const nextWindow = Math.floor((sequence - 1) / REFERENCE_WINDOW);
+        if (window !== nextWindow) {
+            window = nextWindow;
+            references = readReferencePage(bucket, record.id, window, Number(record.revision || 0))?.page;
+        }
+        const ref = references?.entries[String(sequence)];
+        if (!ref) {
+            missing = true;
+            continue;
+        }
+        if (chunkHash !== ref.chunk) {
+            chunkHash = ref.chunk;
+            chunk = readChunk(bucket, record.id, ref.chunk);
+        }
+        const message = chunk?.messages[ref.index];
+        if (!message || message.sequence !== sequence) {
+            missing = true;
+            continue;
+        }
+        found.set(sequence, message);
+    }
+    if (missing) {
+        // Never silently omit data when a rebuildable index is absent. One bounded
+        // range is resolved from the linked source, newest correction first.
+        found.clear();
+        for (const message of historyMessagesNewest(bucket, record))
+            if (message.sequence >= first && message.sequence <= last && !found.has(message.sequence))
+                found.set(message.sequence, message);
+    }
+    if (found.size !== last - first + 1)
+        throw new history_policy_1.HistoryStoreError('history_unavailable');
+    return [...found.values()].sort((a, b) => a.sequence - b.sequence);
+}
+const historyHash = (value) => crypto_1.default.createHash('sha256').update(value).digest('hex');
+exports.historyHash = historyHash;
+const historyDirectory = (bucket, id) => path_1.default.join(bucket, 'history', (0, exports.historyHash)(id).slice(0, 40));
+exports.historyDirectory = historyDirectory;
+function historyOf(record) {
+    const h = record?.history;
+    return h?.version === 3 ? h : undefined;
+}
+/** Account JSON bytes recursively before constructing a potentially enormous string. */
+function jsonBytes(value, max = Number.MAX_SAFE_INTEGER) {
+    let bytes = 0;
+    const add = (n) => { bytes += n; if (bytes > max)
+        throw new history_policy_1.HistoryStoreError('history_payload_too_large', 413); };
+    const visit = (v) => {
+        if (v === undefined) {
+            add(4);
+            return;
+        }
+        if (typeof v === 'string') {
+            add(2 + Buffer.byteLength(v));
+            for (let i = 0; i < v.length; i++) {
+                const code = v.charCodeAt(i);
+                if (code < 32)
+                    add(code === 8 || code === 9 || code === 10 || code === 12 || code === 13 ? 1 : 5);
+                else if (code === 34 || code === 92)
+                    add(1);
+            }
+        }
+        else if (Array.isArray(v)) {
+            add(2);
+            v.forEach((x, i) => { if (i)
+                add(1); visit(x); });
+        }
+        else if (v && typeof v === 'object') {
+            add(2);
+            let n = 0;
+            for (const [k, x] of Object.entries(v)) {
+                if (x === undefined)
+                    continue;
+                if (n++)
+                    add(1);
+                visit(k);
+                add(1);
+                visit(x);
+            }
+        }
+        else
+            add(JSON.stringify(v)?.length ?? 4);
+    };
+    visit(value);
+    return bytes;
+}
+function readChunk(bucket, id, hash) {
+    if (!/^[a-f0-9]{64}$/.test(hash))
+        throw new history_policy_1.HistoryStoreError('history_unavailable');
+    try {
+        const file = path_1.default.join((0, exports.historyDirectory)(bucket, id), `${hash}.json`);
+        if (fs_1.default.statSync(file).size > history_policy_1.HISTORY_POLICY.chunkBytes)
+            throw new Error('oversized chunk');
+        const text = fs_1.default.readFileSync(file, 'utf8');
+        if ((0, exports.historyHash)(text) !== hash)
+            throw new Error('checksum');
+        const chunk = JSON.parse(text);
+        if (chunk.version !== 3 || chunk.conversationId !== id || !Array.isArray(chunk.messages))
+            throw new Error('invalid chunk');
+        return chunk;
+    }
+    catch {
+        throw new history_policy_1.HistoryStoreError('history_unavailable');
+    }
+}
+function writeImmutable(file, content) {
+    if (!fs_1.default.existsSync(file))
+        (0, atomic_json_file_1.writeFileAtomic)(file, content, 'utf8');
+}
+function inlineMessage(bucket, id, raw, sequence) {
+    const result = { ...raw, id: typeof raw.id === 'string' ? raw.id : (0, exports.historyHash)(`${sequence}:${raw.role}:${raw.at ?? raw.createdAt}:${raw.text}`), sequence };
+    if (typeof result.text === 'string' && Buffer.byteLength(result.text) > history_policy_1.HISTORY_POLICY.inlineTextBytes) {
+        const hash = (0, exports.historyHash)(result.text);
+        const bytes = Buffer.byteLength(result.text);
+        writeImmutable(path_1.default.join((0, exports.historyDirectory)(bucket, id), 'blobs', hash), result.text);
+        result.contentRef = { hash, bytes };
+        result.text = Buffer.from(result.text).subarray(0, 4096).toString('utf8');
+    }
+    jsonBytes(result, history_policy_1.HISTORY_POLICY.chunkBytes - 1024);
+    return result;
+}
+function appendChunks(bucket, id, messages, tail, revision, commitId) {
+    let pending = [];
+    let bytes = 512;
+    const flush = () => {
+        if (!pending.length)
+            return;
+        const chunk = { version: 3, conversationId: id, previous: tail, messages: pending };
+        const text = JSON.stringify(chunk);
+        const hash = (0, exports.historyHash)(text);
+        writeImmutable(path_1.default.join((0, exports.historyDirectory)(bucket, id), `${hash}.json`), text);
+        indexChunk(bucket, id, hash, pending, revision, commitId);
+        tail = hash;
+        pending = [];
+        bytes = 512;
+    };
+    for (const message of messages) {
+        const size = jsonBytes(message) + 1;
+        if (bytes + size > history_policy_1.HISTORY_POLICY.chunkBytes)
+            flush();
+        pending.push(message);
+        bytes += size;
+    }
+    flush();
+    return tail;
+}
+/** A newest-revision-first traversal makes corrections authoritative without a growing manifest. */
+function* historyMessagesNewest(bucket, record) {
+    const resident = (record.messages || []);
+    for (let i = resident.length - 1; i >= 0; i--)
+        yield resident[i];
+    let tail = historyOf(record)?.archiveTail;
+    let hops = 0;
+    while (tail) {
+        if (++hops > 1_000_000)
+            throw new history_policy_1.HistoryStoreError('history_unavailable');
+        const chunk = readChunk(bucket, record.id, tail);
+        for (let i = chunk.messages.length - 1; i >= 0; i--)
+            yield chunk.messages[i];
+        tail = chunk.previous;
+    }
+}
+function prepareConversationHead(bucket, incoming, existing) {
+    const revision = Number(existing?.revision || 0) + 1, commitId = crypto_1.default.randomUUID();
+    // Reusing a revision after a failed head publication must not authorize orphaned
+    // reference pages from the failed transaction. A new nonce invalidates those pages.
+    (0, atomic_json_file_1.writeFileAtomic)(path_1.default.join((0, exports.historyDirectory)(bucket, incoming.id), 'commits', `${revision}.json`), JSON.stringify({ id: commitId }), 'utf8');
+    const previousHistory = historyOf(existing);
+    const oldResident = (existing?.messages || []);
+    const messages = incoming.messages ?? oldResident;
+    let lastSequence = previousHistory?.lastSequence ?? 0;
+    const bySequence = new Map();
+    for (const message of oldResident)
+        if (message.sequence)
+            bySequence.set(message.sequence, message);
+    const byId = new Map(oldResident.map(m => [m.id, m]));
+    const archivedCorrections = [];
+    const hasSequencedInput = messages.some(message => Number.isSafeInteger(message.sequence));
+    for (let i = 0; i < messages.length; i++) {
+        let raw = messages[i];
+        let previous = typeof raw.id === 'string' ? byId.get(raw.id) : undefined;
+        if (typeof raw.id !== 'string' && !Number.isSafeInteger(raw.sequence)) {
+            const sequence = i + 1;
+            if (existing && previousHistory && sequence <= previousHistory.lastSequence)
+                previous = historySequenceRange(bucket, existing, sequence, sequence)[0];
+            raw = { ...raw, sequence, ...(previous?.id ? { id: previous.id } : {}) };
+        }
+        if (!previous && !hasSequencedInput && existing && previousHistory && typeof raw.id === 'string' && !raw.sequence) {
+            // Legacy callers can resend a complete transcript. Resolve IDs against cold history
+            // one chunk at a time, rather than inventing duplicate sequence numbers.
+            for (const old of historyMessagesNewest(bucket, existing))
+                if (old.id === raw.id) {
+                    previous = old;
+                    break;
+                }
+        }
+        const supplied = Number(raw.sequence);
+        const sequence = previous?.sequence ?? (Number.isSafeInteger(supplied) && supplied > 0 && supplied <= lastSequence ? supplied : ++lastSequence);
+        const message = inlineMessage(bucket, incoming.id, raw, sequence);
+        if (previousHistory && sequence < previousHistory.firstResidentSequence) {
+            if (!previous || JSON.stringify(previous) !== JSON.stringify(message))
+                archivedCorrections.push(message);
+        }
+        else
+            bySequence.set(sequence, message);
+    }
+    const sorted = [...bySequence.values()].sort((a, b) => a.sequence - b.sequence);
+    let bytes = 2, cut = sorted.length;
+    while (cut > 0 && sorted.length - cut < history_policy_1.HISTORY_POLICY.residentMessages) {
+        const size = jsonBytes(sorted[cut - 1]) + 1;
+        if (bytes + size > history_policy_1.HISTORY_POLICY.residentMessageBytes)
+            break;
+        bytes += size;
+        cut--;
+    }
+    const tail = appendChunks(bucket, incoming.id, [...sorted.slice(0, cut), ...archivedCorrections], previousHistory?.archiveTail ?? null, revision, commitId);
+    const resident = sorted.slice(cut);
+    const result = {
+        ...incoming,
+        revision,
+        messages: resident,
+        history: { version: 3, firstResidentSequence: resident[0]?.sequence ?? lastSequence + 1, lastSequence, archiveTail: tail },
+        messageCount: lastSequence,
+        hasMessageBlobs: Boolean(existing?.hasMessageBlobs || sorted.some(m => m.contentRef || m.rawContentRef) || archivedCorrections.some(m => m.contentRef || m.rawContentRef)),
+        hasReviewHandoff: Object.prototype.hasOwnProperty.call(incoming, 'reviewHandoff') ? incoming.reviewHandoff?.reviewRequired === true : Boolean(incoming.hasReviewHandoff),
+        hasDelegation: Object.prototype.hasOwnProperty.call(incoming, 'delegation') ? Boolean(incoming.delegation) : Boolean(incoming.hasDelegation),
+        uiState: (0, conversation_ui_state_1.conversationUiState)(incoming),
+    };
+    if (result.run) {
+        result.run = { ...result.run };
+        delete result.run.messages;
+    }
+    // Preserve large historical/control values in immutable documents; full consumers must
+    // request the named document, while the interactive body remains bounded.
+    const refs = { ...(existing?.documentRefs || {}) };
+    const storeDocument = (key, value) => {
+        jsonBytes(value, history_policy_1.HISTORY_POLICY.controlDocumentBytes);
+        const text = JSON.stringify(value), hash = (0, exports.historyHash)(text);
+        writeImmutable(path_1.default.join((0, exports.historyDirectory)(bucket, incoming.id), 'documents', hash), text);
+        refs[key] = { hash, bytes: Buffer.byteLength(text) };
+    };
+    if (result.run)
+        for (const [key, value] of Object.entries(result.run)) {
+            if (value !== undefined && jsonBytes(value, history_policy_1.HISTORY_POLICY.controlDocumentBytes) > 32 * 1024) {
+                storeDocument(`run.${key}`, value);
+                delete result.run[key];
+            }
+            else
+                delete refs[`run.${key}`];
+        }
+    for (const [key, value] of Object.entries(result)) {
+        if (['messages', 'history', 'documentRefs'].includes(key) || value === undefined)
+            continue;
+        const size = jsonBytes(value, history_policy_1.HISTORY_POLICY.controlDocumentBytes);
+        if (size > 64 * 1024) {
+            storeDocument(key, value);
+            delete result[key];
+        }
+        else
+            delete refs[key];
+    }
+    if (Object.keys(refs).length)
+        result.documentRefs = refs;
+    else
+        delete result.documentRefs;
+    jsonBytes(result, history_policy_1.HISTORY_POLICY.headBytes - 1024);
+    return result;
+}
+function messagePage(bucket, record, options = {}) {
+    const limit = (0, history_policy_1.historyPageLimit)(options.limit);
+    const before = options.beforeSequence ?? Number.MAX_SAFE_INTEGER;
+    if (!Number.isSafeInteger(before) || before < 1)
+        throw new history_policy_1.HistoryStoreError('invalid_cursor', 400);
+    const last = Math.min(before - 1, historyOf(record)?.lastSequence ?? (record.messages?.length || 0));
+    const firstSequence = Math.max(1, last - limit);
+    const sorted = last >= firstSequence ? historySequenceRange(bucket, record, firstSequence, last).reverse() : [];
+    const messages = [];
+    let bytes = 512;
+    for (const message of sorted.slice(0, limit)) {
+        const size = jsonBytes(message);
+        if (bytes + size > history_policy_1.HISTORY_POLICY.responseBytes)
+            break;
+        messages.push(message);
+        bytes += size;
+    }
+    messages.reverse();
+    const first = messages[0]?.sequence ?? before;
+    return { messages, hasEarlier: sorted.length > messages.length || first > 1, nextBeforeSequence: first > 1 ? first : null, revision: Number(record.revision || 0) };
+}