@kbediako/codex-orchestrator 0.1.3 → 0.1.4

package/dist/orchestrator/src/persistence/ExperienceStore.js

@@ -1,10 +1,11 @@
 import { randomBytes } from 'node:crypto';
-import { readFile, mkdir, rm } from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { appendFile, mkdir, open, rm } from 'node:fs/promises';
 import { join } from 'node:path';
+import { createInterface } from 'node:readline';
 import { listDirectories, resolveEnvironmentPaths } from '../../../scripts/lib/run-manifests.js';
 import { acquireLockWithRetry } from './lockFile.js';
 import { sanitizeTaskId } from './sanitizeTaskId.js';
-import { writeAtomicFile } from '../utils/atomicWrite.js';
 export class ExperienceStoreLockError extends Error {
     taskId;
     constructor(message, taskId) {
@@ -30,7 +31,8 @@ export class ExperienceStore {
             maxAttempts: 5,
             initialDelayMs: 100,
             backoffFactor: 2,
-            maxDelayMs: 1000
+            maxDelayMs: 1000,
+            staleMs: 5 * 60 * 1000
         };
         const overrides = options.lockRetry ?? {};
         const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));
@@ -52,10 +54,10 @@ export class ExperienceStore {
             const targetDir = join(this.outDir, taskId);
             await mkdir(targetDir, { recursive: true });
             const filePath = join(targetDir, 'experiences.jsonl');
-            const existing = await this.readRecords(filePath);
             const nextRecords = inputs.map((input) => this.prepareRecord(input, manifestPath));
-            const serialized = [...existing, ...nextRecords].map((record) => JSON.stringify(record)).join('\n');
-            await writeAtomicFile(filePath, `${serialized}\n`);
+            await this.ensureTrailingNewline(filePath);
+            const payload = nextRecords.map((record) => JSON.stringify(record)).join('\n');
+            await appendFile(filePath, `${payload}\n`, 'utf8');
             return nextRecords;
         }
         finally {
@@ -72,36 +74,26 @@ export class ExperienceStore {
         if (limit === 0) {
             return [];
         }
-        const allRecords = taskFilter
-            ? await this.readRecords(join(this.outDir, taskFilter, 'experiences.jsonl'))
-            : await this.readAllRecords();
-        const filtered = allRecords.filter((record) => {
+        const collector = createTopKCollector(limit, params.minReward);
+        const applyRecord = (record) => {
             if (record.domain !== safeDomain) {
-                return false;
+                return;
             }
             if (taskFilter && record.taskId !== taskFilter) {
-                return false;
+                return;
             }
-            return true;
-        });
-        const scored = filtered
-            .map((record) => ({
-            record,
-            score: record.reward.gtScore + record.reward.relativeRank
-        }))
-            .filter((entry) => {
-            if (params.minReward === undefined) {
-                return true;
-            }
-            return entry.score >= params.minReward;
-        })
-            .sort((a, b) => {
-            if (b.score === a.score) {
-                return b.record.createdAt.localeCompare(a.record.createdAt);
+            collector.add(record);
+        };
+        if (taskFilter) {
+            await this.scanRecords(join(this.outDir, taskFilter, 'experiences.jsonl'), applyRecord);
+        }
+        else {
+            const directories = await listDirectories(this.outDir);
+            for (const dir of directories) {
+                await this.scanRecords(join(this.outDir, dir, 'experiences.jsonl'), applyRecord);
             }
-            return b.score - a.score;
-        });
-        return scored.slice(0, limit).map((entry) => entry.record);
+        }
+        return collector.finalize();
     }
     verifyStamp(record) {
         return HEX_STAMP_PATTERN.test(record.stampSignature);
@@ -156,35 +148,110 @@ export class ExperienceStore {
     async releaseLock(lockPath) {
         await rm(lockPath, { force: true });
     }
-    async readRecords(filePath) {
+    async scanRecords(filePath, onRecord) {
         try {
-            const raw = await readFile(filePath, 'utf8');
-            return raw
-                .split('\n')
-                .filter(Boolean)
-                .map((line) => JSON.parse(line));
+            const stream = createReadStream(filePath, { encoding: 'utf8' });
+            const reader = createInterface({ input: stream, crlfDelay: Infinity });
+            for await (const line of reader) {
+                const trimmed = line.trim();
+                if (!trimmed) {
+                    continue;
+                }
+                try {
+                    const record = JSON.parse(trimmed);
+                    onRecord(record);
+                }
+                catch {
+                    continue;
+                }
+            }
         }
         catch (error) {
             if (error.code === 'ENOENT') {
-                return [];
+                return;
             }
             throw error;
         }
     }
-    async readAllRecords() {
-        const directories = await listDirectories(this.outDir);
-        const all = [];
-        for (const dir of directories) {
-            const filePath = join(this.outDir, dir, 'experiences.jsonl');
-            all.push(...(await this.readRecords(filePath)));
-        }
-        return all;
+    async ensureTrailingNewline(filePath) {
+        try {
+            const handle = await open(filePath, 'r');
+            let needsNewline = false;
+            try {
+                const { size } = await handle.stat();
+                if (size === 0) {
+                    return;
+                }
+                const buffer = Buffer.alloc(1);
+                await handle.read(buffer, 0, 1, size - 1);
+                needsNewline = buffer[0] !== 0x0a;
+            }
+            finally {
+                await handle.close();
+            }
+            if (needsNewline) {
+                await appendFile(filePath, '\n', 'utf8');
+            }
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return;
+            }
+            throw error;
+        }
     }
     generateId() {
         const suffix = randomBytes(3).toString('hex');
         return `exp-${Date.now().toString(36)}-${suffix}`;
     }
 }
+function createTopKCollector(limit, minReward) {
+    const entries = [];
+    const threshold = typeof minReward === 'number' ? minReward : null;
+    const compare = (a, b) => {
+        if (a.score !== b.score) {
+            return a.score - b.score;
+        }
+        const aTime = a.record.createdAt ?? '';
+        const bTime = b.record.createdAt ?? '';
+        return aTime.localeCompare(bTime);
+    };
+    const add = (record) => {
+        const score = record.reward.gtScore + record.reward.relativeRank;
+        if (threshold !== null && score < threshold) {
+            return;
+        }
+        const entry = { record, score };
+        if (entries.length === 0) {
+            entries.push(entry);
+            return;
+        }
+        const worst = entries[0];
+        if (entries.length >= limit && worst && compare(entry, worst) <= 0) {
+            return;
+        }
+        let index = 0;
+        while (index < entries.length && compare(entries[index], entry) <= 0) {
+            index += 1;
+        }
+        entries.splice(index, 0, entry);
+        if (entries.length > limit) {
+            entries.shift();
+        }
+    };
+    const finalize = () => entries
+        .slice()
+        .sort((a, b) => {
+        if (a.score !== b.score) {
+            return b.score - a.score;
+        }
+        const aTime = a.record.createdAt ?? '';
+        const bTime = b.record.createdAt ?? '';
+        return bTime.localeCompare(aTime);
+    })
+        .map((entry) => entry.record);
+    return { add, finalize };
+}
 function truncateSummary(value, maxWords) {
     const tokens = value.trim().split(/\s+/u).filter(Boolean);
     if (tokens.length <= maxWords) {

package/dist/orchestrator/src/persistence/PersistenceCoordinator.js

@@ -33,10 +33,12 @@ export class PersistenceCoordinator {
     }
     async handleRunCompleted(summary) {
         let stateStoreError = null;
-        try {
-            await this.stateStore.recordRun(summary);
-        }
-        catch (error) {
+        const [stateResult, manifestResult] = await Promise.allSettled([
+            this.stateStore.recordRun(summary),
+            this.manifestWriter.write(summary)
+        ]);
+        if (stateResult.status === 'rejected') {
+            const error = stateResult.reason;
             stateStoreError = error;
             if (error instanceof TaskStateStoreLockError) {
                 logger.warn(`Task state snapshot skipped for task ${summary.taskId} (run ${summary.runId}): ${error.message}`);
@@ -45,10 +47,8 @@ export class PersistenceCoordinator {
                 logger.error(`Task state snapshot failed for task ${summary.taskId} (run ${summary.runId})`, error);
             }
         }
-        try {
-            await this.manifestWriter.write(summary);
-        }
-        catch (error) {
+        if (manifestResult.status === 'rejected') {
+            const error = manifestResult.reason;
             this.options.onError?.(error, summary);
             if (!this.options.onError) {
                 logger.error(`PersistenceCoordinator manifest write error for task ${summary.taskId} (run ${summary.runId})`, error);

package/dist/orchestrator/src/persistence/TaskStateStore.js

@@ -27,7 +27,8 @@ export class TaskStateStore {
             maxAttempts: 5,
             initialDelayMs: 100,
             backoffFactor: 2,
-            maxDelayMs: 1000
+            maxDelayMs: 1000,
+            staleMs: 5 * 60 * 1000
         };
         const overrides = options.lockRetry ?? {};
         const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));

package/dist/orchestrator/src/persistence/lockFile.js

@@ -1,8 +1,9 @@
-import { open } from 'node:fs/promises';
+import { open, rm, stat } from 'node:fs/promises';
 import { setTimeout as delay } from 'node:timers/promises';
 export async function acquireLockWithRetry(params) {
     await params.ensureDirectory();
     const { maxAttempts, initialDelayMs, backoffFactor, maxDelayMs } = params.retry;
+    const staleMs = params.retry.staleMs ?? 0;
     let attempt = 0;
     let delayMs = initialDelayMs;
     while (attempt < maxAttempts) {
@@ -16,6 +17,13 @@ export async function acquireLockWithRetry(params) {
             if (error.code !== 'EEXIST') {
                 throw error;
             }
+            if (staleMs > 0) {
+                const cleared = await clearStaleLock(params.lockPath, staleMs);
+                if (cleared) {
+                    attempt -= 1;
+                    continue;
+                }
+            }
             if (attempt >= maxAttempts) {
                 throw params.createError(params.taskId, attempt);
             }
@@ -24,3 +32,20 @@ export async function acquireLockWithRetry(params) {
         }
     }
 }
+async function clearStaleLock(lockPath, staleMs) {
+    try {
+        const stats = await stat(lockPath);
+        const ageMs = Date.now() - stats.mtimeMs;
+        if (!Number.isFinite(ageMs) || ageMs <= staleMs) {
+            return false;
+        }
+        await rm(lockPath, { force: true });
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return false;
+        }
+        return false;
+    }
+}

package/dist/orchestrator/src/sync/CloudSyncWorker.js

@@ -52,7 +52,7 @@ export class CloudSyncWorker {
         }
         catch (error) {
             this.options.onError?.(error, summary, 0);
-            await this.appendAuditLog({
+            await this.safeAppendAuditLog({
                 level: 'error',
                 message: 'Failed to read manifest before sync',
                 summary,
@@ -74,7 +74,7 @@ export class CloudSyncWorker {
                     idempotencyKey
                 });
                 this.options.onSuccess?.(result, summary);
-                await this.appendAuditLog({
+                await this.safeAppendAuditLog({
                     level: 'info',
                     message: 'Cloud sync completed',
                     summary,
@@ -84,7 +84,7 @@ export class CloudSyncWorker {
             }
             catch (error) {
                 this.options.onError?.(error, summary, attempt);
-                await this.appendAuditLog({
+                await this.safeAppendAuditLog({
                     level: 'error',
                     message: 'Cloud sync attempt failed',
                     summary,
@@ -120,6 +120,19 @@ export class CloudSyncWorker {
         });
         await appendFile(logPath, `${line}\n`, 'utf-8');
     }
+    async safeAppendAuditLog(entry) {
+        try {
+            await this.appendAuditLog(entry);
+        }
+        catch (error) {
+            try {
+                this.options.onError?.(error, entry.summary, 0);
+            }
+            catch {
+                // swallow audit log failures
+            }
+        }
+    }
     shouldRetry(error) {
         if (this.options.retryDecider) {
             return this.options.retryDecider(error);
@@ -160,7 +173,7 @@ export class CloudSyncWorker {
             if (repaired) {
                 try {
                     const parsed = JSON.parse(repaired);
-                    await this.appendAuditLog({
+                    await this.safeAppendAuditLog({
                         level: 'info',
                         message: 'Recovered manifest from partial JSON',
                         summary,

package/dist/packages/orchestrator/src/telemetry/otel-exporter.js

@@ -17,6 +17,7 @@ export function createTelemetrySink(options = {}) {
         maxFailures: options.maxFailures ?? DEFAULT_MAX_FAILURES,
         backoffMs: options.backoffMs ?? DEFAULT_BACKOFF_MS,
         maxBackoffMs: options.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS,
+        maxQueueSize: options.maxQueueSize,
         fetch: fetchImpl,
         logger: options.logger ?? console
     });
@@ -43,6 +44,7 @@ class OtelTelemetrySink {
     maxFailures;
     backoffMs;
     maxBackoffMs;
+    maxQueueSize;
     queue = [];
     disabled = false;
     consecutiveFailures = 0;
@@ -54,12 +56,14 @@ class OtelTelemetrySink {
         this.maxFailures = config.maxFailures;
         this.backoffMs = config.backoffMs;
         this.maxBackoffMs = config.maxBackoffMs;
+        this.maxQueueSize = normalizeQueueSize(config.maxQueueSize);
     }
     async record(event) {
         if (this.disabled) {
             return;
         }
         this.queue.push(event);
+        this.trimQueue();
     }
     async recordSummary(summary) {
         if (this.disabled) {
@@ -86,6 +90,7 @@ class OtelTelemetrySink {
         }
         catch (error) {
             this.queue.unshift(...batch);
+            this.trimQueue();
             this.handleFailure(error);
         }
     }
@@ -113,6 +118,12 @@ class OtelTelemetrySink {
             this.logger.warn(`Telemetry disabled after ${this.consecutiveFailures} consecutive failures: ${String(error)}`);
         }
     }
+    trimQueue() {
+        if (this.maxQueueSize === null || this.queue.length <= this.maxQueueSize) {
+            return;
+        }
+        this.queue.splice(0, this.queue.length - this.maxQueueSize);
+    }
 }
 function buildSummaryDimensions(metrics) {
     if (!metrics) {
@@ -140,3 +151,13 @@ async function wait(ms) {
         setTimeout(resolve, ms);
     });
 }
+function normalizeQueueSize(value) {
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return null;
+    }
+    const normalized = Math.floor(value);
+    if (normalized <= 0) {
+        return null;
+    }
+    return normalized;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kbediako/codex-orchestrator",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",
   "bin": {
@@ -100,7 +100,9 @@
     "esbuild": "^0.25.11"
   },
   "dependencies": {
+    "@iarna/toml": "^2.2.5",
     "ajv": "^8.17.1",
+    "canonicalize": "^2.1.0",
     "ink": "^4.4.1",
     "js-yaml": "^4.1.0",
     "react": "^18.3.1"