@kbediako/codex-orchestrator 0.1.2 → 0.1.4

package/dist/orchestrator/src/cli/events/runEventStream.js

@@ -0,0 +1,246 @@
+import { createWriteStream } from 'node:fs';
+import { mkdir, readFile } from 'node:fs/promises';
+import { isoTimestamp } from '../utils/time.js';
+const SCHEMA_VERSION = 1;
+export class RunEventStream {
+    seq;
+    stream;
+    now;
+    taskId;
+    runId;
+    writeQueue = Promise.resolve();
+    streamError = null;
+    constructor(stream, options) {
+        this.stream = stream;
+        this.taskId = options.taskId;
+        this.runId = options.runId;
+        this.now = options.now;
+        this.seq = options.initialSeq;
+        this.stream.on('error', (error) => {
+            this.streamError = error;
+        });
+    }
+    static async create(options) {
+        await mkdir(options.paths.runDir, { recursive: true });
+        const initialSeq = await readLastSeq(options.paths.eventsPath);
+        const stream = createWriteStream(options.paths.eventsPath, { flags: 'a' });
+        return new RunEventStream(stream, {
+            taskId: options.taskId,
+            runId: options.runId,
+            now: options.now ?? isoTimestamp,
+            initialSeq
+        });
+    }
+    async append(input) {
+        if (this.streamError) {
+            throw this.streamError;
+        }
+        const entry = {
+            schema_version: SCHEMA_VERSION,
+            seq: this.nextSeq(),
+            timestamp: input.timestamp ?? this.now(),
+            task_id: this.taskId,
+            run_id: this.runId,
+            event: input.event,
+            actor: input.actor ?? 'runner',
+            payload: input.payload ?? null
+        };
+        const line = `${JSON.stringify(entry)}\n`;
+        this.writeQueue = this.writeQueue
+            .then(() => new Promise((resolve, reject) => {
+            this.stream.write(line, (error) => {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                resolve();
+            });
+        }))
+            .catch((error) => {
+            this.streamError = error;
+            throw error;
+        });
+        await this.writeQueue;
+        if (this.streamError) {
+            throw this.streamError;
+        }
+        return entry;
+    }
+    async close() {
+        try {
+            await this.writeQueue;
+        }
+        catch {
+            // Allow close to proceed even if a prior write failed.
+        }
+        if (this.streamError) {
+            this.stream.destroy();
+            return;
+        }
+        await new Promise((resolve, reject) => {
+            const onFinish = () => {
+                cleanup();
+                resolve();
+            };
+            const onError = (error) => {
+                cleanup();
+                this.stream.destroy();
+                reject(error);
+            };
+            const cleanup = () => {
+                this.stream.off('finish', onFinish);
+                this.stream.off('error', onError);
+            };
+            this.stream.once('finish', onFinish);
+            this.stream.once('error', onError);
+            this.stream.end();
+        });
+    }
+    nextSeq() {
+        this.seq += 1;
+        return this.seq;
+    }
+}
+export function attachRunEventAdapter(emitter, stream, onEntry, onError) {
+    return emitter.on('*', (event) => {
+        const mapped = mapRunEvent(event);
+        if (!mapped) {
+            return;
+        }
+        void stream
+            .append(mapped)
+            .then((entry) => {
+            onEntry?.(entry);
+        })
+            .catch((error) => {
+            onError?.(error, mapped);
+        });
+    });
+}
+function mapRunEvent(event) {
+    // The public event stream uses "step_*" naming for pipeline stages to match UI terminology.
+    switch (event.type) {
+        case 'run:started':
+            return {
+                event: 'run_started',
+                actor: 'runner',
+                payload: {
+                    pipeline_id: event.pipelineId,
+                    pipeline_title: event.pipelineTitle,
+                    status: event.status,
+                    manifest_path: event.manifestPath,
+                    log_path: event.logPath
+                }
+            };
+        case 'stage:started':
+            return {
+                event: 'step_started',
+                actor: 'runner',
+                payload: {
+                    stage_id: event.stageId,
+                    stage_index: event.stageIndex,
+                    title: event.title,
+                    kind: event.kind,
+                    status: event.status,
+                    log_path: event.logPath
+                }
+            };
+        case 'stage:completed':
+            return {
+                event: event.status === 'failed' ? 'step_failed' : 'step_completed',
+                actor: 'runner',
+                payload: {
+                    stage_id: event.stageId,
+                    stage_index: event.stageIndex,
+                    title: event.title,
+                    kind: event.kind,
+                    status: event.status,
+                    exit_code: event.exitCode,
+                    summary: event.summary,
+                    log_path: event.logPath,
+                    sub_run_id: event.subRunId ?? null
+                }
+            };
+        case 'run:completed':
+            return {
+                event: 'run_completed',
+                actor: 'runner',
+                payload: {
+                    pipeline_id: event.pipelineId,
+                    status: event.status,
+                    manifest_path: event.manifestPath,
+                    run_summary_path: event.runSummaryPath,
+                    metrics_path: event.metricsPath,
+                    summary: event.summary
+                }
+            };
+        case 'run:error':
+            return {
+                event: 'run_failed',
+                actor: 'runner',
+                payload: {
+                    pipeline_id: event.pipelineId,
+                    message: event.message,
+                    stage_id: event.stageId ?? null
+                }
+            };
+        case 'log':
+            return {
+                event: 'agent_message',
+                actor: 'runner',
+                payload: {
+                    message: event.message,
+                    level: event.level,
+                    stage_id: event.stageId ?? null,
+                    stage_index: event.stageIndex ?? null,
+                    source: event.source ?? null
+                }
+            };
+        case 'tool:call':
+            return {
+                event: 'tool_called',
+                actor: 'runner',
+                payload: {
+                    tool_name: event.toolName,
+                    status: event.status,
+                    message: event.message ?? null,
+                    attempt: event.attempt ?? null,
+                    stage_id: event.stageId ?? null,
+                    stage_index: event.stageIndex ?? null
+                }
+            };
+        default:
+            return null;
+    }
+}
+async function readLastSeq(pathname) {
+    try {
+        const raw = await readFile(pathname, 'utf8');
+        if (!raw.trim()) {
+            return 0;
+        }
+        const lines = raw.split('\n');
+        for (let index = lines.length - 1; index >= 0; index -= 1) {
+            const line = lines[index]?.trim();
+            if (!line) {
+                continue;
+            }
+            try {
+                const parsed = JSON.parse(line);
+                if (typeof parsed.seq === 'number' && Number.isFinite(parsed.seq)) {
+                    return parsed.seq;
+                }
+            }
+            catch {
+                continue;
+            }
+        }
+        return 0;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return 0;
+        }
+        throw error;
+    }
+}

package/dist/orchestrator/src/cli/exec/context.js

@@ -1,11 +1,12 @@
 import process from 'node:process';
 import { bootstrapManifest } from '../run/manifest.js';
 import { generateRunId } from '../utils/runId.js';
-import { JsonlWriter } from '../utils/jsonlWriter.js';
 import { ExperienceStore } from '../../persistence/ExperienceStore.js';
+import { EnvUtils } from '../../../../packages/shared/config/index.js';
 import { createTelemetrySink } from '../../../../packages/orchestrator/src/telemetry/otel-exporter.js';
 import { createNotificationSink } from '../../../../packages/orchestrator/src/notifications/index.js';
 import { ManifestPersister, persistManifest } from '../run/manifestPersister.js';
+const MAX_TELEMETRY_EVENTS = EnvUtils.getInt('CODEX_ORCHESTRATOR_TELEMETRY_MAX_EVENTS', 1000);
 export async function bootstrapExecContext(context, invocation) {
     const argv = [invocation.command, ...(invocation.args ?? [])];
     const shellCommand = buildShellCommand(argv);
@@ -39,7 +40,8 @@ export async function bootstrapExecContext(context, invocation) {
     await persistManifest(paths, manifest, persister, { force: true });
     const telemetrySink = context.telemetrySink ?? createTelemetrySink({
         endpoint: invocation.otelEndpoint,
-        enabled: Boolean(invocation.otelEndpoint)
+        enabled: Boolean(invocation.otelEndpoint),
+        maxQueueSize: MAX_TELEMETRY_EVENTS
     });
     const envNotifications = parseNotificationEnv(process.env.CODEX_ORCHESTRATOR_NOTIFY);
     const notificationSink = context.notificationSink ??
@@ -47,7 +49,11 @@ export async function bootstrapExecContext(context, invocation) {
             targets: invocation.notifyTargets,
             envTargets: envNotifications
         });
-    const jsonlWriter = outputMode === 'jsonl' ? new JsonlWriter(stdout) : null;
+    const jsonlWriter = outputMode === 'jsonl'
+        ? (event) => {
+            stdout.write(`${JSON.stringify(event)}\n`);
+        }
+        : null;
     return {
         env,
         invocation,

package/dist/orchestrator/src/cli/exec/learning.js

@@ -2,9 +2,6 @@ import process from 'node:process';
 import { persistManifest } from '../run/manifestPersister.js';
 import { isoTimestamp } from '../utils/time.js';
 import { logger } from '../../logger.js';
-import { runLearningHarvester } from '../../learning/harvester.js';
-import { synthesizeScenario } from '../../learning/runner.js';
-import { runScenarioValidation } from '../../learning/validator.js';
 export async function maybeTriggerLearning(runContext, runStatus) {
     const enabled = process.env.LEARNING_PIPELINE_ENABLED === '1';
     if (!enabled) {
@@ -15,6 +12,11 @@ export async function maybeTriggerLearning(runContext, runStatus) {
         return;
     }
     try {
+        const [{ runLearningHarvester }, { synthesizeScenario }, { runScenarioValidation }] = await Promise.all([
+            import('../../learning/harvester.js'),
+            import('../../learning/runner.js'),
+            import('../../learning/validator.js')
+        ]);
         const harvester = await runLearningHarvester(runContext.manifest, {
             repoRoot: runContext.env.repoRoot,
             runsRoot: runContext.env.runsRoot,

package/dist/orchestrator/src/cli/exec/stageRunner.js

@@ -1,20 +1,45 @@
-import { runCommandStage } from '../services/commandRunner.js';
+import { runCommandStage, MAX_CAPTURED_CHUNK_EVENTS } from '../services/commandRunner.js';
 import { serializeExecEvent } from '../../../../packages/shared/events/serializer.js';
 import { sanitizeToolRunRecord } from '../../../../packages/shared/manifest/writer.js';
 export async function runExecStage(context) {
     let runResultSummary = null;
     let toolRecord = null;
+    const maxChunkEvents = MAX_CAPTURED_CHUNK_EVENTS;
+    let capturedChunkEvents = 0;
+    const shouldCaptureEvent = (event) => {
+        if (maxChunkEvents <= 0) {
+            return true;
+        }
+        if (event.type !== 'exec:chunk') {
+            return true;
+        }
+        if (capturedChunkEvents >= maxChunkEvents) {
+            return false;
+        }
+        capturedChunkEvents += 1;
+        return true;
+    };
     const hooks = {
         onEvent: (event) => {
-            context.execEvents.push(event);
-            const serialized = serializeExecEvent(event);
-            context.telemetryTasks.push(Promise.resolve(context.telemetrySink.record(serialized)).then(() => undefined));
+            const captureEvent = shouldCaptureEvent(event);
+            let serialized = null;
+            const getSerialized = () => {
+                if (!serialized) {
+                    serialized = serializeExecEvent(event);
+                }
+                return serialized;
+            };
             if (context.outputMode === 'jsonl' && context.jsonlWriter) {
-                context.jsonlWriter.write(serialized);
+                context.jsonlWriter(getSerialized());
             }
             else if (context.outputMode === 'interactive') {
                 streamInteractive(context.stdout, context.stderr, event);
             }
+            if (captureEvent) {
+                context.execEvents.push(event);
+                const payload = getSerialized();
+                context.telemetryTasks.push(Promise.resolve(context.telemetrySink.record(payload)).then(() => undefined));
+            }
         },
         onResult: (result) => {
             runResultSummary = {

package/dist/orchestrator/src/cli/exec/summary.js

@@ -61,7 +61,7 @@ export function createRunSummaryPayload(params) {
 }
 export function renderRunOutput(context, summaryPayload, summaryEvent) {
     if (context.outputMode === 'jsonl' && context.jsonlWriter) {
-        context.jsonlWriter.write(summaryEvent);
+        context.jsonlWriter(summaryEvent);
         return;
     }
     if (context.outputMode === 'json') {