@kbediako/codex-orchestrator 0.1.0

package/dist/packages/orchestrator/src/exec/unified-exec.js

@@ -0,0 +1,432 @@
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { ToolInvocationFailedError } from '../tool-orchestrator.js';
+import { createStdioTracker } from '../../../shared/streams/stdio.js';
+export class UnifiedExecRunner {
+    orchestrator;
+    sessionManager;
+    executor;
+    maxBufferBytes;
+    now;
+    idGenerator;
+    listeners = new Set();
+    handleService;
+    constructor(options) {
+        this.orchestrator = options.orchestrator;
+        this.sessionManager = options.sessionManager;
+        this.executor = options.executor ?? defaultExecutor;
+        this.maxBufferBytes = options.maxBufferBytes ?? 64 * 1024;
+        this.now = options.now ?? (() => new Date());
+        this.idGenerator = options.idGenerator ?? (() => randomUUID());
+        this.handleService = options.handleService;
+    }
+    on(listener) {
+        this.listeners.add(listener);
+        return () => {
+            this.listeners.delete(listener);
+        };
+    }
+    async run(options) {
+        const args = options.args ?? [];
+        const invocationId = options.invocationId ?? this.idGenerator();
+        const correlationId = this.idGenerator();
+        const issuedHandle = this.handleService ? this.handleService.issueHandle(correlationId) : undefined;
+        const handleId = issuedHandle?.id;
+        let handleClosed = false;
+        const reuse = options.reuseSession ?? Boolean(options.sessionId);
+        const lease = await this.sessionManager.acquire({
+            id: options.sessionId,
+            reuse,
+            persist: options.persistSession,
+            env: options.env
+        });
+        const events = [];
+        const captureState = createEventCaptureState(options.eventCapture);
+        let chunkSequence = 0;
+        const sandboxOptions = options.sandbox
+            ? this.decorateSandboxOptions(options.sandbox, correlationId, events, handleId, captureState)
+            : undefined;
+        const metadata = {
+            ...options.metadata,
+            command: options.command,
+            args,
+            cwd: options.cwd,
+            sessionId: lease.id,
+            correlationId,
+            persisted: lease.persisted,
+            handleId
+        };
+        const toolInvocation = {
+            id: invocationId,
+            tool: options.toolId ?? 'exec',
+            description: options.description,
+            approvalRequired: options.approvalRequired,
+            approvalKey: options.approvalKey,
+            metadata,
+            sandbox: sandboxOptions
+        };
+        let attemptResult;
+        let invocationRecord;
+        try {
+            const result = await this.orchestrator.invoke({
+                ...toolInvocation,
+                run: async ({ attempt, sandboxState }) => {
+                    const attemptStartedAt = this.now();
+                    const tracker = createStdioTracker({
+                        maxBufferBytes: this.maxBufferBytes,
+                        now: this.now,
+                        startSequence: chunkSequence
+                    });
+                    const beginEvent = this.createBeginEvent({
+                        attempt,
+                        correlationId,
+                        command: options.command,
+                        args,
+                        cwd: options.cwd,
+                        sandboxState,
+                        sessionId: lease.id,
+                        persisted: lease.persisted
+                    });
+                    await this.publishEvent(beginEvent, events, handleId, captureState);
+                    let capturedError;
+                    const attemptSummary = {
+                        exitCode: null,
+                        signal: null,
+                        stdout: '',
+                        stderr: '',
+                        durationMs: 0,
+                        status: 'failed',
+                        sandboxState,
+                        sessionId: lease.id
+                    };
+                    try {
+                        const outcome = await this.executor({
+                            command: options.command,
+                            args,
+                            cwd: options.cwd,
+                            env: lease.envSnapshot,
+                            attempt,
+                            sandboxState,
+                            session: lease,
+                            onStdout: (chunk) => {
+                                const sequenced = this.recordChunk(tracker, 'stdout', chunk);
+                                chunkSequence = sequenced.sequence;
+                                const chunkEvent = this.createChunkEvent({
+                                    attempt,
+                                    correlationId,
+                                    chunk: sequenced
+                                });
+                                void this.publishEvent(chunkEvent, events, handleId, captureState);
+                            },
+                            onStderr: (chunk) => {
+                                const sequenced = this.recordChunk(tracker, 'stderr', chunk);
+                                chunkSequence = sequenced.sequence;
+                                const chunkEvent = this.createChunkEvent({
+                                    attempt,
+                                    correlationId,
+                                    chunk: sequenced
+                                });
+                                void this.publishEvent(chunkEvent, events, handleId, captureState);
+                            }
+                        });
+                        attemptSummary.exitCode = outcome.exitCode ?? null;
+                        attemptSummary.signal = outcome.signal ?? null;
+                        attemptSummary.status = outcome.exitCode === 0 ? 'succeeded' : 'failed';
+                        attemptSummary.durationMs = outcome.durationMs ?? 0;
+                    }
+                    catch (error) {
+                        capturedError = error;
+                    }
+                    finally {
+                        const completedAt = this.now();
+                        const derivedDuration = Math.max(0, completedAt.getTime() - attemptStartedAt.getTime());
+                        if (attemptSummary.durationMs <= 0) {
+                            attemptSummary.durationMs = derivedDuration;
+                        }
+                        attemptSummary.stdout = tracker.getBuffered('stdout');
+                        attemptSummary.stderr = tracker.getBuffered('stderr');
+                        const endEvent = this.createEndEvent({
+                            attempt,
+                            correlationId,
+                            summary: attemptSummary
+                        });
+                        await this.publishEvent(endEvent, events, handleId, captureState);
+                    }
+                    if (capturedError) {
+                        throw capturedError;
+                    }
+                    return attemptSummary;
+                }
+            });
+            attemptResult = result.output;
+            invocationRecord = result.record;
+            this.attachManifestMetadata(invocationRecord, events, lease, options, correlationId, attemptResult, handleId ?? undefined);
+            const runOutcome = {
+                correlationId,
+                stdout: attemptResult.stdout,
+                stderr: attemptResult.stderr,
+                exitCode: attemptResult.exitCode,
+                signal: attemptResult.signal,
+                durationMs: attemptResult.durationMs,
+                status: attemptResult.status,
+                sandboxState: attemptResult.sandboxState,
+                record: invocationRecord,
+                events,
+                handle: undefined
+            };
+            if (handleId && this.handleService) {
+                this.handleService.close(handleId);
+                handleClosed = true;
+                runOutcome.handle = this.handleService.getDescriptor(handleId);
+            }
+            return runOutcome;
+        }
+        catch (error) {
+            if (error instanceof ToolInvocationFailedError &&
+                error.record) {
+                this.attachManifestMetadata(error.record, events, lease, options, correlationId, attemptResult, handleId ?? undefined);
+            }
+            if (handleId && this.handleService && !handleClosed) {
+                this.handleService.close(handleId);
+                handleClosed = true;
+            }
+            throw error;
+        }
+        finally {
+            if (handleId && this.handleService && !handleClosed) {
+                this.handleService.close(handleId);
+            }
+            await lease.release();
+        }
+    }
+    recordChunk(tracker, stream, chunk) {
+        return tracker.push(stream, chunk);
+    }
+    async publishEvent(event, sink, handleId, captureState) {
+        if (shouldCaptureEvent(event, captureState)) {
+            sink.push(event);
+        }
+        if (handleId && this.handleService) {
+            await this.handleService.append(handleId, event);
+        }
+        for (const listener of this.listeners) {
+            listener(event);
+        }
+    }
+    createBeginEvent(params) {
+        return {
+            type: 'exec:begin',
+            correlationId: params.correlationId,
+            attempt: params.attempt,
+            timestamp: this.now().toISOString(),
+            payload: {
+                command: params.command,
+                args: params.args,
+                cwd: params.cwd,
+                sandboxState: params.sandboxState,
+                sessionId: params.sessionId,
+                persisted: params.persisted
+            }
+        };
+    }
+    createChunkEvent(params) {
+        return {
+            type: 'exec:chunk',
+            correlationId: params.correlationId,
+            attempt: params.attempt,
+            timestamp: params.chunk.timestamp,
+            payload: {
+                stream: params.chunk.stream,
+                sequence: params.chunk.sequence,
+                bytes: params.chunk.bytes,
+                data: params.chunk.data
+            }
+        };
+    }
+    createEndEvent(params) {
+        return {
+            type: 'exec:end',
+            correlationId: params.correlationId,
+            attempt: params.attempt,
+            timestamp: this.now().toISOString(),
+            payload: {
+                exitCode: params.summary.exitCode,
+                signal: params.summary.signal,
+                durationMs: params.summary.durationMs,
+                stdout: params.summary.stdout,
+                stderr: params.summary.stderr,
+                sandboxState: params.summary.sandboxState,
+                sessionId: params.summary.sessionId,
+                status: params.summary.status
+            }
+        };
+    }
+    createRetryEvent(params) {
+        return {
+            type: 'exec:retry',
+            correlationId: params.correlationId,
+            attempt: params.attempt,
+            timestamp: this.now().toISOString(),
+            payload: {
+                delayMs: params.delayMs,
+                sandboxState: params.sandboxState,
+                errorMessage: getErrorMessage(params.error)
+            }
+        };
+    }
+    decorateSandboxOptions(input, correlationId, events, handleId, captureState) {
+        const { onRetry, ...rest } = input;
+        const decorated = {
+            ...rest,
+            onRetry: async (context) => {
+                const retryEvent = this.createRetryEvent({
+                    attempt: context.attempt,
+                    correlationId,
+                    delayMs: context.delayMs,
+                    sandboxState: context.sandboxState,
+                    error: context.error
+                });
+                await this.publishEvent(retryEvent, events, handleId, captureState);
+                if (onRetry) {
+                    await onRetry.call(input, context);
+                }
+            }
+        };
+        return decorated;
+    }
+    attachManifestMetadata(record, events, lease, options, correlationId, attemptResult, handleId) {
+        const execMetadata = {
+            ...(record.metadata ?? {}),
+            exec: {
+                command: options.command,
+                args: options.args ?? [],
+                cwd: options.cwd,
+                sessionId: lease.id,
+                persisted: lease.persisted,
+                correlationId,
+                handleId,
+                exitCode: attemptResult?.exitCode ?? null,
+                signal: attemptResult?.signal ?? null
+            }
+        };
+        record.metadata = execMetadata;
+        if (attemptResult) {
+            record.status = attemptResult.status;
+            record.sandboxState = attemptResult.sandboxState;
+        }
+        else if (record.status !== 'failed') {
+            record.status = 'failed';
+        }
+        record.events = events.map((event) => this.toManifestEvent(event));
+    }
+    toManifestEvent(event) {
+        switch (event.type) {
+            case 'exec:begin':
+                return {
+                    type: 'exec:begin',
+                    correlationId: event.correlationId,
+                    timestamp: event.timestamp,
+                    attempt: event.attempt,
+                    command: event.payload.command,
+                    args: event.payload.args,
+                    cwd: event.payload.cwd,
+                    sandboxState: event.payload.sandboxState,
+                    sessionId: event.payload.sessionId,
+                    persisted: event.payload.persisted
+                };
+            case 'exec:chunk':
+                return {
+                    type: 'exec:chunk',
+                    correlationId: event.correlationId,
+                    timestamp: event.timestamp,
+                    attempt: event.attempt,
+                    stream: event.payload.stream,
+                    sequence: event.payload.sequence,
+                    bytes: event.payload.bytes,
+                    data: event.payload.data
+                };
+            case 'exec:end':
+                return {
+                    type: 'exec:end',
+                    correlationId: event.correlationId,
+                    timestamp: event.timestamp,
+                    attempt: event.attempt,
+                    exitCode: event.payload.exitCode,
+                    signal: event.payload.signal,
+                    durationMs: event.payload.durationMs,
+                    stdout: event.payload.stdout,
+                    stderr: event.payload.stderr,
+                    sandboxState: event.payload.sandboxState,
+                    sessionId: event.payload.sessionId,
+                    status: event.payload.status
+                };
+            case 'exec:retry':
+                return {
+                    type: 'exec:retry',
+                    correlationId: event.correlationId,
+                    timestamp: event.timestamp,
+                    attempt: event.attempt,
+                    delayMs: event.payload.delayMs,
+                    sandboxState: event.payload.sandboxState,
+                    errorMessage: event.payload.errorMessage
+                };
+        }
+    }
+}
+function createEventCaptureState(options) {
+    const maxChunkEvents = options?.maxChunkEvents ?? null;
+    if (!maxChunkEvents || maxChunkEvents <= 0) {
+        return null;
+    }
+    return { maxChunkEvents, capturedChunkEvents: 0 };
+}
+function shouldCaptureEvent(event, captureState) {
+    if (!captureState || captureState.maxChunkEvents === null) {
+        return true;
+    }
+    if (event.type !== 'exec:chunk') {
+        return true;
+    }
+    if (captureState.capturedChunkEvents >= captureState.maxChunkEvents) {
+        return false;
+    }
+    captureState.capturedChunkEvents += 1;
+    return true;
+}
+function getErrorMessage(error) {
+    if (error instanceof Error && error.message) {
+        return error.message;
+    }
+    if (typeof error === 'string') {
+        return error;
+    }
+    return String(error);
+}
+const defaultExecutor = async (request) => {
+    const child = spawn(request.command, request.args, {
+        cwd: request.cwd,
+        env: request.env,
+        stdio: ['ignore', 'pipe', 'pipe']
+    });
+    if (!child.stdout || !child.stderr) {
+        throw new Error('Command must provide stdout and stderr streams.');
+    }
+    child.stdout.on('data', request.onStdout);
+    child.stderr.on('data', request.onStderr);
+    return await new Promise((resolve, reject) => {
+        const handleExit = (exitCode, signal) => {
+            cleanup();
+            resolve({ exitCode, signal });
+        };
+        const handleError = (error) => {
+            cleanup();
+            reject(error);
+        };
+        const cleanup = () => {
+            child.off('exit', handleExit);
+            child.off('error', handleError);
+        };
+        child.once('exit', handleExit);
+        child.once('error', handleError);
+    });
+};

package/dist/packages/orchestrator/src/index.js

@@ -0,0 +1,3 @@
+export { ToolOrchestrator, SandboxRetryableError, ApprovalRequiredError, ApprovalDeniedError, ToolInvocationFailedError } from './tool-orchestrator.js';
+export { ExecSessionManager } from './exec/session-manager.js';
+export { UnifiedExecRunner } from './exec/unified-exec.js';

package/dist/packages/orchestrator/src/instructions/loader.js

@@ -0,0 +1,101 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { loadPromptPacks } from './promptPacks.js';
+const INSTRUCTION_STAMP_PATTERN = /^(?:\uFEFF)?<!--\s*codex:instruction-stamp\s+([a-f0-9]{64})\s*-->(?:\r?\n)?/i;
+const EXPERIENCE_WORD_LIMIT = 32;
+export async function loadInstructionSet(repoRoot) {
+    const candidates = [
+        { path: join(repoRoot, 'AGENTS.md'), required: true },
+        { path: join(repoRoot, 'docs', 'AGENTS.md'), required: false },
+        { path: join(repoRoot, '.agent', 'AGENTS.md'), required: false }
+    ];
+    const verifier = new InstructionStampVerifier();
+    const sources = [];
+    for (const candidate of candidates) {
+        const source = await readStampedInstruction(candidate.path, repoRoot, verifier, candidate.required);
+        if (source) {
+            sources.push(source);
+        }
+    }
+    const combined = sources.map((source) => source.content).join('\n\n').trim();
+    const hash = combined ? createHash('sha256').update(combined, 'utf8').digest('hex') : '';
+    const promptPacks = await loadPromptPacks(repoRoot);
+    const experienceMaxWords = resolveExperienceWordLimit();
+    return {
+        hash,
+        sources,
+        combined,
+        promptPacks,
+        experienceMaxWords
+    };
+}
+class InstructionStampVerifier {
+    verify(raw, relativePath) {
+        const match = INSTRUCTION_STAMP_PATTERN.exec(raw);
+        if (!match) {
+            logInstructionGuard(`Instruction ${relativePath} is missing a codex:instruction-stamp header.`);
+            throw new Error(`Instruction ${relativePath} is not stamped.`);
+        }
+        const declared = match[1].toLowerCase();
+        const body = raw.slice(match[0].length);
+        const computed = createHash('sha256').update(body, 'utf8').digest('hex');
+        if (computed !== declared) {
+            logInstructionGuard(`Instruction ${relativePath} stamp mismatch. expected ${declared}, computed ${computed}.`);
+            throw new Error(`Instruction ${relativePath} stamp mismatch.`);
+        }
+        return { content: body, stamp: declared };
+    }
+}
+async function readStampedInstruction(absolutePath, repoRoot, verifier, required) {
+    try {
+        const raw = await readFile(absolutePath, 'utf8');
+        if (!raw.trim()) {
+            return null;
+        }
+        const relativePath = relative(repoRoot, absolutePath);
+        const verified = verifier.verify(raw, relativePath);
+        const trimmed = verified.content.trim();
+        if (!trimmed) {
+            return null;
+        }
+        return {
+            path: relativePath,
+            content: trimmed,
+            stamp: verified.stamp
+        };
+    }
+    catch (error) {
+        const err = error;
+        if (err && err.code === 'ENOENT') {
+            return null;
+        }
+        if (!required) {
+            const relativePath = relative(repoRoot, absolutePath);
+            logInstructionGuard(`Skipping optional instruction ${relativePath}: ${error?.message ?? String(error)}`);
+            return null;
+        }
+        throw error;
+    }
+}
+function resolveExperienceWordLimit(env = process.env) {
+    const configured = env.TFGRPO_EXPERIENCE_MAX_WORDS;
+    if (!configured || !configured.trim()) {
+        return EXPERIENCE_WORD_LIMIT;
+    }
+    const parsed = Number.parseInt(configured, 10);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        throw new Error('TFGRPO_EXPERIENCE_MAX_WORDS must be a positive integer.');
+    }
+    if (parsed > EXPERIENCE_WORD_LIMIT) {
+        const message = `TFGRPO_EXPERIENCE_MAX_WORDS ${parsed} exceeds the guardrail of ${EXPERIENCE_WORD_LIMIT}.`;
+        logInstructionGuard(message);
+        throw new Error(message);
+    }
+    return parsed;
+}
+function logInstructionGuard(message) {
+    if (typeof process !== 'undefined' && typeof process.emitWarning === 'function') {
+        process.emitWarning(message, { code: 'instruction-guard' });
+    }
+}

package/dist/packages/orchestrator/src/instructions/promptPacks.js

@@ -0,0 +1,151 @@
+import { createHash } from 'node:crypto';
+import { readFile, readdir, stat } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+const PROMPT_SECTIONS = ['system', 'inject', 'summarize', 'extract', 'optimize'];
+const PROMPT_PACK_DIR = ['.agent', 'prompts', 'prompt-packs'];
+export async function loadPromptPacks(repoRoot) {
+    const manifestPaths = await discoverPromptPackManifests(repoRoot);
+    const packs = [];
+    for (const manifestPath of manifestPaths) {
+        packs.push(await loadPromptPack(manifestPath, repoRoot));
+    }
+    packs.sort((a, b) => {
+        if (a.domain === b.domain) {
+            return a.id.localeCompare(b.id);
+        }
+        return a.domain.localeCompare(b.domain);
+    });
+    return packs;
+}
+async function discoverPromptPackManifests(repoRoot) {
+    const baseDir = join(repoRoot, ...PROMPT_PACK_DIR);
+    let entries;
+    try {
+        entries = await readdir(baseDir, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    const manifests = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        const manifestPath = join(baseDir, entry.name, 'manifest.json');
+        try {
+            await stat(manifestPath);
+            manifests.push(manifestPath);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                continue;
+            }
+            throw error;
+        }
+    }
+    return manifests;
+}
+async function loadPromptPack(manifestPath, repoRoot) {
+    let parsed;
+    try {
+        const raw = await readFile(manifestPath, 'utf8');
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`Failed to read prompt pack manifest at ${relative(repoRoot, manifestPath)}: ${String(error)}`);
+    }
+    validateManifest(parsed, manifestPath, repoRoot);
+    const sections = {
+        system: [],
+        inject: [],
+        summarize: [],
+        extract: [],
+        optimize: []
+    };
+    sections.system = [await loadSectionSource(repoRoot, 'system', parsed.system, manifestPath)];
+    sections.inject = await loadSectionArray(repoRoot, 'inject', parsed.inject ?? [], manifestPath);
+    sections.summarize = await loadSectionArray(repoRoot, 'summarize', parsed.summarize ?? [], manifestPath);
+    sections.extract = await loadSectionArray(repoRoot, 'extract', parsed.extract ?? [], manifestPath);
+    sections.optimize = await loadSectionArray(repoRoot, 'optimize', parsed.optimize ?? [], manifestPath);
+    const allSources = PROMPT_SECTIONS.flatMap((section) => sections[section]);
+    if (allSources.length === 0) {
+        throw new Error(`Prompt pack ${parsed.id} ${parsed.domain} has no sources defined (${relative(repoRoot, manifestPath)})`);
+    }
+    const computedStamp = computePromptPackStamp(allSources);
+    if (!parsed.stamp) {
+        throw new Error(`Prompt pack ${parsed.id} is missing a stamp (manifest: ${relative(repoRoot, manifestPath)})`);
+    }
+    if (computedStamp !== parsed.stamp) {
+        throw new Error(`Prompt pack ${parsed.id} stamp mismatch. expected ${parsed.stamp}, computed ${computedStamp} (${relative(repoRoot, manifestPath)})`);
+    }
+    const experienceSlots = Number.isInteger(parsed.experienceSlots) && parsed.experienceSlots >= 0
+        ? parsed.experienceSlots
+        : 0;
+    return {
+        id: parsed.id,
+        domain: parsed.domain,
+        stamp: parsed.stamp,
+        experienceSlots,
+        sections,
+        sources: allSources
+    };
+}
+function validateManifest(manifest, manifestPath, repoRoot) {
+    const missing = [];
+    if (!manifest.id) {
+        missing.push('id');
+    }
+    if (!manifest.domain) {
+        missing.push('domain');
+    }
+    if (!manifest.system) {
+        missing.push('system');
+    }
+    if (missing.length > 0) {
+        throw new Error(`Prompt pack manifest ${relative(repoRoot, manifestPath)} missing required fields: ${missing.join(', ')}`);
+    }
+}
+async function loadSectionArray(repoRoot, section, paths, manifestPath) {
+    const sources = [];
+    for (const relPath of paths) {
+        sources.push(await loadSectionSource(repoRoot, section, relPath, manifestPath));
+    }
+    return sources;
+}
+async function loadSectionSource(repoRoot, section, relativePath, manifestPath) {
+    const absolutePath = join(repoRoot, relativePath);
+    let raw;
+    try {
+        raw = await readFile(absolutePath, 'utf8');
+    }
+    catch (error) {
+        throw new Error(`Failed to read prompt source ${relative(repoRoot, absolutePath)} referenced by ${relative(repoRoot, manifestPath)}: ${String(error)}`);
+    }
+    const content = raw.trim();
+    if (!content) {
+        throw new Error(`Prompt source ${relative(repoRoot, absolutePath)} referenced by ${relative(repoRoot, manifestPath)} is empty`);
+    }
+    return {
+        section,
+        path: relative(repoRoot, absolutePath),
+        content
+    };
+}
+export function computePromptPackStamp(sources) {
+    const hash = createHash('sha256');
+    const sorted = [...sources].sort((a, b) => {
+        if (a.section === b.section) {
+            return a.path.localeCompare(b.path);
+        }
+        return PROMPT_SECTIONS.indexOf(a.section) - PROMPT_SECTIONS.indexOf(b.section);
+    });
+    for (const source of sorted) {
+        hash.update(`${source.section}:${source.path}\n`, 'utf8');
+        hash.update(source.content, 'utf8');
+        hash.update('\n', 'utf8');
+    }
+    return hash.digest('hex');
+}