@kbediako/codex-orchestrator 0.1.0

package/dist/orchestrator/src/persistence/PersistenceCoordinator.js

@@ -0,0 +1,65 @@
+import { logger } from '../logger.js';
+import { TaskStateStoreLockError } from './TaskStateStore.js';
+/**
+ * Subscribes to `run:completed` events and persists run metadata locally
+ * leveraging the TaskStateStore and RunManifestWriter. Errors are surfaced via
+ * the optional `onError` callback but do not throw to avoid crashing the manager.
+ */
+export class PersistenceCoordinator {
+    eventBus;
+    stateStore;
+    manifestWriter;
+    options;
+    unsubscribe;
+    constructor(eventBus, stateStore, manifestWriter, options = {}) {
+        this.eventBus = eventBus;
+        this.stateStore = stateStore;
+        this.manifestWriter = manifestWriter;
+        this.options = options;
+    }
+    start() {
+        if (this.unsubscribe) {
+            return;
+        }
+        this.unsubscribe = this.eventBus.on('run:completed', (event) => {
+            void this.handleRunCompleted(event.payload);
+        });
+    }
+    stop() {
+        if (this.unsubscribe) {
+            this.unsubscribe();
+            this.unsubscribe = undefined;
+        }
+    }
+    async handleRunCompleted(summary) {
+        let stateStoreError = null;
+        try {
+            await this.stateStore.recordRun(summary);
+        }
+        catch (error) {
+            stateStoreError = error;
+            if (error instanceof TaskStateStoreLockError) {
+                logger.warn(`Task state snapshot skipped for task ${summary.taskId} (run ${summary.runId}): ${error.message}`);
+            }
+            else {
+                logger.error(`Task state snapshot failed for task ${summary.taskId} (run ${summary.runId})`, error);
+            }
+        }
+        try {
+            await this.manifestWriter.write(summary);
+        }
+        catch (error) {
+            this.options.onError?.(error, summary);
+            if (!this.options.onError) {
+                logger.error(`PersistenceCoordinator manifest write error for task ${summary.taskId} (run ${summary.runId})`, error);
+            }
+            return;
+        }
+        if (stateStoreError) {
+            this.options.onError?.(stateStoreError, summary);
+            if (!this.options.onError) {
+                logger.warn(`Task state snapshot not recorded for task ${summary.taskId} (run ${summary.runId})`);
+            }
+        }
+    }
+}

package/dist/orchestrator/src/persistence/RunManifestWriter.js

@@ -0,0 +1,23 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { sanitizeTaskId } from './sanitizeTaskId.js';
+import { sanitizeRunId } from './sanitizeRunId.js';
+/**
+ * Stores the run-level manifest under `.runs/<taskId>/<runId>/manifest.json`
+ * so downstream tooling (cloud-sync worker, reviewers) can ingest the payload.
+ */
+export class RunManifestWriter {
+    runsDir;
+    constructor(options = {}) {
+        this.runsDir = options.runsDir ?? join(process.cwd(), '.runs');
+    }
+    async write(summary) {
+        const safeTaskId = sanitizeTaskId(summary.taskId);
+        const safeRunId = sanitizeRunId(summary.runId);
+        const runDir = join(this.runsDir, safeTaskId, safeRunId);
+        await mkdir(runDir, { recursive: true });
+        const manifestPath = join(runDir, 'manifest.json');
+        await writeFile(manifestPath, JSON.stringify(summary, null, 2), 'utf-8');
+        return manifestPath;
+    }
+}

package/dist/orchestrator/src/persistence/TaskStateStore.js

@@ -0,0 +1,172 @@
+import { mkdir, readFile, rm } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { acquireLockWithRetry } from './lockFile.js';
+import { sanitizeTaskId } from './sanitizeTaskId.js';
+import { writeAtomicFile } from './writeAtomicFile.js';
+export class TaskStateStoreLockError extends Error {
+    taskId;
+    constructor(message, taskId) {
+        super(message);
+        this.taskId = taskId;
+        this.name = 'TaskStateStoreLockError';
+    }
+}
+/**
+ * Persists orchestrator run history per task under `/out/<taskId>/runs.json`.
+ * Uses advisory lock files (`.runs/<taskId>.lock`) to guard concurrent writers
+ * and performs atomic writes via temporary files + rename.
+ */
+export class TaskStateStore {
+    outDir;
+    runsDir;
+    lockRetry;
+    constructor(options = {}) {
+        this.outDir = options.outDir ?? join(process.cwd(), 'out');
+        this.runsDir = options.runsDir ?? join(process.cwd(), '.runs');
+        const defaults = {
+            maxAttempts: 5,
+            initialDelayMs: 100,
+            backoffFactor: 2,
+            maxDelayMs: 1000
+        };
+        const overrides = options.lockRetry ?? {};
+        const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));
+        this.lockRetry = { ...defaults, ...sanitizedOverrides };
+    }
+    async recordRun(summary) {
+        const safeTaskId = sanitizeTaskId(summary.taskId);
+        const lockPath = this.buildLockPath(safeTaskId);
+        await this.acquireLock(safeTaskId, lockPath);
+        try {
+            await this.ensureDirectory(this.outDir);
+            const taskOutDir = join(this.outDir, safeTaskId);
+            await this.ensureDirectory(taskOutDir);
+            const snapshotPath = join(taskOutDir, 'runs.json');
+            const snapshot = await this.loadSnapshot(snapshotPath, safeTaskId);
+            const updated = this.mergeSnapshot(snapshot, { ...summary, taskId: safeTaskId });
+            await writeAtomicFile(snapshotPath, JSON.stringify(updated, null, 2));
+        }
+        finally {
+            await this.releaseLock(lockPath);
+        }
+    }
+    buildLockPath(taskId) {
+        const safeTaskId = sanitizeTaskId(taskId);
+        return join(this.runsDir, `${safeTaskId}.lock`);
+    }
+    async acquireLock(taskId, lockPath) {
+        await acquireLockWithRetry({
+            taskId,
+            lockPath,
+            retry: this.lockRetry,
+            ensureDirectory: () => this.ensureDirectory(dirname(lockPath)),
+            createError: (id, attempts) => new TaskStateStoreLockError(`Failed to acquire task state lock for ${id} after ${attempts} attempts`, id)
+        });
+    }
+    async releaseLock(lockPath) {
+        await rm(lockPath, { force: true });
+    }
+    async ensureDirectory(path) {
+        await mkdir(path, { recursive: true });
+    }
+    async loadSnapshot(path, taskId) {
+        try {
+            const data = await readFile(path, 'utf-8');
+            const parsed = JSON.parse(data);
+            if (!parsed || parsed.taskId !== taskId) {
+                throw new Error(`Invalid snapshot contents for task ${taskId}`);
+            }
+            return parsed;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                const legacyPath = join(dirname(path), 'state.json');
+                const legacy = await this.tryLoadLegacySnapshot(legacyPath, taskId);
+                return legacy ?? { taskId, lastRunAt: '', runs: [] };
+            }
+            if (error instanceof SyntaxError) {
+                throw new Error(`Corrupted snapshot at ${path}: ${error.message}`);
+            }
+            throw error;
+        }
+    }
+    async tryLoadLegacySnapshot(path, taskId) {
+        try {
+            const data = await readFile(path, 'utf-8');
+            const parsed = JSON.parse(data);
+            if (parsed &&
+                typeof parsed.taskId === 'string' &&
+                parsed.taskId === taskId &&
+                Array.isArray(parsed.runs)) {
+                return parsed;
+            }
+            return null;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return null;
+            }
+            return null;
+        }
+    }
+    mergeSnapshot(snapshot, summary) {
+        const runs = snapshot.runs;
+        const existingIndex = runs.findIndex((run) => run.runId === summary.runId);
+        if (existingIndex !== -1) {
+            const updatedRuns = runs.slice();
+            updatedRuns[existingIndex] = summary;
+            updatedRuns.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+            const lastRunAt = updatedRuns.length > 0 ? updatedRuns[updatedRuns.length - 1].timestamp : snapshot.lastRunAt;
+            return {
+                taskId: snapshot.taskId,
+                lastRunAt,
+                runs: updatedRuns
+            };
+        }
+        if (runs.length === 0) {
+            runs.push(summary);
+            return {
+                taskId: snapshot.taskId,
+                lastRunAt: summary.timestamp,
+                runs
+            };
+        }
+        const lastTimestamp = runs[runs.length - 1].timestamp;
+        if (lastTimestamp.localeCompare(summary.timestamp) <= 0) {
+            runs.push(summary);
+            return {
+                taskId: snapshot.taskId,
+                lastRunAt: summary.timestamp,
+                runs
+            };
+        }
+        const insertIndex = this.findInsertIndex(runs, summary.timestamp);
+        runs.splice(insertIndex, 0, summary);
+        const lastRunAt = runs[runs.length - 1].timestamp;
+        return {
+            taskId: snapshot.taskId,
+            lastRunAt,
+            runs
+        };
+    }
+    findInsertIndex(runs, timestamp) {
+        let low = 0;
+        let high = runs.length - 1;
+        let result = runs.length;
+        while (low <= high) {
+            const mid = (low + high) >> 1;
+            const candidate = runs[mid];
+            if (!candidate) {
+                break;
+            }
+            if (candidate.timestamp.localeCompare(timestamp) > 0) {
+                result = mid;
+                high = mid - 1;
+            }
+            else {
+                low = mid + 1;
+            }
+        }
+        return result;
+    }
+}

package/dist/orchestrator/src/persistence/identifierGuards.js

@@ -0,0 +1 @@
+export const WINDOWS_FORBIDDEN_CHARACTERS = new Set(['<', '>', ':', '"', '|', '?', '*']);

package/dist/orchestrator/src/persistence/lockFile.js

@@ -0,0 +1,26 @@
+import { open } from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+export async function acquireLockWithRetry(params) {
+    await params.ensureDirectory();
+    const { maxAttempts, initialDelayMs, backoffFactor, maxDelayMs } = params.retry;
+    let attempt = 0;
+    let delayMs = initialDelayMs;
+    while (attempt < maxAttempts) {
+        attempt += 1;
+        try {
+            const handle = await open(params.lockPath, 'wx');
+            await handle.close();
+            return;
+        }
+        catch (error) {
+            if (error.code !== 'EEXIST') {
+                throw error;
+            }
+            if (attempt >= maxAttempts) {
+                throw params.createError(params.taskId, attempt);
+            }
+            await delay(Math.min(delayMs, maxDelayMs));
+            delayMs = Math.min(delayMs * backoffFactor, maxDelayMs);
+        }
+    }
+}

package/dist/orchestrator/src/persistence/sanitizeIdentifier.js

@@ -0,0 +1,26 @@
+import { WINDOWS_FORBIDDEN_CHARACTERS } from './identifierGuards.js';
+export function sanitizeIdentifier(kind, value) {
+    const label = kind === 'task' ? 'task' : 'run';
+    if (!value) {
+        throw new Error(`Invalid ${label} ID: value must be a non-empty string.`);
+    }
+    for (const char of value) {
+        const codePoint = char.codePointAt(0);
+        if (codePoint !== undefined && (codePoint <= 31 || codePoint === 127)) {
+            throw new Error(`Invalid ${label} ID "${value}": control characters are not allowed.`);
+        }
+        if (WINDOWS_FORBIDDEN_CHARACTERS.has(char)) {
+            throw new Error(`Invalid ${label} ID "${value}": character "${char}" is not allowed.`);
+        }
+    }
+    if (value.startsWith('.')) {
+        throw new Error(`Invalid ${label} ID "${value}": leading dots are not allowed.`);
+    }
+    if (value === '..') {
+        throw new Error(`Invalid ${label} ID "${value}": traversal sequences are not allowed.`);
+    }
+    if (value.includes('/') || value.includes('\\')) {
+        throw new Error(`Invalid ${label} ID "${value}": slashes are not allowed.`);
+    }
+    return value;
+}

package/dist/orchestrator/src/persistence/sanitizeRunId.js

@@ -0,0 +1,8 @@
+import { sanitizeIdentifier } from './sanitizeIdentifier.js';
+/**
+ * Validates run identifiers before they are used in filesystem paths.
+ * Rejects traversal sequences and characters that are unsafe on Windows.
+ */
+export function sanitizeRunId(runId) {
+    return sanitizeIdentifier('run', runId);
+}

package/dist/orchestrator/src/persistence/sanitizeTaskId.js

@@ -0,0 +1,8 @@
+import { sanitizeIdentifier } from './sanitizeIdentifier.js';
+/**
+ * Validates task identifiers before they are used in filesystem paths.
+ * Rejects traversal sequences and characters that would break directory layout.
+ */
+export function sanitizeTaskId(taskId) {
+    return sanitizeIdentifier('task', taskId);
+}

package/dist/orchestrator/src/persistence/writeAtomicFile.js

@@ -0,0 +1,4 @@
+import { writeAtomicFile as writeAtomicFileInternal } from '../utils/atomicWrite.js';
+export async function writeAtomicFile(destination, contents) {
+    await writeAtomicFileInternal(destination, contents, { ensureDir: false, encoding: 'utf-8' });
+}

package/dist/orchestrator/src/privacy/guard.js

@@ -0,0 +1,111 @@
+const REDACTION_TOKEN = '[REDACTED]';
+export class PrivacyGuard {
+    now;
+    mode;
+    decisions = [];
+    totals = {
+        total: 0,
+        redacted: 0,
+        blocked: 0,
+        allowed: 0
+    };
+    constructor(options = {}) {
+        this.mode = options.mode ?? 'shadow';
+        this.now = options.now ?? (() => new Date());
+    }
+    getMode() {
+        return this.mode;
+    }
+    async process(frame, context) {
+        this.totals.total += 1;
+        const decision = this.evaluateFrame(frame);
+        const finalDecision = this.mode === 'shadow' && decision.action !== 'allow'
+            ? { action: 'allow', rule: decision.rule, reason: 'shadow-mode' }
+            : decision;
+        const timestamp = this.now().toISOString();
+        this.decisions.push({
+            handleId: context.handleId,
+            sequence: frame.sequence,
+            action: finalDecision.action,
+            rule: finalDecision.rule,
+            reason: finalDecision.reason,
+            timestamp
+        });
+        if (finalDecision.action === 'allow') {
+            this.totals.allowed += 1;
+            return { frame, decision: finalDecision };
+        }
+        if (finalDecision.action === 'redact') {
+            this.totals.redacted += 1;
+            const redactedFrame = this.redactFrame(frame);
+            return { frame: redactedFrame, decision: finalDecision };
+        }
+        this.totals.blocked += 1;
+        return { frame: null, decision: finalDecision };
+    }
+    getMetrics() {
+        return {
+            mode: this.mode,
+            totalFrames: this.totals.total,
+            redactedFrames: this.totals.redacted,
+            blockedFrames: this.totals.blocked,
+            allowedFrames: this.totals.allowed,
+            decisions: [...this.decisions]
+        };
+    }
+    reset() {
+        this.decisions.length = 0;
+        this.totals = {
+            total: 0,
+            redacted: 0,
+            blocked: 0,
+            allowed: 0
+        };
+    }
+    evaluateFrame(frame) {
+        const event = frame.event;
+        if (event.type !== 'exec:chunk') {
+            return { action: 'allow' };
+        }
+        const { data } = event.payload;
+        if (!data) {
+            return { action: 'allow' };
+        }
+        if (PRIVATE_KEY_PATTERN.test(data)) {
+            return {
+                action: 'block',
+                rule: 'private-key',
+                reason: 'detected private key material'
+            };
+        }
+        if (SENSITIVE_TOKEN_PATTERN.test(data) || AWS_KEY_PATTERN.test(data)) {
+            return {
+                action: 'redact',
+                rule: 'sensitive-token',
+                reason: 'detected high-risk token'
+            };
+        }
+        return { action: 'allow' };
+    }
+    redactFrame(frame) {
+        const event = frame.event;
+        if (event.type !== 'exec:chunk') {
+            return frame;
+        }
+        const payload = {
+            ...event.payload,
+            data: REDACTION_TOKEN,
+            bytes: Buffer.byteLength(REDACTION_TOKEN, 'utf8')
+        };
+        return {
+            ...frame,
+            event: {
+                ...event,
+                payload
+            }
+        };
+    }
+}
+const PRIVATE_KEY_PATTERN = /-----BEGIN (?:RSA |EC )?PRIVATE KEY-----/i;
+const SENSITIVE_TOKEN_PATTERN = /(secret|password|api[_-]?key|token)\s*[:=]\s*[^\s]{6,}/i;
+const AWS_KEY_PATTERN = /AKIA[0-9A-Z]{16}/;

package/dist/orchestrator/src/scheduler/index.js

@@ -0,0 +1 @@
+export { createSchedulerPlan, finalizeSchedulerPlan, serializeSchedulerPlan, buildSchedulerRunSummary } from './plan.js';

package/dist/orchestrator/src/scheduler/plan.js

@@ -0,0 +1,171 @@
+export function createSchedulerPlan(request, options = {}) {
+    const now = options.now ?? (() => new Date());
+    const requestedAt = now().toISOString();
+    const prefix = options.instancePrefix ?? request.task.id.toLowerCase();
+    const groupSize = extractTfgrpoGroupSize(request);
+    const recovery = {
+        heartbeatIntervalSeconds: request.schedule.recovery.heartbeatIntervalSeconds,
+        missingHeartbeatTimeoutSeconds: request.schedule.recovery.missingHeartbeatTimeoutSeconds,
+        maxRetries: request.schedule.recovery.maxRetries
+    };
+    const slots = buildCapabilitySlots(request);
+    const { minInstances, maxInstances } = request.schedule;
+    if (minInstances > maxInstances) {
+        throw new Error(`Scheduler minInstances (${minInstances}) cannot exceed maxInstances (${maxInstances}).`);
+    }
+    if (minInstances > slots.length) {
+        const capacity = slots.length;
+        const slotLabel = capacity === 1 ? 'slot' : 'slots';
+        throw new Error(`Scheduler requires at least ${minInstances} instances but only ${capacity} fan-out ${slotLabel} ` +
+            'are available. Adjust the schedule fanOut or lower minInstances.');
+    }
+    const targetCount = Math.max(minInstances, Math.min(maxInstances, slots.length));
+    const assignments = [];
+    const capabilityCounters = new Map();
+    for (let i = 0; i < targetCount; i += 1) {
+        const slot = slots[i];
+        const counter = (capabilityCounters.get(slot.capability) ?? 0) + 1;
+        capabilityCounters.set(slot.capability, counter);
+        const instanceId = `${prefix}-${slot.capability}-${String(counter).padStart(2, '0')}`;
+        const attempt = {
+            number: 1,
+            assignedAt: requestedAt,
+            startedAt: null,
+            completedAt: null,
+            status: 'pending',
+            recoveryCheckpoints: []
+        };
+        const groupIndex = groupSize && groupSize > 0 ? ((i % groupSize) + 1) : null;
+        assignments.push({
+            instanceId,
+            capability: slot.capability,
+            status: 'assigned',
+            assignedAt: requestedAt,
+            completedAt: null,
+            attempts: [attempt],
+            metadata: {
+                weight: slot.weight,
+                maxConcurrency: slot.maxConcurrency,
+                ...(groupIndex ? { groupIndex } : {})
+            }
+        });
+    }
+    return {
+        mode: 'multi-instance',
+        requestedAt,
+        minInstances: request.schedule.minInstances,
+        maxInstances: request.schedule.maxInstances,
+        recovery,
+        assignments
+    };
+}
+export function finalizeSchedulerPlan(plan, finalStatus, timestamp) {
+    const attemptStatus = finalStatus === 'succeeded' ? 'completed' : finalStatus === 'running' ? 'running' : 'failed';
+    const isTerminal = finalStatus !== 'running';
+    for (const assignment of plan.assignments) {
+        assignment.status = finalStatus;
+        if (isTerminal) {
+            assignment.completedAt = timestamp;
+        }
+        const latest = assignment.attempts[assignment.attempts.length - 1];
+        if (latest) {
+            if (!latest.startedAt) {
+                latest.startedAt = timestamp;
+            }
+            if (isTerminal) {
+                latest.completedAt = timestamp;
+            }
+            latest.status = attemptStatus;
+            if (attemptStatus === 'failed') {
+                latest.recoveryCheckpoints.push({
+                    timestamp,
+                    reason: 'missed-heartbeat',
+                    action: 'requeue',
+                    detail: 'Finalizer detected incomplete execution; queued for recovery.'
+                });
+            }
+        }
+    }
+}
+export function serializeSchedulerPlan(plan) {
+    return {
+        mode: plan.mode,
+        requested_at: plan.requestedAt,
+        min_instances: plan.minInstances,
+        max_instances: plan.maxInstances,
+        recovery: {
+            heartbeat_interval_seconds: plan.recovery.heartbeatIntervalSeconds,
+            missing_heartbeat_timeout_seconds: plan.recovery.missingHeartbeatTimeoutSeconds,
+            max_retries: plan.recovery.maxRetries
+        },
+        assignments: plan.assignments.map(serializeAssignment)
+    };
+}
+export function buildSchedulerRunSummary(plan) {
+    const assignments = plan.assignments.map((assignment) => ({
+        instanceId: assignment.instanceId,
+        capability: assignment.capability,
+        status: assignment.status,
+        attempts: assignment.attempts.length,
+        lastCompletedAt: assignment.completedAt
+    }));
+    return {
+        mode: plan.mode,
+        recovery: plan.recovery,
+        assignments
+    };
+}
+function buildCapabilitySlots(request) {
+    const slots = [];
+    for (const entry of request.schedule.fanOut) {
+        const maxConcurrency = Math.max(1, entry.maxConcurrency ?? 1);
+        for (let index = 0; index < maxConcurrency; index += 1) {
+            slots.push({
+                capability: entry.capability,
+                weight: entry.weight,
+                maxConcurrency
+            });
+        }
+    }
+    if (slots.length === 0) {
+        slots.push({ capability: 'general', weight: 1, maxConcurrency: 1 });
+    }
+    slots.sort((a, b) => b.weight - a.weight);
+    return slots;
+}
+function serializeAssignment(assignment) {
+    return {
+        instance_id: assignment.instanceId,
+        capability: assignment.capability,
+        status: assignment.status,
+        assigned_at: assignment.assignedAt,
+        completed_at: assignment.completedAt,
+        metadata: assignment.metadata,
+        attempts: assignment.attempts.map(serializeAttempt)
+    };
+}
+function serializeAttempt(attempt) {
+    return {
+        number: attempt.number,
+        assigned_at: attempt.assignedAt,
+        started_at: attempt.startedAt ?? null,
+        completed_at: attempt.completedAt ?? null,
+        status: attempt.status,
+        recovery_checkpoints: attempt.recoveryCheckpoints
+    };
+}
+function extractTfgrpoGroupSize(request) {
+    const metadata = request.metadata;
+    if (!metadata || typeof metadata !== 'object') {
+        return null;
+    }
+    const tfgrpo = metadata.tfgrpo;
+    if (!tfgrpo || typeof tfgrpo !== 'object') {
+        return null;
+    }
+    const value = tfgrpo.groupSize;
+    if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
+        return null;
+    }
+    return Math.trunc(value);
+}

package/dist/orchestrator/src/scheduler/types.js

@@ -0,0 +1 @@
+export {};

package/dist/orchestrator/src/sync/CloudRunsClient.js

@@ -0,0 +1 @@
+export {};