openworkgraph 0.2.12

package/dist/scheduler.js

@@ -0,0 +1,344 @@
+import { randomUUID } from 'node:crypto';
+import { isTerminalRunStatus } from "./vendor/protocol/index.js";
+import { ServiceError } from './errors.js';
+import { Runs } from './runs.js';
+export class Scheduler {
+    runs;
+    backend;
+    options;
+    epoch;
+    jobs = new Map();
+    releases = new Map();
+    lockAcquisitions = new Map();
+    pendingReplies = new Set();
+    replyBuffers = new Map();
+    contendedProjects = new Set();
+    publicationRetries = new Map();
+    ticking = false;
+    dispatched = false;
+    get isUnstarted() { return !this.dispatched && !this.ticking && !this.jobs.size && !this.releases.size && !this.lockAcquisitions.size; }
+    constructor(runs, backend, options = {}) {
+        this.runs = runs;
+        this.backend = backend;
+        this.options = options;
+        this.epoch = options.epoch ?? randomUUID();
+    }
+    context(id) { return { epoch: this.epoch, emit: event => this.emit(id, event) }; }
+    owned(id) { return this.runs.db.prepare('SELECT epoch FROM run_runtime WHERE run_id=?').get(id)?.['epoch'] === this.epoch; }
+    lockScope(run) {
+        const runtime = this.runs.runtime(run.id);
+        if (runtime.details.kind !== 'execution' || !this.options.acquireProjectLock)
+            return undefined;
+        const canonicalPath = String(runtime.details.canonicalPath);
+        // The kernel lock excludes other runtimes, not independent runs in this
+        // scheduler. Concurrent runs (including different batches) share it.
+        return { key: canonicalPath, canonicalPath };
+    }
+    activeProjectRuns(canonicalPath) {
+        return Boolean(this.runs.db.prepare(`SELECT 1 FROM occupancy o JOIN run_runtime rt ON rt.run_id=o.run_id
+      WHERE rt.epoch=? AND json_extract(rt.details,'$.kind')='execution'
+        AND json_extract(rt.details,'$.canonicalPath')=? LIMIT 1`).get(this.epoch, canonicalPath));
+    }
+    release(id) {
+        const run = this.runs.get(id), scope = this.lockScope(run);
+        if (!scope || this.activeProjectRuns(scope.canonicalPath))
+            return;
+        const release = this.releases.get(scope.key);
+        if (release) {
+            release();
+            this.releases.delete(scope.key);
+        }
+    }
+    launch(key, work) {
+        if (this.jobs.has(key))
+            return;
+        const job = Promise.resolve().then(work).catch(error => {
+            const id = key.split(':')[1];
+            if (id && this.owned(id))
+                this.runs.record(id, 'scheduler.error', { message: error instanceof Error ? error.message : String(error) });
+        }).finally(() => { this.jobs.delete(key); });
+        this.jobs.set(key, job);
+    }
+    deferPublication(id, error) {
+        if (!this.owned(id))
+            return;
+        const now = this.options.now?.() ?? Date.now();
+        const message = error instanceof Error ? error.message : String(error);
+        const previous = this.publicationRetries.get(id);
+        const attempts = Math.min((previous?.attempts ?? 0) + 1, 16);
+        const recordedAt = previous?.recordedAt ?? now;
+        if (!previous || previous.message !== message || now - previous.recordedAt >= 30_000) {
+            this.runs.record(id, 'scheduler.error', { message });
+        }
+        this.publicationRetries.set(id, {
+            attempts,
+            nextAt: now + Math.min(30_000, 250 * 2 ** (attempts - 1)),
+            message,
+            recordedAt: !previous || previous.message !== message || now - previous.recordedAt >= 30_000 ? now : recordedAt,
+        });
+    }
+    uncertain(id, error) {
+        if (!this.owned(id))
+            return;
+        const run = this.runs.get(id);
+        if (isTerminalRunStatus(run.status) || ['reconciling', 'finalizing'].includes(run.status))
+            return;
+        this.runs.transition(id, this.runs.runtime(id), 'reconciling', { uncertainty: error instanceof Error ? error.message : String(error) });
+    }
+    /** Epoch fencing plus state checks discard late success after cancellation/recovery. */
+    emit(id, event) {
+        if (!this.owned(id))
+            return;
+        const run = this.runs.get(id);
+        const runtime = this.runs.runtime(id);
+        if (isTerminalRunStatus(run.status))
+            return;
+        if (runtime.details.cancelRequested === true && event.type !== 'stopped' && event.type !== 'progress')
+            return;
+        const buffer = this.replyBuffers.get(id);
+        if (buffer && ['started', 'completed'].includes(event.type)) {
+            buffer.push(event);
+            return;
+        }
+        switch (event.type) {
+            case 'started':
+                if (run.status === 'preparing')
+                    this.runs.transition(id, runtime, 'running', { backendIdentity: event.identity ?? {} });
+                break;
+            case 'progress':
+                this.runs.record(id, 'backend.progress', event.payload);
+                break;
+            case 'question':
+            case 'approval':
+                if (['running', 'waiting_answer', 'waiting_approval'].includes(run.status))
+                    this.runs.openInteraction(id, runtime, event.type === 'question' ? 'question' : 'approval', event.payload, event.id);
+                break;
+            case 'completed':
+                if (this.runs.activeInteraction(id))
+                    this.runs.transition(id, runtime, 'reconciling', { uncertainty: 'Backend completed with unresolved independent actions', unresolvedCompletion: true });
+                else if (run.status === 'running')
+                    this.runs.transition(id, runtime, 'agent_completed', { result: event.result });
+                break;
+            case 'failed':
+                if (['preparing', 'running', 'waiting_answer', 'waiting_approval'].includes(run.status))
+                    this.runs.transition(id, runtime, 'failed', { error: event.error });
+                break;
+            case 'stopped':
+                if (run.status === 'cancelling')
+                    this.runs.transition(id, runtime, 'cancelled');
+                else if (run.status === 'reconciling' && runtime.details.cancelRequested === true) {
+                    this.runs.transition(id, runtime, 'cancelling');
+                    this.runs.transition(id, this.runs.runtime(id), 'cancelled');
+                }
+                break;
+        }
+        if (isTerminalRunStatus(this.runs.get(id).status))
+            this.release(id);
+    }
+    async start(run) {
+        try {
+            if (!await this.ensureProjectLock(run)) {
+                if (!this.owned(run.id))
+                    return;
+                if (this.runs.get(run.id).status === 'cancelling')
+                    this.emit(run.id, { type: 'stopped' });
+                else if (this.runs.get(run.id).status === 'preparing')
+                    this.runs.requeueUnstarted(run.id, this.runs.runtime(run.id));
+                this.contendedProjects.add(run.projectId);
+                // Refill immediately even with capacity=1; retry this project next tick.
+                this.dispatch();
+                return;
+            }
+            if (!this.owned(run.id) || this.runs.get(run.id).status !== 'preparing') {
+                if (this.owned(run.id) && this.runs.get(run.id).status === 'cancelling')
+                    this.emit(run.id, { type: 'stopped' });
+                return;
+            }
+            this.runs.markBackendLaunching(run.id, this.runs.runtime(run.id));
+            await this.backend.start(this.runs.get(run.id), this.runs.snapshot(run.id), this.context(run.id));
+        }
+        catch (error) {
+            this.uncertain(run.id, error);
+        }
+    }
+    dispatch() {
+        let claimed;
+        while ((claimed = this.runs.claimNext(this.epoch, [...this.contendedProjects]))) {
+            const run = claimed;
+            if (this.jobs.has('start:' + run.id)) {
+                this.runs.requeueUnstarted(run.id, this.runs.runtime(run.id));
+                this.contendedProjects.add(run.projectId);
+                continue;
+            }
+            this.launch('start:' + run.id, () => this.start(run));
+        }
+    }
+    async cancel(run) {
+        // A start call may still be acquiring a lock or spawning. Do not confirm stop
+        // until that call settles, otherwise it could launch after capacity is freed.
+        const start = this.jobs.get('start:' + run.id);
+        if (start)
+            return;
+        if (this.runs.runtime(run.id).details.backendLaunchAttempted !== true) {
+            this.emit(run.id, { type: 'stopped' });
+            return;
+        }
+        try {
+            if (await this.backend.cancel(run, this.context(run.id)) === 'confirmed')
+                this.emit(run.id, { type: 'stopped' });
+        }
+        catch (error) {
+            this.uncertain(run.id, error);
+        }
+    }
+    async reconcile(run) {
+        try {
+            if (!await this.ensureProjectLock(run))
+                return;
+            if (!this.backend.reconcile)
+                return;
+            const result = await this.backend.reconcile(run, this.context(run.id));
+            if (!this.owned(run.id) || this.runs.get(run.id).status !== 'reconciling')
+                return;
+            const runtime = this.runs.runtime(run.id);
+            if (result === 'interrupted') {
+                if (runtime.details.cancelRequested === true) {
+                    this.runs.transition(run.id, runtime, 'cancelling');
+                    this.emit(run.id, { type: 'stopped' });
+                }
+                else {
+                    this.runs.transition(run.id, runtime, 'interrupted');
+                    this.release(run.id);
+                }
+            }
+            else if (result === 'resumed' && runtime.details.unresolvedCompletion !== true)
+                this.runs.transition(run.id, runtime, runtime.details.cancelRequested === true ? 'cancelling' : 'running');
+        }
+        catch (error) {
+            this.runs.record(run.id, 'reconcile.error', { message: String(error) });
+        }
+    }
+    async ensureProjectLock(run) {
+        if (!this.owned(run.id) || isTerminalRunStatus(this.runs.get(run.id).status))
+            return false;
+        const scope = this.lockScope(run);
+        if (!scope || this.releases.has(scope.key))
+            return true;
+        let acquisition = this.lockAcquisitions.get(scope.key);
+        if (!acquisition) {
+            acquisition = (async () => {
+                const release = await this.options.acquireProjectLock(run, scope.canonicalPath);
+                if (!release)
+                    return false;
+                const needed = this.activeProjectRuns(scope.canonicalPath);
+                if (!needed) {
+                    release();
+                    return false;
+                }
+                this.releases.set(scope.key, release);
+                return true;
+            })().finally(() => { this.lockAcquisitions.delete(scope.key); });
+            this.lockAcquisitions.set(scope.key, acquisition);
+        }
+        return acquisition;
+    }
+    async deliverReply(run, interaction, answer) {
+        if (!this.backend.reply || this.pendingReplies.has(interaction.id))
+            return;
+        this.pendingReplies.add(interaction.id);
+        const token = this.runs.runtime(run.id);
+        // Some adapters synchronously emit the next action before their reply promise
+        // resolves. Buffer those callbacks until delivery is acknowledged, rather than
+        // dropping completion/questions while the durable Run still says waiting.
+        const events = [];
+        this.replyBuffers.set(run.id, events);
+        try {
+            await this.backend.reply(run, interaction, answer, this.context(run.id));
+            if (!this.owned(run.id))
+                return;
+            const status = this.runs.get(run.id).status;
+            if (['waiting_answer', 'waiting_approval'].includes(status))
+                this.runs.acknowledgeReply(run.id, token.epoch, interaction.id);
+            this.replyBuffers.delete(run.id);
+            for (const event of events)
+                this.emit(run.id, event);
+        }
+        catch (error) {
+            this.uncertain(run.id, error);
+        }
+        finally {
+            this.replyBuffers.delete(run.id);
+            this.pendingReplies.delete(interaction.id);
+        }
+    }
+    /** Nonblocking dispatch. Host calls on events/timer; it never awaits a full turn. */
+    async tick() {
+        if (this.runs.db.isTransaction)
+            throw new ServiceError('CONFLICT', 'Scheduler cannot dispatch inside a database transaction');
+        this.dispatched = true;
+        if (this.ticking)
+            return;
+        this.ticking = true;
+        this.contendedProjects.clear();
+        try {
+            for (const run of this.runs.list()) {
+                if (!this.owned(run.id))
+                    continue;
+                if (isTerminalRunStatus(run.status)) {
+                    this.publicationRetries.delete(run.id);
+                    this.release(run.id);
+                    continue;
+                }
+                if (run.status === 'cancelling')
+                    this.launch('cancel:' + run.id, () => this.cancel(run));
+                if (run.status === 'reconciling')
+                    this.launch('reconcile:' + run.id, () => this.reconcile(run));
+                if (run.status === 'agent_completed')
+                    this.runs.transition(run.id, this.runs.runtime(run.id), 'finalizing');
+                const publicationRetry = this.publicationRetries.get(run.id);
+                const publicationReady = !publicationRetry || (this.options.now?.() ?? Date.now()) >= publicationRetry.nextAt;
+                if (this.runs.get(run.id).status === 'finalizing' && this.options.publish && publicationReady)
+                    this.launch('publish:' + run.id, async () => {
+                        if (!await this.ensureProjectLock(run))
+                            return;
+                        if (!this.owned(run.id) || this.runs.get(run.id).status !== 'finalizing')
+                            return;
+                        try {
+                            await this.options.publish(this.runs.get(run.id), this.runs.runtime(run.id));
+                            this.publicationRetries.delete(run.id);
+                        }
+                        catch (error) {
+                            // Invalid output is permanent. Transport/filesystem faults remain
+                            // finalizing for manifest-based recovery instead of backend reruns.
+                            const permanent = (error instanceof ServiceError && ['INVALID_REQUEST', 'INPUT_BLOCKED', 'UNSUPPORTED_MEDIA_TYPE', 'PAYLOAD_TOO_LARGE', 'NOT_FOUND', 'PROJECT_INACTIVE'].includes(error.code)) || error instanceof SyntaxError || (error instanceof Error && ['ENOENT', 'ELOOP', 'ENOTDIR'].includes(String(error.code)));
+                            if (permanent && this.owned(run.id) && this.runs.get(run.id).status === 'finalizing') {
+                                this.publicationRetries.delete(run.id);
+                                this.runs.transition(run.id, this.runs.runtime(run.id), 'failed', { error: error instanceof Error ? error.message : 'Invalid output', failurePhase: 'publication' });
+                            }
+                            else
+                                this.deferPublication(run.id, error);
+                        }
+                        if (isTerminalRunStatus(this.runs.get(run.id).status))
+                            this.release(run.id);
+                    });
+                if (['waiting_answer', 'waiting_approval'].includes(run.status)) {
+                    const interaction = this.runs.activeInteraction(run.id);
+                    if (interaction?.status === 'answered') {
+                        const row = this.runs.db.prepare('SELECT answer FROM interactions WHERE id=?').get(interaction.id);
+                        this.launch('reply:' + run.id, () => this.deliverReply(run, interaction, JSON.parse(String(row.answer))));
+                    }
+                }
+            }
+            this.dispatch();
+        }
+        finally {
+            this.ticking = false;
+        }
+    }
+    /** Only invoke after host verifies previous instance lost ownership. */
+    async recover() { this.runs.recover(this.epoch); await this.tick(); }
+    /** Test/controlled-adapter drain; not used by production request handlers. */
+    async settled() { while (this.jobs.size)
+        await Promise.all([...this.jobs.values()]); }
+}
+//# sourceMappingURL=scheduler.js.map

package/dist/service.d.ts

@@ -0,0 +1,29 @@
+import type { ServiceInfo } from "./vendor/protocol/index.js";
+import { Auth } from "./auth.js";
+import { Projects } from "./projects.js";
+import { ProjectProbe } from "./project-probe.js";
+import { BusinessApi } from "./api.js";
+import { Events } from "./events.js";
+import { WorkflowRuntime, type WorkflowRuntimeOptions } from "./runtime.js";
+export declare function createService(dataDir?: string, options?: {
+    now?: () => number;
+    codexHome?: string;
+    projectProbe?: ProjectProbe;
+    runtime?: WorkflowRuntimeOptions;
+    npmInstallation?: () => Promise<boolean>;
+    latestNpmVersion?: () => Promise<string>;
+    updateRuntime?: (onExit: () => void) => Promise<void>;
+}): {
+    server: import("http").Server<typeof import("http").IncomingMessage, typeof import("http").ServerResponse>;
+    db: import("node:sqlite").DatabaseSync;
+    directories: import("./directories.js").DataDirectories;
+    info: ServiceInfo;
+    auth: Auth;
+    projects: Projects;
+    api: BusinessApi;
+    events: Events;
+    runtime: WorkflowRuntime;
+    instanceId: `${string}-${string}-${string}-${string}-${string}`;
+    close: () => Promise<void>;
+    abortStartup: () => Promise<void>;
+};