@maiaos/node 0.1.0-canary.44938d4.manual.1

package/dist/harness/codex/transport.js

@@ -0,0 +1,645 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { Effect, Queue, SchemaError, Stream } from "effect";
+import { makeNodeRuntimeError, } from "../../runtime/errors.js";
+import { codexProbePolicy } from "./policy.js";
+import { codexInitializeRequest, codexPostInitializeRecords, parseCodexExecutableVersion, parseCodexInitializeResponse, requireCodexAccountReadResponse, } from "./protocol.js";
+const noOperation = () => undefined;
+const newlineByte = 10;
+const maximumCodexInputRecords = 64;
+const codexResourceLimits = {
+    maximumCpuTimeMilliseconds: 60 * 60 * 1000,
+    maximumMemoryBytes: 2 * 1024 * 1024 * 1024,
+    maximumProcessCount: 64,
+    maximumWallTimeMilliseconds: 65 * 60 * 1000,
+};
+const codexStreamLimits = {
+    maximumDataChunkBytes: 16 * 1024,
+    maximumPendingInputBytes: 64 * 1024,
+    maximumPendingOutputBytes: 256 * 1024,
+};
+export class CodexProbeFailure extends Error {
+    reason;
+    constructor(reason) {
+        super(reason);
+        this.reason = reason;
+        this.name = "CodexProbeFailure";
+    }
+}
+const codexProbeEnvironment = () => {
+    const environment = {};
+    const keys = [
+        "CODEX_HOME",
+        "HOME",
+        "LANG",
+        "LC_ALL",
+        "LOCALAPPDATA",
+        "PATH",
+        "SystemRoot",
+        "TEMP",
+        "TMP",
+        "TMPDIR",
+        "USERPROFILE",
+    ];
+    keys.forEach((key) => {
+        const value = process.env[key];
+        if (value !== undefined)
+            environment[key] = value;
+    });
+    return environment;
+};
+const escalateCleanupFailure = (failure) => Effect.logFatal("maia.node.codex_process_cleanup_failed", new Map([["code", failure.code]])).pipe(Effect.andThen(Effect.sync(() => {
+    try {
+        process.kill(process.pid, "SIGTERM");
+    }
+    catch {
+        // The fatal local diagnostic already records the failed escalation.
+    }
+})));
+const makeCleanupFailureHandler = (command) => {
+    const handle = command.handleCleanupFailure ?? escalateCleanupFailure;
+    let handled = false;
+    return (failure) => Effect.suspend(() => {
+        if (handled)
+            return Effect.void;
+        handled = true;
+        return handle(failure);
+    });
+};
+const makeTestCleanupBarrier = (command, treeId) => {
+    let started = false;
+    return () => {
+        if (command.testCleanupBarrier !== true || started)
+            return Effect.void;
+        started = true;
+        return Effect.tryPromise({
+            catch: () => makeNodeRuntimeError("transport_failed", "The Codex cleanup test barrier could not be written."),
+            try: () => writeFile(join(command.workingDirectory, `codex-cleanup-${treeId}.barrier`), "acknowledged\n", { encoding: "utf8", flag: "wx", mode: 0o600 }),
+        });
+    };
+};
+export class ProcessObserver {
+    onFinish;
+    #resume;
+    #settled = false;
+    constructor(resume, onFinish = noOperation) {
+        this.onFinish = onFinish;
+        this.#resume = resume;
+    }
+    get settled() {
+        return this.#settled;
+    }
+    finish(effect) {
+        if (this.#settled)
+            return;
+        this.#settled = true;
+        this.onFinish();
+        this.#resume(effect);
+    }
+}
+const releaseOwned = Effect.fn("CodexProbe.releaseProcess")(function* (owned) {
+    const cleanup = yield* owned.process.terminate;
+    if (cleanup.state !== "clean") {
+        const failure = makeNodeRuntimeError("resource_cleanup_failed", "The Codex process tree cleanup outcome is unknown.");
+        yield* owned.handleCleanupFailure(failure);
+        return yield* Effect.fail(failure);
+    }
+    yield* owned.writeCleanupBarrier();
+});
+export class CodexTransportFailure extends Error {
+    reason;
+    constructor(reason) {
+        super(reason);
+        this.reason = reason;
+        this.name = "CodexTransportFailure";
+    }
+}
+const transportOutcomeEffect = (outcome) => outcome.kind === "failure"
+    ? Effect.fail(outcome.failure)
+    : Effect.die(outcome.defect);
+class CodexJsonRouteBudget {
+    options;
+    #bytes = 0;
+    #chunkIds = new Set();
+    #records = 0;
+    constructor(options) {
+        this.options = options;
+    }
+    acceptRecord(record) {
+        this.#bytes += record.rawBytes;
+        this.#records += 1;
+        record.chunkIds.forEach((chunkId) => this.#chunkIds.add(chunkId));
+        return this.withinBounds();
+    }
+    acceptStderr(chunkId, rawBytes) {
+        this.#bytes += rawBytes;
+        this.#chunkIds.add(chunkId);
+        return this.withinBounds();
+    }
+    acceptsPartial(rawBytes, chunkCount) {
+        return (this.#bytes + rawBytes <= this.options.maximumBytes &&
+            this.#chunkIds.size + chunkCount <= this.options.maximumChunks);
+    }
+    withinBounds() {
+        return (this.#bytes <= this.options.maximumBytes &&
+            this.#chunkIds.size <= this.options.maximumChunks &&
+            this.#records <= this.options.maximumRecords);
+    }
+}
+const idleRouteOptions = {
+    maximumBytes: codexProbePolicy.maximumTurnOutputBytes,
+    maximumChunks: codexProbePolicy.maximumTurnOutputChunks,
+    maximumRecords: codexProbePolicy.maximumTurnOutputRecords,
+    onFailure: noOperation,
+    onRecord: () => "defer",
+    outputFailureReason: "output_limit_exceeded",
+};
+/**
+ * Owns App Server stdout for the full child-process scope. Parsed values move
+ * through one synchronous route at a time; values produced without a route are
+ * retained in a hard-bounded idle queue for the next phase.
+ */
+export class CodexJsonLineTransport {
+    offerInput;
+    #active;
+    #chunkId = 0;
+    #idleBudget = new CodexJsonRouteBudget(idleRouteOptions);
+    #idleRecords = [];
+    #outcome;
+    #pendingChunkIds = new Set();
+    #pendingParts = [];
+    #pendingRawBytes = 0;
+    #released = false;
+    #waiters = new Set();
+    constructor(offerInput) {
+        this.offerInput = offerInput;
+    }
+    acceptRuntimeExit() {
+        if (!this.finalizePending())
+            return;
+        this.fail(new CodexTransportFailure("runtime_exited"));
+    }
+    acceptRuntimeFailure() {
+        if (this.#outcome !== undefined)
+            return false;
+        this.fail(new CodexTransportFailure("runtime_exited"));
+        return true;
+    }
+    acceptStderr(bytes) {
+        if (this.#outcome !== undefined)
+            return;
+        const chunkId = (this.#chunkId += 1);
+        const route = this.#active;
+        const accepted = route === undefined
+            ? this.#idleBudget.acceptStderr(chunkId, bytes.byteLength)
+            : route.budget.acceptStderr(chunkId, bytes.byteLength);
+        if (!accepted) {
+            this.fail(new CodexTransportFailure(route?.options.outputFailureReason ??
+                idleRouteOptions.outputFailureReason));
+        }
+    }
+    acceptStdout(bytes) {
+        if (this.#outcome !== undefined)
+            return;
+        const chunkId = (this.#chunkId += 1);
+        this.consumeStdoutChunk(Buffer.from(bytes), chunkId, 0);
+    }
+    failure() {
+        return Effect.callback((resume) => {
+            if (this.#outcome !== undefined) {
+                resume(transportOutcomeEffect(this.#outcome));
+                return;
+            }
+            this.#waiters.add(resume);
+            return Effect.sync(() => {
+                this.#waiters.delete(resume);
+            });
+        });
+    }
+    register(options) {
+        if (this.#active !== undefined) {
+            throw new Error("The Codex JSON transport already has an active route.");
+        }
+        const route = {
+            budget: new CodexJsonRouteBudget(options),
+            options,
+            released: false,
+        };
+        this.#active = route;
+        if (this.#outcome === undefined) {
+            this.drainIdle(route);
+        }
+        else {
+            options.onFailure(transportOutcomeEffect(this.#outcome));
+        }
+        return {
+            release: () => {
+                if (route.released)
+                    return;
+                route.released = true;
+                if (this.#active === route)
+                    this.#active = undefined;
+            },
+        };
+    }
+    release() {
+        if (this.#released)
+            return;
+        this.#released = true;
+        this.#active = undefined;
+        this.#waiters.clear();
+    }
+    write(record) {
+        const accepted = this.offerInput(Buffer.from(`${record}\n`, "utf8"));
+        if (!accepted) {
+            this.fail(new CodexTransportFailure("input_limit_exceeded"));
+        }
+    }
+    acceptIdle(record) {
+        if (!this.#idleBudget.acceptRecord(record)) {
+            this.fail(new CodexTransportFailure(idleRouteOptions.outputFailureReason));
+            return;
+        }
+        this.#idleRecords.push(record);
+    }
+    acceptRecord(record) {
+        const route = this.#active;
+        if (route === undefined) {
+            this.acceptIdle(record);
+            return;
+        }
+        if (!route.budget.acceptRecord(record)) {
+            this.fail(new CodexTransportFailure(route.options.outputFailureReason));
+            return;
+        }
+        let disposition;
+        try {
+            disposition = route.options.onRecord(record.value);
+        }
+        catch (error) {
+            this.failObservation(error);
+            return;
+        }
+        if (disposition === "defer")
+            this.acceptIdle(record);
+    }
+    decodeRecord(bytes) {
+        let line;
+        try {
+            line = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        }
+        catch (error) {
+            if (error instanceof TypeError) {
+                throw new CodexTransportFailure("protocol_invalid");
+            }
+            throw error;
+        }
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            if (error instanceof SyntaxError) {
+                throw new CodexTransportFailure("protocol_invalid");
+            }
+            throw error;
+        }
+    }
+    canDrain(route) {
+        return (this.#outcome === undefined && this.#active === route && !route.released);
+    }
+    drainIdle(route) {
+        if (this.#idleRecords.length === 0) {
+            this.#idleBudget = new CodexJsonRouteBudget(idleRouteOptions);
+            return;
+        }
+        const records = this.#idleRecords.splice(0);
+        const deferred = [];
+        let processed = 0;
+        records.some((record) => {
+            if (!this.canDrain(route))
+                return true;
+            if (!route.budget.acceptRecord(record)) {
+                this.fail(new CodexTransportFailure(route.options.outputFailureReason));
+                return true;
+            }
+            processed += 1;
+            let disposition;
+            try {
+                disposition = route.options.onRecord(record.value);
+            }
+            catch (error) {
+                this.failObservation(error);
+                return true;
+            }
+            if (disposition === "defer")
+                deferred.push(record);
+            return false;
+        });
+        this.#idleRecords.unshift(...deferred, ...records.slice(processed));
+        this.rebuildIdleBudget();
+    }
+    fail(failure) {
+        this.finish({ failure, kind: "failure" });
+    }
+    failDefect(defect) {
+        this.finish({ defect, kind: "defect" });
+    }
+    failObservation(error) {
+        if (error instanceof CodexTransportFailure) {
+            this.fail(error);
+            return;
+        }
+        if (error instanceof CodexProbeFailure) {
+            this.fail(new CodexTransportFailure("protocol_invalid"));
+            return;
+        }
+        this.failDefect(error);
+    }
+    finalizePending() {
+        if (this.#pendingRawBytes === 0)
+            return true;
+        this.fail(new CodexTransportFailure("protocol_invalid"));
+        return false;
+    }
+    finish(outcome) {
+        if (this.#outcome !== undefined)
+            return;
+        this.#outcome = outcome;
+        const effect = transportOutcomeEffect(outcome);
+        this.#active?.options.onFailure(effect);
+        this.#waiters.forEach((resume) => {
+            resume(effect);
+        });
+        this.#waiters.clear();
+    }
+    partialWithinBounds() {
+        const budget = this.#active?.budget ?? this.#idleBudget;
+        return budget.acceptsPartial(this.#pendingRawBytes, this.#pendingChunkIds.size);
+    }
+    rebuildIdleBudget() {
+        this.#idleBudget = new CodexJsonRouteBudget(idleRouteOptions);
+        this.#idleRecords.every((record) => {
+            if (this.#idleBudget.acceptRecord(record))
+                return true;
+            this.fail(new CodexTransportFailure(idleRouteOptions.outputFailureReason));
+            return false;
+        });
+    }
+    recordWithinAbsoluteBounds(rawBytes, chunkCount) {
+        const options = this.#active?.options ?? idleRouteOptions;
+        return (rawBytes <= options.maximumBytes && chunkCount <= options.maximumChunks);
+    }
+    appendPartial(remainder, chunkId) {
+        if (remainder.byteLength > 0) {
+            this.#pendingParts.push(Buffer.from(remainder));
+            this.#pendingRawBytes += remainder.byteLength;
+            this.#pendingChunkIds.add(chunkId);
+        }
+        if (this.partialWithinBounds())
+            return;
+        this.fail(new CodexTransportFailure(this.outputFailureReason()));
+    }
+    consumeStdoutChunk(chunk, chunkId, offset) {
+        if (offset >= chunk.byteLength || this.#outcome !== undefined)
+            return;
+        const newline = chunk.indexOf(newlineByte, offset);
+        if (newline === -1) {
+            this.appendPartial(chunk.subarray(offset), chunkId);
+            return;
+        }
+        this.processRecordSegment(chunk.subarray(offset, newline), chunkId);
+        this.consumeStdoutChunk(chunk, chunkId, newline + 1);
+    }
+    outputFailureReason() {
+        return (this.#active?.options.outputFailureReason ??
+            idleRouteOptions.outputFailureReason);
+    }
+    processRecordSegment(segment, chunkId) {
+        const bytes = this.#pendingParts.length === 0
+            ? segment
+            : Buffer.concat([...this.#pendingParts, segment]);
+        const chunkIds = [...this.#pendingChunkIds, chunkId];
+        this.#pendingParts.length = 0;
+        this.#pendingRawBytes = 0;
+        this.#pendingChunkIds.clear();
+        const rawBytes = bytes.byteLength + 1;
+        if (!this.recordWithinAbsoluteBounds(rawBytes, chunkIds.length)) {
+            this.fail(new CodexTransportFailure(this.outputFailureReason()));
+            return;
+        }
+        try {
+            this.acceptRecord({
+                chunkIds,
+                rawBytes,
+                value: this.decodeRecord(bytes),
+            });
+        }
+        catch (error) {
+            if (error instanceof CodexTransportFailure)
+                this.fail(error);
+            else
+                this.failDefect(error);
+        }
+    }
+}
+const missingCodexInput = () => Effect.fail(makeNodeRuntimeError("transport_failed", "The managed Codex process did not expose bounded standard input."));
+const offerCodexInput = (pendingInput, state, bytes) => {
+    if (state.released ||
+        state.pendingBytes + bytes.byteLength >
+            codexStreamLimits.maximumPendingInputBytes) {
+        return false;
+    }
+    state.pendingBytes += bytes.byteLength;
+    if (Queue.offerUnsafe(pendingInput, bytes))
+        return true;
+    state.pendingBytes -= bytes.byteLength;
+    return false;
+};
+const acceptRuntimeFailure = (transport, failure) => Effect.sync(() => transport.acceptRuntimeFailure()).pipe(Effect.flatMap((first) => first && failure !== undefined
+    ? Effect.logWarning("maia.node.codex_transport_failed", new Map([
+        ["failureCode", failure.code],
+        ["failureMessage", failure.message],
+    ]))
+    : Effect.void));
+const runCodexInputWriter = (context) => Effect.forever(Queue.take(context.pendingInput).pipe(Effect.flatMap((bytes) => context.input.write(bytes).pipe(Effect.ensuring(Effect.sync(() => {
+    context.state.pendingBytes -= bytes.byteLength;
+})))))).pipe(Effect.catch((error) => acceptRuntimeFailure(context.transport, error)));
+const consumeCodexOutput = (stream, accept, transport) => Stream.runForEach(stream, (bytes) => Effect.sync(() => {
+    accept(bytes);
+})).pipe(Effect.catch((error) => acceptRuntimeFailure(transport, error)));
+const consumeCodexStdout = (process_, transport) => consumeCodexOutput(process_.standardOutput, (bytes) => {
+    transport.acceptStdout(bytes);
+}, transport);
+const consumeCodexStderr = (process_, transport) => consumeCodexOutput(process_.standardError, (bytes) => {
+    transport.acceptStderr(bytes);
+}, transport);
+const superviseCodexOutput = (process_, transport) => Effect.all([
+    consumeCodexStdout(process_, transport),
+    consumeCodexStderr(process_, transport),
+], { concurrency: 2, discard: true }).pipe(Effect.matchEffect({
+    onFailure: (error) => acceptRuntimeFailure(transport, error),
+    onSuccess: () => Effect.sync(() => {
+        transport.acceptRuntimeExit();
+    }),
+}));
+const superviseCodexExit = (process_, transport) => process_.exit.pipe(Effect.matchEffect({
+    onFailure: (error) => acceptRuntimeFailure(transport, error),
+    onSuccess: (exit) => exit.state === "failed" && exit.hostFailure !== undefined
+        ? acceptRuntimeFailure(transport, makeNodeRuntimeError("transport_failed", `The Codex process host reported ${exit.hostFailure.phase}/${exit.hostFailure.code}: ${exit.hostFailure.diagnostic}.`))
+        : Effect.sync(() => {
+            transport.acceptRuntimeExit();
+        }),
+}));
+const makeCodexTransport = Effect.fn("CodexTransport.make")(function* (process_) {
+    const input = process_.standardInput;
+    if (input === undefined)
+        return yield* missingCodexInput();
+    const pendingInput = yield* Queue.bounded(maximumCodexInputRecords);
+    const state = { pendingBytes: 0, released: false };
+    const transport = new CodexJsonLineTransport((bytes) => offerCodexInput(pendingInput, state, bytes));
+    yield* Effect.forkScoped(runCodexInputWriter({ input, pendingInput, state, transport }));
+    yield* Effect.forkScoped(superviseCodexOutput(process_, transport));
+    yield* Effect.forkScoped(superviseCodexExit(process_, transport));
+    yield* Effect.addFinalizer(() => Effect.sync(() => {
+        state.released = true;
+        transport.release();
+    }).pipe(Effect.andThen(Queue.shutdown(pendingInput))));
+    return transport;
+});
+const codexLaunchRequest = (command, owner) => ({
+    arguments_: [
+        ...command.prefixArguments,
+        "app-server",
+        "--stdio",
+        "--disable",
+        "plugins",
+        "--disable",
+        "apps",
+    ],
+    environment: codexProbeEnvironment(),
+    executable: command.executable,
+    limits: codexResourceLimits,
+    owner,
+    standardInput: "pipe",
+    streamLimits: codexStreamLimits,
+    workingDirectory: command.workingDirectory,
+});
+export const acquireCodexAppServerProcess = Effect.fn("CodexTransport.acquireAppServer")(function* (manager, command, owner) {
+    const process_ = yield* manager.spawn(codexLaunchRequest(command, owner));
+    if (process_.control !== "process") {
+        return yield* Effect.fail(makeNodeRuntimeError("transport_failed", "The Codex process received an invalid Terminal control authority."));
+    }
+    const transport = yield* makeCodexTransport(process_);
+    const owned = {
+        handleCleanupFailure: makeCleanupFailureHandler(command),
+        process: process_,
+        transport,
+        writeCleanupBarrier: makeTestCleanupBarrier(command, process_.treeId),
+    };
+    yield* Effect.addFinalizer(() => releaseOwned(owned).pipe(Effect.orDie));
+    return owned;
+});
+const withOwnedProcess = (input, use) => Effect.scoped(acquireCodexAppServerProcess(input.manager, input.command, input.owner).pipe(Effect.flatMap(use)));
+const decodeProtocolResponse = (value) => {
+    if (value.id === 1) {
+        requireCodexAccountReadResponse(value);
+        return { kind: "accountRead" };
+    }
+    if (value.id !== 0)
+        return;
+    const result = parseCodexInitializeResponse(value);
+    const executableVersion = parseCodexExecutableVersion(result);
+    if (executableVersion === undefined) {
+        throw new CodexProbeFailure("protocol_invalid");
+    }
+    return {
+        kind: "initialize",
+        value: { ...result, executableVersion },
+    };
+};
+const parseProtocolValue = (value) => {
+    if (typeof value !== "object" || value === null || !("id" in value)) {
+        return;
+    }
+    try {
+        return decodeProtocolResponse(value);
+    }
+    catch (error) {
+        if (error instanceof CodexProbeFailure)
+            throw error;
+        if (SchemaError.isSchemaError(error)) {
+            throw new CodexProbeFailure("protocol_invalid");
+        }
+        throw error;
+    }
+};
+class HandshakeObserver {
+    observer;
+    maiaVersion;
+    transport;
+    #initialize;
+    constructor(observer, maiaVersion, transport) {
+        this.observer = observer;
+        this.maiaVersion = maiaVersion;
+        this.transport = transport;
+    }
+    start() {
+        this.transport.write(codexInitializeRequest(this.maiaVersion));
+    }
+    onRecord(value) {
+        const response = parseProtocolValue(value);
+        if (response?.kind === "initialize") {
+            if (this.#initialize !== undefined) {
+                throw new CodexProbeFailure("protocol_invalid");
+            }
+            this.#initialize = response.value;
+            codexPostInitializeRecords.forEach((record) => {
+                this.transport.write(record);
+            });
+        }
+        else if (response?.kind === "accountRead") {
+            if (this.#initialize === undefined) {
+                throw new CodexProbeFailure("protocol_invalid");
+            }
+            this.observer.finish(Effect.succeed(this.#initialize));
+        }
+        return "consume";
+    }
+}
+export const cleanupCodexOwnedProcess = (owned) => releaseOwned(owned).pipe(Effect.orDie);
+const probeFailureReason = (reason) => {
+    if (reason === "executable_not_found")
+        return "executable_not_found";
+    if (reason === "protocol_invalid" || reason === "output_limit_exceeded") {
+        return "protocol_invalid";
+    }
+    return "handshake_failed";
+};
+const registerHandshakeRoute = (owned, observer, handshake) => owned.transport.register({
+    maximumBytes: codexProbePolicy.maximumOutputBytes,
+    maximumChunks: codexProbePolicy.maximumOutputChunks,
+    maximumRecords: codexProbePolicy.maximumOutputRecords,
+    onFailure: (failure) => {
+        observer.finish(failure.pipe(Effect.mapError((transportFailure) => new CodexProbeFailure(probeFailureReason(transportFailure.reason)))));
+    },
+    onRecord: (value) => handshake.onRecord(value),
+    outputFailureReason: "protocol_invalid",
+});
+const observeHandshake = (owned, maiaVersion) => Effect.callback((resume) => {
+    let releaseRoute = noOperation;
+    const observer = new ProcessObserver(resume, () => {
+        releaseRoute();
+    });
+    const handshake = new HandshakeObserver(observer, maiaVersion, owned.transport);
+    const registration = registerHandshakeRoute(owned, observer, handshake);
+    releaseRoute = registration.release;
+    if (observer.settled)
+        releaseRoute();
+    handshake.start();
+    return Effect.sync(releaseRoute).pipe(Effect.andThen(cleanupCodexOwnedProcess(owned)));
+});
+export const runCodexHandshake = (manager, command, input) => withOwnedProcess({ command, manager, owner: input.owner }, (owned) => observeHandshake(owned, input.maiaVersion).pipe(Effect.timeoutOrElse({
+    duration: codexProbePolicy.handshakeDeadline,
+    orElse: () => Effect.fail(new CodexProbeFailure("handshake_failed")),
+})));
+//# sourceMappingURL=transport.js.map