humanish 0.105.0 → 0.106.0

package/dist/actors/codex/restricted-participant.js

@@ -0,0 +1,432 @@
+import { isRecordedCodexCliVersion, } from "../contract.js";
+import { defaultCodexCliVersion } from "./qualified-versions.js";
+import { CuaProviderError, isCuaProviderError, } from "../computer-use/provider-error.js";
+import { validateBrowserControlPng, validateHeardSpeech } from "../../browser-control/protocol.js";
+import { createRestrictedCodexSession, } from "./restricted-session.js";
+import { PARTICIPANT_PROFILE, PARTICIPANT_LIMITS as L, PARTICIPANT_FINAL_SCHEMA, participantToolSchema, parseParticipantTool, parseParticipantFinal, } from "./restricted-participant-policy.js";
+const codeOf = (code) => {
+    if (code === "cancelled" || code === "timeout" || code === "refusal")
+        return code === "refusal" ? "refused" : code;
+    if (code === "codex_cleanup_failed")
+        return "cleanup_unconfirmed";
+    if (code === "codex_busy")
+        return "busy";
+    if (code === "codex_protocol_error" || code === "codex_tool_call")
+        return "protocol_error";
+    if (code === "invalid_response" || code === "response_too_large" || code === "output_incomplete")
+        return "invalid_response";
+    if (code === "invalid_request")
+        return "request_rejected";
+    if (code === "codex_process_failed")
+        return "process_failed";
+    return "unavailable";
+};
+const noDispatch = () => ({
+    dispatched: false,
+    usageComplete: false,
+    cleanup: "confirmed",
+});
+function deferred() {
+    let resolve, reject;
+    const promise = new Promise((yes, no) => {
+        resolve = yes;
+        reject = no;
+    });
+    // A native request can settle during executor work or shutdown, between consumers.
+    void promise.catch(() => undefined);
+    return { promise, resolve, reject };
+}
+const toolDescription = (speechEnabled) => `Act on the participant's browser through humanish. Submit one to four UI actions${speechEnabled ? ", including speak when you need to reply aloud," : ""} and a short public comment; never private reasoning. humanish returns a JSON STRING with execution acknowledgments${speechEnabled ? ", speech heard from the actual participant speaker sink," : ""} and a fresh screenshot. In Code Mode use: const r = JSON.parse(await tools.humanish_ui({narration: "...", actions: [...]})); text({acknowledgments: r.acknowledgments${speechEnabled ? ", heardSpeech: r.heardSpeech" : ""}, contextHint: r.contextHint, closing: r.closing}); image(r.imageUrl). Call serially and inspect each returned screenshot${speechEnabled ? " and heardSpeech array" : ""} before deciding what to do next.${speechEnabled ? " Use speak only after the visible UI shows that you joined the call and your microphone is unmuted; it sends audio into that call." : ""} Acknowledged input does not prove an application outcome. If closing is true, stop calling tools and give your final account.`;
+const participantInstructions = (study, speechEnabled) => `${study}\n\nYou are the study participant throughout this conversation, including its closing account. Use only the supplied screenshots and humanish_ui tool to interact. The tool returns a JSON string: parse it, inspect acknowledgments${speechEnabled ? " and heardSpeech captured from the actual participant speaker sink" : ""}, and display imageUrl with Code Mode image(). Do not print the image data URL as text. Keep your persona and earlier observations throughout the session.${speechEnabled ? " A speak action plays into the participant microphone; use it only after the visible UI shows that you joined the call and the microphone is unmuted." : ""} Speak publicly about your experience, never reveal private reasoning. Completed inputs do not prove application outcomes; verify on the next screenshot. When the task ends, return only the required final JSON with outcome, summary and frictionReports. Report observed confusion and recovered mistakes as well as blockers. Do not invent observations.`;
+/** Turns and failures from the native run, handed in order to the next provider request. */
+class ParticipantEvents {
+    queued = [];
+    waiter;
+    get waiting() {
+        return this.queued.length > 0;
+    }
+    emit(event) {
+        if (this.waiter) {
+            const current = this.waiter;
+            this.waiter = undefined;
+            current.resolve(event);
+        }
+        else
+            this.queued.push(event);
+    }
+    async next() {
+        const event = this.queued.shift() ?? (await (this.waiter ??= deferred()).promise);
+        if ("error" in event)
+            throw event.error;
+        return event.turn;
+    }
+}
+const ACKNOWLEDGMENT_STATUSES = ["completed", "skipped", "not_dispatched", "outcome_uncertain"];
+/**
+ * Request validation: true when the request may not enter the conversation. Each case is refused
+ * as request_rejected before anything is dispatched.
+ */
+function rejectsParticipantRequest(req, conversation) {
+    if (typeof req.instructions !== "string" ||
+        Buffer.byteLength(req.instructions) > L.instructions ||
+        (req.contextHint !== undefined &&
+            (typeof req.contextHint !== "string" || Buffer.byteLength(req.contextHint) > L.hint)) ||
+        req.previousResponseId !== undefined ||
+        (req.acknowledgedSafetyChecks?.length ?? 0) > 0 ||
+        (conversation.instructions !== undefined && conversation.instructions !== req.instructions))
+        return true;
+    const frame = req.observation.screenshot;
+    try {
+        if (!Buffer.isBuffer(frame))
+            throw new Error();
+        validateBrowserControlPng(frame);
+    }
+    catch {
+        return true;
+    }
+    if (req.observation.heardSpeech !== undefined) {
+        if (!conversation.speechEnabled)
+            return true;
+        try {
+            validateHeardSpeech(req.observation.heardSpeech);
+        }
+        catch {
+            return true;
+        }
+    }
+    const acknowledgments = req.previousExecution?.actions;
+    return ((conversation.awaitingAcknowledgments && acknowledgments === undefined) ||
+        (acknowledgments !== undefined &&
+            (conversation.lastActionCount === undefined ||
+                !Array.isArray(acknowledgments) ||
+                acknowledgments.length !== conversation.lastActionCount ||
+                acknowledgments.some((a, i) => !a || a.index !== i || !ACKNOWLEDGMENT_STATUSES.includes(a.status)))));
+}
+/** Launch: the one native run a batch of tool calls and its final account share. */
+function participantRunRequest(req, run) {
+    return {
+        ...(run.model === undefined ? {} : { model: run.model }),
+        instructions: participantInstructions(run.instructions, run.speechEnabled),
+        evidence: JSON.stringify({
+            phase: run.closing ? "closing" : "interaction",
+            contextHint: req.contextHint ?? null,
+            instruction: run.closing
+                ? "Interaction has ended. Do not call tools; give your closing account."
+                : "Use humanish_ui to act. Inspect each result before choosing the next batch. Finish when appropriate.",
+            width: req.observation.screenshot.readUInt32BE(16),
+            height: req.observation.screenshot.readUInt32BE(20),
+            previousExecution: req.previousExecution ?? null,
+            ...(run.speechEnabled ? { heardSpeech: req.observation.heardSpeech ?? [] } : {}),
+        }),
+        images: [{ evidenceId: "current-frame", dataUrl: run.imageUrl }],
+        schema: PARTICIPANT_FINAL_SCHEMA,
+        maxOutputTokens: null,
+        timeoutMs: run.timeoutMs,
+        signal: run.signal,
+    };
+}
+/** Turn dispatch: the tool result that answers a waiting native tool call. */
+function toolReply(req, imageUrl, speechEnabled, closing) {
+    return JSON.stringify({
+        acknowledgments: req.previousExecution.actions.map(({ index, status }) => ({ index, status })),
+        imageUrl,
+        ...(speechEnabled ? { heardSpeech: req.observation.heardSpeech ?? [] } : {}),
+        contextHint: req.contextHint ?? null,
+        closing,
+    });
+}
+/** Receipt: what a native run's result records, whatever its outcome. */
+const runReceipt = (result) => ({
+    dispatched: result.dispatched,
+    usageComplete: result.usageComplete,
+    cleanup: result.errorCode === "codex_cleanup_failed" ? "unconfirmed" : "confirmed",
+});
+/**
+ * The final turn a native run's result yields, or the error it fails with. A run that ends after
+ * the participant stopped fails as cancelled, or cleanup_unconfirmed once cleanup failed.
+ */
+function finalTurn(result, receipt, usage, stopped) {
+    if (stopped.stopped)
+        throw new CuaProviderError(stopped.failedCleanup ? "cleanup_unconfirmed" : "cancelled", receipt, usage, result.failurePhase);
+    if (result.status !== "completed" || result.errorCode !== null)
+        throw new CuaProviderError(codeOf(result.errorCode), receipt, usage, result.failurePhase);
+    let turn;
+    try {
+        turn = parseParticipantFinal(result.output);
+    }
+    catch {
+        throw new CuaProviderError("invalid_response", receipt, usage, "response");
+    }
+    return {
+        ...turn,
+        ...(usage === undefined ? {} : { usage: { ...usage, turns: result.inferenceUsage ?? [] } }),
+        providerRequest: receipt,
+    };
+}
+/** Drain: resolves false when the work outlasts the cleanup budget counted from the abort. */
+async function withinCleanupBudget(work, abortAt) {
+    let timer;
+    try {
+        return await Promise.race([
+            work,
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(false), Math.max(0, L.cleanupMs - (performance.now() - abortAt)));
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/** The session whose one tool, humanish_ui, hands each native tool call to `call`. */
+function participantSession(options, speechEnabled, effort, call) {
+    return createRestrictedCodexSession({
+        ...options.session,
+        participant: {
+            ...(options.authMode === "operator" ? { authMode: "operator" } : {}),
+            reasoningEffort: effort,
+            tool: {
+                name: "humanish_ui",
+                description: toolDescription(speechEnabled),
+                inputSchema: participantToolSchema(speechEnabled),
+                call,
+            },
+        },
+    });
+}
+/**
+ * The profile a run records: the detected release, or the host default before the first launch.
+ * An operator-auth session has one only once it resolved a model on a ChatGPT account.
+ */
+function participantExecutionProfile(session, operator, effort) {
+    const detected = session.cliVersion;
+    const cliVersion = isRecordedCodexCliVersion(detected) ? detected : defaultCodexCliVersion();
+    if (!operator)
+        return { ...PARTICIPANT_PROFILE, cliVersion };
+    return session.authentication === "chatgpt-account" && session.resolvedModel
+        ? {
+            ...PARTICIPANT_PROFILE,
+            cliVersion,
+            requestedModel: session.resolvedModel,
+            reasoningEffort: effort,
+        }
+        : undefined;
+}
+const participantCapabilities = (operator) => ({
+    headless: true,
+    structuredTrace: true,
+    lanes: ["computer-use"],
+    producesScreenshots: true,
+    byoModel: operator,
+    preGrantableApprovals: false,
+    inProcessTools: false,
+    license: "proprietary",
+});
+/** The session's tool callback: one native tool call becomes the turn the loop executes. */
+async function acceptToolCall(settings, state, args) {
+    if (state.closed || state.closingPhase || state.continuation || !state.active)
+        throw new Error("Unexpected participant tool call");
+    const turn = parseParticipantTool(args, settings.speechEnabled);
+    const reply = deferred();
+    state.continuation = reply;
+    state.lastActionCount = turn.actions.length;
+    state.events.emit({ turn });
+    return reply.promise;
+}
+/** Closes the native session once; an unconfirmed close marks cleanup failed. */
+function closeParticipantSession(state) {
+    return (state.sessionClosing ??= Promise.resolve()
+        .then(() => state.session.close())
+        .catch(() => false)
+        .then((confirmed) => {
+        if (!confirmed)
+            state.failedCleanup = true;
+        return confirmed;
+    }));
+}
+/** Stops the conversation: abort the native run, reject a waiting tool call, close the session. */
+function revokeParticipant(state) {
+    state.closed = true;
+    state.abortAt ??= performance.now();
+    state.controller?.abort();
+    state.continuation?.reject(new Error("Participant closed"));
+    state.continuation = undefined;
+    void closeParticipantSession(state);
+}
+/** Starts the one native run; its final turn or failure arrives as a participant event. */
+function launchNativeRun(settings, state, req, imageUrl) {
+    state.active = true;
+    const signal = (state.controller = new AbortController()).signal;
+    state.nativeTask = (async () => {
+        let receipt = noDispatch(), usage;
+        try {
+            const result = await state.session.run(participantRunRequest(req, {
+                instructions: state.instructions,
+                closing: state.closingPhase,
+                speechEnabled: settings.speechEnabled,
+                model: settings.model,
+                imageUrl,
+                timeoutMs: settings.timeoutMs,
+                signal,
+            }));
+            receipt = runReceipt(result);
+            usage = result.usage ?? undefined;
+            if (result.dispatched && !result.usageComplete)
+                state.incompleteUsage = true;
+            if (receipt.cleanup === "unconfirmed")
+                state.failedCleanup = true;
+            const stopped = state.closed || signal.aborted;
+            state.events.emit({
+                turn: finalTurn(result, receipt, usage, { stopped, failedCleanup: state.failedCleanup }),
+            });
+        }
+        catch (error) {
+            revokeParticipant(state);
+            state.events.emit({
+                error: isCuaProviderError(error)
+                    ? error
+                    : new CuaProviderError("process_failed", { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" }, usage),
+            });
+        }
+        finally {
+            state.active = false;
+        }
+    })();
+}
+/** One request: answer the waiting tool call, or start the native run, then yield its next turn. */
+async function runParticipantTurn(settings, state, req, signal, debrief) {
+    if (state.events.waiting)
+        return state.events.next();
+    if (signal.aborted) {
+        revokeParticipant(state);
+        throw new CuaProviderError("cancelled", noDispatch());
+    }
+    const conversation = {
+        instructions: state.instructions,
+        speechEnabled: settings.speechEnabled,
+        awaitingAcknowledgments: state.continuation !== undefined,
+        lastActionCount: state.lastActionCount,
+    };
+    if (rejectsParticipantRequest(req, conversation))
+        throw new CuaProviderError("request_rejected", noDispatch());
+    const onAbort = () => {
+        revokeParticipant(state);
+    };
+    signal.addEventListener("abort", onAbort, { once: true });
+    try {
+        state.instructions ??= req.instructions;
+        state.closingPhase = debrief;
+        const imageUrl = `data:image/png;base64,${req.observation.screenshot.toString("base64")}`;
+        if (state.continuation) {
+            const reply = state.continuation;
+            state.continuation = undefined;
+            reply.resolve(toolReply(req, imageUrl, settings.speechEnabled, debrief));
+        }
+        else if (!state.active)
+            launchNativeRun(settings, state, req, imageUrl);
+        else
+            throw new CuaProviderError("busy", noDispatch());
+        return await state.events.next();
+    }
+    finally {
+        // The shared loop aborts each request's signal after it yields. The native
+        // turn must remain alive while humanish executes and records its actions.
+        signal.removeEventListener("abort", onAbort);
+    }
+}
+/** The CuaProvider the loop drives; `start` admits one request at a time. */
+function participantProvider(settings, state, start) {
+    const { operator, effort } = settings;
+    return {
+        id: "codex-participant",
+        requiresFrame: true,
+        requestPolicy: "fail_closed",
+        get version() {
+            return state.session.resolvedModel ?? settings.model;
+        },
+        get executionProfile() {
+            return participantExecutionProfile(state.session, operator, effort);
+        },
+        modelSettings: { reasoningEffort: effort },
+        capabilities: participantCapabilities(operator),
+        get pendingRequestUsage() {
+            const usage = state.session.pendingUsage;
+            return usage === undefined
+                ? undefined
+                : { ...usage, turns: state.session.pendingInferenceUsage ?? [] };
+        },
+        get interactionUsageIncomplete() {
+            return state.incompleteUsage || state.active;
+        },
+        get historyTurnsOmitted() {
+            return 0;
+        },
+        nextTurn: (req, signal) => start(req, signal, false),
+        debrief: (req, signal) => start(req, signal, true),
+    };
+}
+/** One native tool-calling conversation; the existing CUA loop owns every input. */
+export function createRestrictedCodexParticipant(options = {}) {
+    const timeoutMs = options.requestTimeoutMs ?? L.requestMs;
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > L.requestMs)
+        throw new CuaProviderError("request_rejected", noDispatch());
+    const operator = options.authMode === "operator";
+    const speechEnabled = options.speechEnabled === true;
+    const settings = {
+        options,
+        timeoutMs,
+        operator,
+        speechEnabled,
+        model: options.model ?? (operator ? undefined : PARTICIPANT_PROFILE.requestedModel),
+        effort: options.reasoningEffort ?? "low",
+    };
+    const state = {
+        // Assigned below: the session's tool callback needs this record.
+        session: undefined,
+        events: new ParticipantEvents(),
+        closed: false,
+        failedCleanup: false,
+        incompleteUsage: false,
+        active: false,
+        closingPhase: false,
+        instructions: undefined,
+        lastActionCount: undefined,
+        continuation: undefined,
+        nativeTask: undefined,
+        controller: undefined,
+        abortAt: undefined,
+        sessionClosing: undefined,
+    };
+    state.session = participantSession(options, speechEnabled, settings.effort, (args) => acceptToolCall(settings, state, args));
+    let pending, closing;
+    const start = (req, signal, debrief) => {
+        if (state.closed && !state.events.waiting)
+            return Promise.reject(new CuaProviderError("request_rejected", noDispatch()));
+        if (pending)
+            return Promise.reject(new CuaProviderError("busy", noDispatch()));
+        const task = runParticipantTurn(settings, state, req, signal, debrief);
+        pending = task;
+        void task
+            .finally(() => {
+            if (pending === task)
+                pending = undefined;
+        })
+            .catch(() => undefined);
+        return task;
+    };
+    return {
+        provider: participantProvider(settings, state, start),
+        close: () => {
+            if (closing)
+                return closing;
+            revokeParticipant(state);
+            closing = (async () => {
+                const work = Promise.all([state.nativeTask, closeParticipantSession(state)]).then(([, ok]) => ok);
+                if (!(await withinCleanupBudget(work, state.abortAt)))
+                    state.failedCleanup = true;
+                return { status: state.failedCleanup ? "unconfirmed" : "confirmed" };
+            })();
+            return closing;
+        },
+    };
+}

package/dist/actors/codex/restricted-policy.d.ts

@@ -0,0 +1,70 @@
+import type { ReasoningEffort } from "../reasoning-effort.js";
+/** Versioned, deliberately narrow profile qualified with an account-backed vision turn.
+ * The CLI still advertises code-mode tools: disabling their host is the enforcement
+ * boundary. Feature names alone are not proof that a tool has been removed. The admitted
+ * CLI releases are per host, in qualified-versions.ts. */
+export declare const RESTRICTED_CODEX_ANALYSIS_IDENTITY: {
+    readonly provider: "codex";
+    readonly authMode: "chatgpt";
+    readonly modelProvider: "openai";
+    readonly toolPolicy: "restricted-codex-v1";
+    readonly reasoningEffort: "low";
+};
+export declare const RESTRICTED_CODEX_ANALYSIS_MODELS: readonly ["gpt-6-astra"];
+export type RestrictedCodexAnalysisErrorCode = "invalid_request" | "cancelled" | "timeout" | "response_too_large" | "invalid_response" | "refusal" | "output_incomplete" | "codex_unavailable" | "codex_unsupported_version" | "codex_unsupported_platform" | "codex_login_required" | "codex_unsupported_auth" | "codex_unsafe_configuration" | "codex_model_unavailable" | "codex_protocol_error" | "codex_tool_call" | "codex_process_failed" | "codex_cleanup_failed" | "codex_busy";
+export interface RestrictedCodexRequest {
+    model?: string | undefined;
+    instructions: string;
+    evidence: string;
+    images: {
+        evidenceId: string;
+        dataUrl: string;
+    }[];
+    schema: Record<string, unknown>;
+    maxOutputTokens: number | null;
+    timeoutMs: number;
+    signal?: AbortSignal;
+}
+export interface RestrictedCodexUsage {
+    input: number;
+    output: number;
+    cachedInput?: number;
+    cacheWriteInput?: number;
+}
+export interface RestrictedCodexResult {
+    failurePhase?: import("../computer-use/provider-error.js").CuaProviderFailurePhase;
+    status: "completed" | "incomplete" | "refused" | "failed" | "cancelled" | "timed_out";
+    output: unknown;
+    usage: RestrictedCodexUsage | null;
+    inferenceUsage?: RestrictedCodexUsage[];
+    usageComplete: boolean;
+    dispatched: boolean;
+    errorCode: RestrictedCodexAnalysisErrorCode | null;
+}
+export declare const CODEX_MAX_REQUEST_BYTES: number;
+export declare const CODEX_MAX_OUTPUT_BYTES: number;
+export declare const CODEX_MAX_STDOUT_BYTES: number;
+export declare const CODEX_MAX_EVENTS = 65536;
+export declare const CODEX_IMAGE: RegExp;
+export declare const codexRecord: (value: unknown) => Record<string, unknown>;
+export interface RestrictedCodexConfigMode {
+    participantCodeMode?: boolean;
+    reasoningEffort?: ReasoningEffort;
+    operatorAuth?: boolean;
+}
+export declare function restrictedCodexConfig(model: string | undefined, mode?: RestrictedCodexConfigMode): {
+    toml: string;
+    overrides: Record<string, unknown>;
+};
+/** Inspect before thread/start, which could otherwise start inherited integrations. */
+export declare function admitsRestrictedCodexConfig(raw: unknown, configPath: string, model: string | undefined, mode?: RestrictedCodexConfigMode): boolean;
+export declare function admitsRestrictedCodexThread(raw: unknown, model: string, cwd: string, reasoningEffort: ReasoningEffort, cliVersion: string): boolean;
+export declare function restrictedCodexRequestError(request: RestrictedCodexRequest, operatorParticipant?: boolean): RestrictedCodexAnalysisErrorCode | null;
+export declare function restrictedCodexUsage(raw: unknown): RestrictedCodexUsage | null;
+export declare function restrictedCodexFailure(errorCode: RestrictedCodexAnalysisErrorCode, dispatched?: boolean, usage?: RestrictedCodexUsage | null): RestrictedCodexResult;
+/**
+ * The tool policy each app-server event must pass: raw items of an allowed type only, `exec` as
+ * the only custom tool, `wait` as the only function, and no agent message that delivers
+ * asynchronously or asks the user a question. True means the request stops with codex_tool_call.
+ */
+export declare function toolPolicyViolation(method: string, item: Record<string, unknown>, allowedRawItemTypes: readonly string[]): boolean;