humanish 0.104.3 → 0.106.0

package/dist/actors/computer-use/loop/backstop.js

@@ -0,0 +1,202 @@
+import { actionFingerprint, isIdleTurn } from "./actions.js";
+// The friction backstop. Abandonment is persona-judged first; this force-ends a session only on
+// unambiguous pathology: an idle streak (turns that take no material action) or a no-progress
+// streak (turns that neither change the UI state nor try something new). It is a pure fold over
+// turns, never a turn budget.
+const IDLE_PROGRESS_FORGIVENESS_STEPS = 2;
+/** How many recent turns the repetition check looks back over (#383). */
+const ACTION_REPEAT_WINDOW = 3;
+/** Pixels of vertical scroll per progress bucket (#393): a real scroll step (typically >=100px)
+ *  crosses a bucket and counts as progress; sub-bucket jiggle does not, so an actor nudging the
+ *  same dead panel cannot stay "progressing" forever. */
+const SCROLL_PROGRESS_BUCKET_PX = 200;
+// Caps for stableProgressKey. The progress key is a coarse turn-over-turn comparison input, not a
+// faithful serialization, so it bounds depth, breadth, string length and total output, and a huge
+// or deeply nested appState cannot blow up the comparison. The values are generous (real
+// route/turn/modal projections are tiny) but finite.
+const STABLE_KEY_MAX_DEPTH = 6;
+const STABLE_KEY_MAX_KEYS = 64;
+const STABLE_KEY_MAX_ARRAY = 64;
+const STABLE_KEY_MAX_STRING = 256;
+const STABLE_KEY_MAX_TOTAL = 8192;
+/**
+ * A deterministic, bounded, sorted-key projection of an appState object, used as the friction
+ * loop's progress key. Two structurally-equal states (regardless of key insertion order) map
+ * to the SAME string, so key reordering can never fabricate a progress delta; two different
+ * states map to different strings (within the caps).
+ *
+ * Correctness-load-bearing: it MUST NOT throw on a cyclic or huge input. Cycles are detected
+ * with a seen-set (a back-edge degrades to the marker "[Circular]"); depth, key count, array
+ * length, string length, and total output length are all capped so an adversarial or merely
+ * large appState degrades to a bounded value rather than crashing the loop. Pure: it never
+ * mutates the input. (See docs/architecture/state-driven-executor.md.)
+ */
+export function stableProgressKey(appState) {
+    const seen = new Set();
+    let truncated = false;
+    const encode = (value, depth) => {
+        if (truncated)
+            return '"…"';
+        if (value === null)
+            return "null";
+        const type = typeof value;
+        if (type === "number")
+            return Number.isFinite(value) ? JSON.stringify(value) : `"${String(value)}"`;
+        if (type === "boolean")
+            return value ? "true" : "false";
+        if (type === "bigint")
+            return `"${value.toString()}"`;
+        if (type === "string") {
+            const s = value;
+            return JSON.stringify(s.length > STABLE_KEY_MAX_STRING ? `${s.slice(0, STABLE_KEY_MAX_STRING)}…` : s);
+        }
+        if (type === "function" || type === "symbol" || type === "undefined")
+            return `"[${type}]"`;
+        // object or array
+        if (depth >= STABLE_KEY_MAX_DEPTH)
+            return '"[MaxDepth]"';
+        if (seen.has(value))
+            return '"[Circular]"';
+        seen.add(value);
+        try {
+            if (Array.isArray(value)) {
+                const cap = Math.min(value.length, STABLE_KEY_MAX_ARRAY);
+                const parts = [];
+                for (let i = 0; i < cap; i += 1) {
+                    parts.push(encode(value[i], depth + 1));
+                    if (truncated)
+                        break;
+                }
+                if (value.length > STABLE_KEY_MAX_ARRAY)
+                    parts.push('"…"');
+                return `[${parts.join(",")}]`;
+            }
+            const record = value;
+            const keys = Object.keys(record).sort();
+            const cap = Math.min(keys.length, STABLE_KEY_MAX_KEYS);
+            const parts = [];
+            for (let i = 0; i < cap; i += 1) {
+                const key = keys[i];
+                parts.push(`${JSON.stringify(key)}:${encode(record[key], depth + 1)}`);
+                if (truncated)
+                    break;
+            }
+            if (keys.length > STABLE_KEY_MAX_KEYS)
+                parts.push('"…":"…"');
+            return `{${parts.join(",")}}`;
+        }
+        finally {
+            // Leave the set so sibling subtrees that legitimately repeat a shared reference still
+            // serialize once per occurrence-path without false "Circular" hits across siblings.
+            seen.delete(value);
+        }
+    };
+    let out = encode(appState, 0);
+    if (out.length > STABLE_KEY_MAX_TOTAL) {
+        truncated = true;
+        out = `${out.slice(0, STABLE_KEY_MAX_TOTAL)}…`;
+    }
+    return out;
+}
+/**
+ * The friction progress key: a stable projection of appState when present, else stateSignature.
+ * A state executor with a constant signature still registers progress when its appState changes,
+ * and a vision executor behaves exactly as before.
+ */
+function progressKeyOf(observation) {
+    const base = observation.appState !== undefined
+        ? stableProgressKey(observation.appState)
+        : observation.stateSignature;
+    // Scroll position is state (#393): a scroll-pinned section keeps the frame hash constant while
+    // the participant genuinely advances, so the offset rides the key — bucketed, never raw.
+    return observation.scrollY === undefined
+        ? base
+        : `${base}#s${Math.round(observation.scrollY / SCROLL_PROGRESS_BUCKET_PX)}`;
+}
+export function startBackstop(observation) {
+    return {
+        progressKey: progressKeyOf(observation),
+        recentFingerprints: [],
+        consecutiveIdle: 0,
+        consecutiveNoProgress: 0,
+        idleProgressForgivenessUsed: 0,
+    };
+}
+/** Fold one acted turn and the observation after it into the backstop. */
+export function advanceBackstop(previous, turn, limits) {
+    const idle = isIdleTurn(turn.actions);
+    const progressKey = progressKeyOf(turn.observation);
+    const frameChanged = progressKey !== previous.progressKey;
+    // CORROBORATION (#383). A stale frame alone is NOT evidence of a stuck agent. The frame hash is
+    // a coarse whole-screen measure, and on a light-themed web app it can miss a renamed row, a new
+    // list item, or an opened panel — a measured run had 9 visibly different consecutive frames hash
+    // identically while the agent was a foreign key away from finishing. Ending a lane on that
+    // signal alone recorded working sessions as `gave_up`, capping every browser run at roughly
+    // noProgressSteps turns and writing harness artifacts into evidence as actor behavior.
+    //
+    // So a no-progress turn now requires BOTH a stale frame AND the agent repeating something it
+    // just tried. An agent doing varied work is never counted stuck, however blind the hash is;
+    // an agent re-clicking the same dead control trips it as fast as it did before — arguably
+    // faster, since that is the actual signature of being stuck.
+    const fingerprint = actionFingerprint(turn.actions);
+    const repeatingRecentAction = fingerprint.length > 0 && previous.recentFingerprints.includes(fingerprint);
+    const recentFingerprints = [...previous.recentFingerprints, fingerprint].slice(-ACTION_REPEAT_WINDOW);
+    // Corroboration governs the FRAME-STALENESS backstop only. The idle backstop below is a direct
+    // behavioral signal already (the agent took nothing but screenshots and waits), so it keeps
+    // reading the frame on its own — a repeated screenshot is exactly what an idle streak IS, and
+    // feeding repetition into it would grant an extra forgiveness step for being idle.
+    const observedProgress = frameChanged || turn.heardNewSpeech;
+    const progressed = observedProgress || !repeatingRecentAction;
+    // A screenshot/wait turn while the UI visibly changes may be patience through loading or a
+    // transition, so grant a bounded recovery window. Do not grant infinite immunity: animated
+    // pixels or state-executor turn counters can otherwise keep a screenshot/wait loop alive
+    // until the wall-clock timeout.
+    let { consecutiveIdle, idleProgressForgivenessUsed } = previous;
+    if (!idle) {
+        idleProgressForgivenessUsed = 0;
+        consecutiveIdle = 0;
+    }
+    else if (observedProgress && idleProgressForgivenessUsed < IDLE_PROGRESS_FORGIVENESS_STEPS) {
+        idleProgressForgivenessUsed += 1;
+        consecutiveIdle = 0;
+    }
+    else {
+        consecutiveIdle += 1;
+    }
+    const consecutiveNoProgress = progressed ? 0 : previous.consecutiveNoProgress + 1;
+    // An unchanged screen can be legitimate waiting (for example, a shared-world lobby).
+    // Recovery may suggest another approach, but must not instruct early abandonment while
+    // the task still calls for waiting. The counters, time and spend guards own hard stops.
+    const noProgressRecoverySteps = Math.min(Math.max(1, limits.noProgressSteps - 1), 3);
+    const idleRecoverySteps = Math.min(Math.max(1, limits.idleSteps - 1), 3);
+    const hints = [];
+    if (consecutiveNoProgress >= noProgressRecoverySteps &&
+        consecutiveNoProgress < limits.noProgressSteps) {
+        hints.push(`No visible progress for ${consecutiveNoProgress} step(s). ` +
+            "If your task calls for waiting for another participant or a pending transition, you may continue waiting. " +
+            "Choose whether to continue or stop based on your situation and what you observe.");
+    }
+    if (consecutiveIdle >= idleRecoverySteps && consecutiveIdle < limits.idleSteps) {
+        hints.push(`You are only waiting or taking screenshots for ${consecutiveIdle} step(s). ` +
+            "If your task calls for waiting, you may continue within the remaining session time. " +
+            "When the relevant controls become actionable, continue your task; describe any blocker you actually encounter.");
+    }
+    const tripReason = consecutiveIdle >= limits.idleSteps
+        ? `gave up: ${consecutiveIdle} consecutive turns with no material UI action (only screenshot/wait)`
+        : consecutiveNoProgress >= limits.noProgressSteps
+            ? `gave up: ${consecutiveNoProgress} consecutive turns that repeated a recent action on an unchanged UI state`
+            : undefined;
+    return {
+        backstop: {
+            progressKey,
+            recentFingerprints,
+            consecutiveIdle,
+            consecutiveNoProgress,
+            idleProgressForgivenessUsed,
+        },
+        idle,
+        progressed,
+        hints,
+        tripReason,
+    };
+}

package/dist/actors/computer-use/loop/debrief.d.ts

@@ -0,0 +1,19 @@
+import type { ActorTrace, ParticipantClosingReport } from "../../contract.js";
+import type { DebriefTrigger } from "./ending.js";
+import type { LoopSession } from "./session.js";
+import type { CuaTurnRequest } from "./types.js";
+type Debrief = NonNullable<ActorTrace["debrief"]>;
+/** The conversation a debrief request continues. */
+export interface DebriefContext {
+    readonly previousResponseId: string | undefined;
+    readonly previousExecution: CuaTurnRequest["previousExecution"];
+    readonly acknowledgedSafetyChecks: CuaTurnRequest["acknowledgedSafetyChecks"];
+}
+/** Runtime validation is also required for third-party provider ports. */
+export declare function validClosingReport(value: unknown): value is ParticipantClosingReport;
+/**
+ * Ask the participant for its closing report after a structured stop, or record why the request
+ * was skipped. Either way the result is the trace's debrief, and the watcher gets a final flush.
+ */
+export declare function requestDebrief(session: LoopSession, context: DebriefContext, trigger: DebriefTrigger): Promise<Debrief>;
+export {};

package/dist/actors/computer-use/loop/debrief.js

@@ -0,0 +1,164 @@
+import { isCuaAdmissionLimitError } from "../admission-limit.js";
+import { singleDispatch } from "./provider-call.js";
+import { CuaDeadlineError, raceSessionDeadline, requestScope } from "./race.js";
+import { notice } from "./trace.js";
+import { isCompleteTurnUsage } from "./usage.js";
+const DEBRIEF_REQUEST_CAP_MS = 30_000;
+const DEBRIEF_HINT = "The interactive session has ended. Return a closing account with summary and frictionReports. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task.";
+/** Runtime validation is also required for third-party provider ports. */
+export function validClosingReport(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return false;
+    const report = value;
+    return (Object.keys(report).length === 2 &&
+        typeof report.summary === "string" &&
+        report.summary.trim().length > 0 &&
+        report.summary.length <= 4_000 &&
+        Array.isArray(report.frictionReports) &&
+        report.frictionReports.length <= 8 &&
+        report.frictionReports.every((item) => typeof item === "string" && item.trim().length > 0 && item.length <= 2_000));
+}
+function supportsDebrief(provider) {
+    return provider.debrief !== undefined;
+}
+/**
+ * Ask the participant for its closing report after a structured stop, or record why the request
+ * was skipped. Either way the result is the trace's debrief, and the watcher gets a final flush.
+ */
+export async function requestDebrief(session, context, trigger) {
+    const record = (status, detail, usageReported) => {
+        const debrief = {
+            trigger: trigger.kind,
+            status,
+            reason: session.redactNarration(detail),
+            ...(usageReported === undefined ? {} : { usageReported }),
+        };
+        session.trace.record("notice", () => notice(status === "completed" ? "ok" : "warn", `participant debrief ${status}`, session.redactNarration(detail)));
+        return debrief;
+    };
+    const plan = planDebrief(session, trigger);
+    const debrief = "skip" in plan
+        ? record("skipped", plan.skip)
+        : await debriefExchange(session, plan.provider, context, trigger, plan.boundMs, record);
+    session.flush();
+    return debrief;
+}
+function planDebrief(session, trigger) {
+    const { provider, signal, usage } = session;
+    const { maxUsd, overRunBudget, estimateTurnCostUsd } = session.settings;
+    if (session.trace.counts.turns === 0)
+        return { skip: "the study stopped before any participant turn" };
+    if (!supportsDebrief(provider))
+        return { skip: "this provider does not support read-only closing reports" };
+    if (usage.cleanupUnconfirmed)
+        return { skip: "participant request cleanup is unconfirmed" };
+    if (signal?.aborted)
+        return { skip: "the study was cancelled" };
+    if (session.remaining() <= 0)
+        return { skip: "the session deadline was reached" };
+    if (provider.requiresFrame && trigger.observation.screenshot === undefined)
+        return { skip: "the final observation has no required frame" };
+    if ((maxUsd !== undefined || overRunBudget !== undefined) && usage.unavailableForCap()) {
+        return {
+            skip: "remaining model budget is unknown because an earlier participant turn did not report complete usage",
+        };
+    }
+    if (maxUsd !== undefined) {
+        const estimate = usage.sawUsage || usage.knownPending() !== undefined
+            ? estimateTurnCostUsd?.(usage.forCap())
+            : undefined;
+        if (estimate === undefined || estimate === null || !Number.isFinite(estimate))
+            return { skip: "remaining model budget could not be established" };
+        if (estimate >= maxUsd)
+            return { skip: "the estimated model budget was reached" };
+    }
+    if (overRunBudget?.(usage.forCap()) != null)
+        return { skip: "the study model budget was reached" };
+    const boundMs = Math.max(0, Math.min(DEBRIEF_REQUEST_CAP_MS, session.turnTimeoutMs, session.remaining()));
+    if (boundMs <= 0 || signal?.aborted) {
+        return {
+            skip: signal?.aborted ? "the study was cancelled" : "the session deadline was reached",
+        };
+    }
+    return { provider, boundMs };
+}
+async function debriefExchange(session, provider, context, trigger, boundMs, record) {
+    const { signal } = session;
+    const scope = requestScope(signal);
+    const timer = setTimeout(() => scope.end(), boundMs);
+    timer.unref?.();
+    session.trace.bump("debriefCalls");
+    try {
+        const request = {
+            instructions: session.settings.instructions,
+            observation: trigger.observation,
+            ...(provider.requestPolicy !== "fail_closed" || context.previousExecution === undefined
+                ? {}
+                : { previousExecution: context.previousExecution }),
+            ...(context.previousResponseId === undefined
+                ? {}
+                : { previousResponseId: context.previousResponseId }),
+            ...(context.acknowledgedSafetyChecks === undefined
+                ? {}
+                : { acknowledgedSafetyChecks: context.acknowledgedSafetyChecks }),
+            contextHint: DEBRIEF_HINT,
+        };
+        const turn = provider.requestPolicy === "fail_closed"
+            ? await singleDispatch(session, "debrief", (dispatchSignal) => provider.debrief(request, dispatchSignal), boundMs)
+            : await raceSessionDeadline(provider.debrief(request, scope.signal), boundMs, signal);
+        return acceptDebriefTurn(session, turn, record);
+    }
+    catch (error) {
+        // A failed optional report cannot rewrite the already observed structured completion.
+        if (isCuaAdmissionLimitError(error)) {
+            return record("skipped", "the adapter reported a local admission limit before provider dispatch; no closing request was sent");
+        }
+        const detail = signal?.aborted
+            ? "cancelled"
+            : scope.signal.aborted || error instanceof CuaDeadlineError
+                ? "closing report deadline reached"
+                : error instanceof Error
+                    ? error.message
+                    : String(error);
+        const debriefReceipt = provider.requestPolicy === "fail_closed" ? session.usage.requests.at(-1) : undefined;
+        const usageReported = debriefReceipt?.usageComplete === true && isCompleteTurnUsage(debriefReceipt.usage);
+        return record("failed", `${detail}; closing request usage is ${usageReported ? "reported" : "unreported"}`, usageReported);
+    }
+    finally {
+        clearTimeout(timer);
+        scope.end();
+    }
+}
+function acceptDebriefTurn(session, turn, record) {
+    const { maxUsd, overRunBudget, estimateTurnCostUsd } = session.settings;
+    session.usage.record(turn, "debrief");
+    session.lastResponseId = turn.responseId ?? session.lastResponseId;
+    // Refresh a shared budget with all reported usage, without changing the completed task.
+    const sharedStop = overRunBudget?.(session.usage.forCap());
+    const finalEstimate = maxUsd === undefined ? undefined : estimateTurnCostUsd?.(session.usage.forCap());
+    if (sharedStop != null ||
+        (maxUsd !== undefined && finalEstimate != null && finalEstimate > maxUsd)) {
+        session.trace.record("notice", () => notice("warn", "model budget reached during closing report", "The closing request crossed an estimated budget; no further requests or actions followed. Task completion is unchanged."));
+    }
+    const usageReported = isCompleteTurnUsage(turn.usage) && turn.providerRequest?.usageComplete !== false;
+    if (turn.actions.length > 0 || turn.pendingSafetyChecks.length > 0) {
+        return record("failed", "the closing response requested actions or safety checks; none were executed and its report was not accepted", usageReported);
+    }
+    if (!validClosingReport(turn.closingReport)) {
+        return record("failed", "the closing response did not contain a valid structured participant report", usageReported);
+    }
+    const report = {
+        summary: session.redactNarration(turn.closingReport.summary.trim()),
+        frictionReports: [
+            ...new Set(turn.closingReport.frictionReports.map((text) => session.redactNarration(text.trim()))),
+        ],
+    };
+    const messageId = session.trace.record("message", () => ({
+        lifecycle: "completed",
+        title: "participant closing report",
+        text: [report.summary, ...report.frictionReports].join("\n\n"),
+    }));
+    session.trace.bump("messages");
+    const debrief = record("completed", "one read-only report; no additional desktop actions; original stop and task outcomes preserved", usageReported);
+    return { ...debrief, report, messageId };
+}

package/dist/actors/computer-use/loop/ending.d.ts

@@ -0,0 +1,86 @@
+import type { ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../../contract.js";
+import type { StopConditionMatch } from "../../stop-conditions.js";
+import type { LoopSession } from "./session.js";
+import { type Evidence } from "./trace.js";
+import type { CuaObservation, CuaProvider, CuaTurn } from "./types.js";
+/** A harness-observed completion after which the participant is asked for a debrief. */
+export interface DebriefTrigger {
+    readonly kind: "stop_when" | "dwell";
+    /** The final observation, already captured; the debrief request sends it without observing. */
+    readonly observation: CuaObservation;
+}
+export interface Stop {
+    readonly completionReason: ActorCompletionReason;
+    readonly reason: string;
+    readonly stopCause?: ActorStopCause;
+    /** Trace evidence for this stop, recorded when the stop is concluded. */
+    readonly evidence?: Evidence;
+    readonly debriefTrigger?: DebriefTrigger;
+}
+/**
+ * Read the fixed closing line the prompt asks for (#570): the FIRST non-empty line of the
+ * participant's last message, exactly one of three phrases, punctuation and case forgiven. Anything
+ * else is absence, never a guess. Exported for tests.
+ */
+export declare function declaredOutcomeFromClosingLine(message: string | undefined): ParticipantDeclaredOutcome | undefined;
+/** The participant's own word for how it ended: a schema field first, then the closing line (#570). */
+export declare function declaredOutcomeOf(turn: CuaTurn): ParticipantDeclaredOutcome | undefined;
+/**
+ * The participant reported a natural endpoint. "not_reached" is a participant who stopped without
+ * finishing: gave_up, which tallies as abandoned. "blocked" keeps goal_satisfied here (the actor
+ * did stop on purpose) and the lane's credibility read turns it into a blocked participant, the
+ * same path a narrated blocker takes.
+ */
+export declare function participantEnded(turn: CuaTurn, declaredOutcome: ParticipantDeclaredOutcome | undefined, redactNarration: (text: string) => string): Stop;
+export declare const harnessAborted: Stop;
+/**
+ * The wall-clock deadline. A session that took at least one material (non-idle) action before the
+ * cap reached its BUDGET rather than stalling; a deadline hit with zero material actions is still
+ * an honest failure (timed_out).
+ */
+export declare function timeLimit(session: LoopSession): Stop;
+/**
+ * A harness spend cap interrupts the session regardless of prior activity. budget_reached maps to
+ * incomplete; zero executed actions do not establish that the participant gave up. Cite the
+ * running estimate, cap, and progress so the operator can inspect the interruption.
+ */
+export declare function spendLimit(session: LoopSession, estimate: number, maxUsd: number): Stop;
+export declare const nonFiniteEstimate: Stop;
+/** A study-level stop is a recruiting decision hitting its limit, not this participant's runaway. */
+export declare function studySpendLimit(reason: string): Stop;
+export declare const accountBilledCaps: Stop;
+export declare const adapterLimit: Stop;
+export declare const usageUnreported: Stop;
+/**
+ * A capped session lost a request without a reply and cannot bound what it cost: no output-token
+ * limit is set, so its worst case is the model's whole output allowance.
+ */
+export declare function lostRequestUnbounded(turnNumber: number, lost: LostRequest): Stop;
+/** How a request was lost without a reply. */
+export type LostRequest = "stalled" | "failed in transit";
+export declare function providerStalledTwice(turnNumber: number, afterMs: number): Stop;
+/**
+ * A provider can exhaust its response budget before producing visible text, or midway through an
+ * action. Never interpret either as participant completion or dispatch actions from an explicitly
+ * incomplete response.
+ */
+export declare function providerInterrupted(interruption: NonNullable<CuaTurn["interruption"]>): Stop;
+/**
+ * Safety-check categories are provider-defined enums (e.g. "malicious_instructions"), not free
+ * text; the evidence records them (redacted for defense-in-depth) so it shows WHY the run paused.
+ */
+export declare function blockedOnSafetyChecks(checks: string): Stop;
+/**
+ * A vision provider against a screenshot-less observation is a fail-closed harness error, not a
+ * silent crash. The provider sets requiresFrame; a state-reasoning provider omits it.
+ */
+export declare function missingFrame(provider: CuaProvider, observation: CuaObservation): Stop | undefined;
+export declare function stopWhenMatched(match: StopConditionMatch, observation: CuaObservation, redactNarration: (text: string) => string): Stop;
+export declare function dwellCompleted(heldMs: number, when: string, observation: CuaObservation): Stop;
+/** The friction backstop tripped: cite the reason, the last material action and recent actions. */
+export declare function gaveUp(session: LoopSession, reason: string): Stop;
+/**
+ * Classify an error that ended the session, with the diagnostics a reader needs to place it.
+ * A stop cause committed by an earlier stop that failed to record is kept.
+ */
+export declare function stopForError(session: LoopSession, error: unknown): Stop;