@beeos-ai/device-agent 0.4.0

package/dist/agent-runtime/core.d.ts

@@ -0,0 +1,170 @@
+/**
+ * agent-runtime/core — Plan-Act-Reflect loop.
+ *
+ * Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/loop/core.py
+ *
+ * Each iteration:
+ *   1. **Observe** — pull a fresh screenshot from the MCP server via
+ *      `McpClient`. The uiautomator dump is fetched only when the active
+ *      `IModelAdapter.consumesUiDump === true` (e.g. GUI-Owl); the default
+ *      pure-vision flow (Open-AutoGLM, UI-TARS, Qwen-VL, ShowUI) skips it
+ *      to save ~1–3 s per step on Android.
+ *   2. **Predict** — ask the registered `IModelAdapter` for the next
+ *      `Action`, given the observation and rolling step history.
+ *   3. **Act** — terminal verbs (`done` / `fail` / `call_for_user`) short-
+ *      circuit the loop. Everything else is dispatched to the MCP `/act`
+ *      route. Action payloads carrying `_coordSystem: "0-1000" |
+ *      "smart_resize"` are scaled to absolute screen pixels before
+ *      dispatch.
+ *   4. **Reflect** — heuristic stall detection. A non-null veto string
+ *      ends the run with `terminator: "reflection_veto"`.
+ *
+ * Termination conditions:
+ *   - `done` action → success
+ *   - `fail` action → failure
+ *   - `call_for_user` action → handled per `waitForUser` flag
+ *   - step budget exceeded
+ *   - wall-clock timeout exceeded
+ *   - reflection veto
+ *   - external `AbortSignal` aborted
+ */
+import { type Action, type Observation } from "@beeos-ai/device-common";
+import type { IMcpRuntime } from "../mcp-client/runtime.js";
+import type { IModelAdapter, ModelUsage, Prediction, StepHistory } from "./adapters/base.js";
+import { HierarchicalReflector } from "./reflection.js";
+import type { TrajectoryVerdict } from "./reflectors/trajectory-reflector.js";
+export type AgentLoopTerminator = "done" | "fail" | "call_for_user" | "step_budget" | "timeout" | "reflection_veto" | "cancelled" | "error";
+export interface AgentRunHooks {
+    /** Streaming token delta from the underlying model (best-effort). */
+    onText?: (delta: string) => void | Promise<void>;
+    /** Reasoning channel (separate stream for o1-style models). */
+    onReasoning?: (delta: string) => void | Promise<void>;
+    /** Fired exactly once per step, after the action has been parsed. */
+    onStep?: (step: AgentStepEvent) => void | Promise<void>;
+    /** Fired before the action is dispatched to MCP `/act`. */
+    onTool?: (tool: AgentToolEvent) => void | Promise<void>;
+    /** Fired on every error, fatal or recoverable. */
+    onError?: (err: AgentErrorEvent) => void | Promise<void>;
+}
+export interface AgentStepEvent {
+    /** 1-based step index. */
+    index: number;
+    rationale: string;
+    action: Action;
+    prediction: Prediction;
+    /** `true` when the action terminated the loop. */
+    finished: boolean;
+}
+export interface AgentToolEvent {
+    /** Action verb ≡ MCP tool name. */
+    name: Action["type"];
+    args: Record<string, unknown>;
+    /** Raw response from the MCP `/act` route. */
+    result: unknown;
+    durationMs: number;
+}
+export interface AgentErrorEvent {
+    code: AgentErrorCode;
+    message: string;
+    fatal: boolean;
+    cause?: unknown;
+}
+export type AgentErrorCode = "MODEL_PREDICT_FAILED" | "MCP_ACT_FAILED" | "MCP_OBSERVE_FAILED" | "STEP_BUDGET" | "TIMEOUT" | "REFLECTION_VETO" | "MODEL_FAIL" | "CALL_FOR_USER_TIMEOUT" | "ABORTED";
+export interface AgentRunOptions {
+    task: string;
+    env: Observation["env"];
+    mcp: IMcpRuntime;
+    adapter: IModelAdapter;
+    /** Per-call max steps. Defaults to 30. */
+    maxSteps?: number;
+    /** Wall-clock timeout in milliseconds. Defaults to 5 minutes. */
+    timeoutMs?: number;
+    /**
+     * When the model emits `call_for_user`, should the loop block waiting
+     * for an out-of-band resolution callback? In Phase-1 we surface the
+     * event and immediately stop (`callForUser` terminator).
+     */
+    waitForUser?: boolean;
+    /** Optional outside cancellation. */
+    signal?: AbortSignal;
+    /** Run hooks for streaming events. */
+    hooks?: AgentRunHooks;
+    /** Optional pre-built reflector (test injection). */
+    reflector?: HierarchicalReflector;
+}
+export interface AgentRunResult {
+    finalText: string;
+    steps: number;
+    cancelled: boolean;
+    terminator: AgentLoopTerminator;
+    history: StepHistory[];
+    usage: ModelUsage;
+    /** Last error (when `terminator === "error"` or non-fatal errors fired). */
+    lastError?: AgentErrorEvent;
+    /**
+     * Phase 2.1 — notes the agent collected mid-task via the `take_note`
+     * action verb. Empty when the underlying adapter never emits notes.
+     * The PlanExecutor surfaces these into `PlanContext.notes` on replan
+     * so the planner can avoid re-asking for already-captured information
+     * (verification codes, computed prices, etc.).
+     */
+    notes: string[];
+    /**
+     * Phase 2.2 — final verdict from the {@link ITrajectoryReflector}, if any.
+     * Set when the periodic trajectory check flipped to `off_track` or
+     * `stuck`. The PlanExecutor surfaces this into `PlanContext.trajectoryVerdict`
+     * on replan so the planner can pivot the plan rather than re-deriving the
+     * failure mode.
+     */
+    trajectoryVerdict?: {
+        verdict: Exclude<TrajectoryVerdict, "on_track">;
+        reason: string;
+    };
+}
+export declare class AgentLoop {
+    private readonly opts;
+    private static readonly DEFAULT_MAX_STEPS;
+    private static readonly DEFAULT_TIMEOUT_MS;
+    private readonly history;
+    private readonly reflector;
+    private readonly notes;
+    private trajectoryVerdict?;
+    constructor(opts: AgentRunOptions);
+    run(): Promise<AgentRunResult>;
+    private observe;
+    /**
+     * Translate a model-frame action into an MCP-frame action by scaling
+     * coordinates to the absolute screen size, depending on the
+     * `_coordSystem` payload hint that `parsedToAction` plants.
+     */
+    private scaleAction;
+    /**
+     * Phase 1.4 — invoke the optional model-based action reflector. Verdicts
+     * are advisory: a `passed: false` result is appended to the just-pushed
+     * StepHistory entry's `rationale` (so the next `predict` sees it), and
+     * the entry's `actionResult` is upgraded from `ok` → `act_failed` so
+     * downstream code (skill evolver, trajectory reflector) treats it as a
+     * miss.
+     *
+     * The reflector itself is responsible for {@link IActionReflector.shouldReflect}
+     * gating; this helper just composes the wiring. Cost is rolled into the
+     * surrounding {@link AgentLoop.run} `usage` aggregator.
+     */
+    private maybeRunActionReflector;
+    /**
+     * Phase 2.2 — periodic global progress assessment. The reflector decides
+     * its own gating (default: every 5 steps); we just compose the wiring.
+     *
+     * Returns a non-empty veto string only on `stuck`. `off_track` injects an
+     * advisory line into the most recent history entry's rationale so the
+     * grounder sees it next predict, but the loop keeps running. The verdict
+     * itself is stashed on `this.trajectoryVerdict` so the PlanExecutor can
+     * thread it into `PlanContext.trajectoryVerdict` on replan.
+     */
+    private maybeRunTrajectoryReflector;
+    private recordHistory;
+    private fireStep;
+    private fire;
+    private finish;
+}