@beeos-ai/device-agent 0.4.0

package/dist/agent-runtime/planner/prompts.js

@@ -0,0 +1,165 @@
+/**
+ * agent-runtime/planner/prompts — system + user prompt builders for the
+ * LLM-driven `LLMPlanner`. Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/planner/prompts.py
+ *
+ * The planner is *strategy-only*: it never picks pixel coordinates, never
+ * emits a single Action — it emits a Plan (ordered list of small,
+ * verifiable sub-tasks) that the inner `AgentLoop` then executes one by
+ * one with a vision-grounding adapter (AutoGLM-Phone-9B today).
+ */
+/**
+ * The ALLOW-LIST surfaced into the system prompt, kept in lock-step with
+ * `device-common` `ActionType`. We intentionally drop the terminal verbs
+ * (`done`, `fail`, `call_for_user`) — those are reserved for the inner
+ * adapter's last step, not for plan items.
+ */
+const ACTION_KIND_ENUM = [
+    "click",
+    "double_click",
+    "right_click",
+    "tap",
+    "long_press",
+    "drag",
+    "swipe",
+    "scroll",
+    "type",
+    "key",
+    "hotkey",
+    "back",
+    "home",
+    "navigate",
+    "launch_app",
+    "wait",
+    "screenshot",
+    "ui_dump",
+    "execute_command",
+    "run_code",
+].join("|");
+const SYSTEM_PROMPT_TEMPLATE = `\
+You are DeviceAgent's PLANNER — a strategist that decomposes a user goal into
+an ordered list of small, verifiable steps that another agent (the GROUNDER /
+mobile VLM) will execute on a {ENV}.
+
+You DO NOT pick pixel coordinates. The GROUNDER does that. You DO describe each
+step using natural language ("the blue 'Submit' button", "the search input at
+the top") so that the grounder can localise it from a screenshot or
+accessibility tree.
+
+Reply in **strict JSON only**, matching this schema:
+
+{
+  "rationale": "<one paragraph, 60 words max>",
+  "plan": {
+    "goal": "<echo of the user goal>",
+    "steps": [
+      {
+        "idx": 0,
+        "description": "<one short sentence, present tense>",
+        "action_kind": "{ACTION_KIND_ENUM}",
+        "target_desc": "<English description of which element, or null for non-targeted actions>",
+        "payload": { },
+        "expected_outcome": "<short post-condition you can visually verify>"
+      }
+    ]
+  }
+}
+
+Rules:
+  - Plans should be 2–8 steps. Prefer SHORT plans; the executor will replan
+    if a step fails, so you do not need to plan every micro-tap.
+  - One coarse user-visible action per step ("open WeChat", "tap Moments",
+    "type the message"). Do NOT split a single tap into pre/post screenshots.
+  - Coordinates are NEVER part of your plan; let the grounder pick them.
+  - Screen size is {SCREEN_W}x{SCREEN_H}. Use English / 中文 descriptors that
+    pinpoint elements ("top-right camera icon", "bottom 'Send' button") if it
+    helps disambiguation.
+  - When the user can authenticate / solve a captcha faster than the agent,
+    insert an explicit step with action_kind="wait" and an expected_outcome
+    that keys off the post-login state.
+  - The current local date is {DATE}. Inject it when a task references "today"
+    / "now".
+  - If a "Notes" section is present, treat it as authoritative ground truth
+    captured by the agent during execution (verification codes, prices, user
+    inputs, etc.). DO NOT plan steps to re-collect already-recorded info.
+  - If a "Trajectory verdict" is present, the inner loop bailed mid-execution
+    because progress stalled ("off_track" / "stuck"). Address the cited
+    "reason" head-on: pivot the plan, drop the failing branch, or insert an
+    explicit recovery step (e.g. "go back", "wait for captcha", etc.).
+
+Reply ONLY with valid JSON matching the schema. No markdown. No prose.
+`;
+const REPLAN_SUFFIX_TEMPLATE = `
+
+REPLAN CONTEXT
+The previous plan failed at step {LAST_STEP_IDX}. Failure reason:
+{FAILURE_REASON}
+
+Inspect the latest observation, then emit a NEW plan that takes the current
+state into account. You may keep, drop, or reorder steps.`;
+export function renderSystemPrompt(opts) {
+    const date = opts.date ?? new Date().toISOString().slice(0, 10);
+    return SYSTEM_PROMPT_TEMPLATE
+        .replace("{ENV}", opts.env || "device")
+        .replace("{ACTION_KIND_ENUM}", ACTION_KIND_ENUM)
+        .replace("{SCREEN_W}", String(opts.screenW || 0))
+        .replace("{SCREEN_H}", String(opts.screenH || 0))
+        .replace("{DATE}", date);
+}
+export function renderUserPrompt(opts) {
+    const lines = [];
+    lines.push(`Task: ${opts.goal}`);
+    if (opts.observation) {
+        lines.push(`Screen: ${opts.observation.screenW}x${opts.observation.screenH}`);
+        if (opts.observation.uiDump) {
+            lines.push(`UI dump (truncated): ${opts.observation.uiDump.slice(0, 4000)}`);
+        }
+    }
+    if (opts.notes && opts.notes.length > 0) {
+        lines.push("");
+        lines.push("Notes (authoritative — captured by the agent mid-task):");
+        for (const note of opts.notes) {
+            lines.push(`  - ${note}`);
+        }
+    }
+    if (opts.trajectoryVerdict) {
+        lines.push("");
+        lines.push(`Trajectory verdict: ${opts.trajectoryVerdict.verdict} — ${opts.trajectoryVerdict.reason}`);
+    }
+    lines.push("");
+    lines.push("Recent actions:");
+    lines.push(renderHistoryBlock(opts.history));
+    if (opts.failureReason !== undefined) {
+        lines.push(REPLAN_SUFFIX_TEMPLATE
+            .replace("{LAST_STEP_IDX}", opts.lastStepIdx !== undefined ? String(opts.lastStepIdx) : "-1")
+            .replace("{FAILURE_REASON}", opts.failureReason)
+            .trimEnd());
+    }
+    return lines.join("\n");
+}
+function renderHistoryBlock(history) {
+    if (history.length === 0)
+        return "(no prior steps)";
+    const lines = [];
+    for (const h of history.slice(-8)) {
+        const verb = h.action?.type ?? "noop";
+        const target = formatTarget(h.action?.target);
+        const summary = h.observationSummary ? ` :: ${h.observationSummary}` : "";
+        const rationale = h.rationale ? ` (${h.rationale.slice(0, 80)})` : "";
+        lines.push(`  - step ${h.stepIdx}: ${verb}${target}${rationale}${summary}`);
+    }
+    return lines.join("\n");
+}
+function formatTarget(t) {
+    if (!t)
+        return "";
+    if (t.kind === "point")
+        return ` @(${t.x},${t.y})`;
+    if (t.kind === "box")
+        return ` [box ${t.x1},${t.y1}->${t.x2},${t.y2}]`;
+    if (t.kind === "text")
+        return ` "${String(t.value).slice(0, 40)}"`;
+    return "";
+}
+export const STRICT_SCHEMA_HINT = "Reply ONLY with valid JSON matching the schema. No markdown, no prose.";
+//# sourceMappingURL=prompts.js.map

package/dist/agent-runtime/planner/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../../src/agent-runtime/planner/prompts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG;IACvB,OAAO;IACP,cAAc;IACd,aAAa;IACb,KAAK;IACL,YAAY;IACZ,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,KAAK;IACL,QAAQ;IACR,MAAM;IACN,MAAM;IACN,UAAU;IACV,YAAY;IACZ,MAAM;IACN,YAAY;IACZ,SAAS;IACT,iBAAiB;IACjB,UAAU;CACX,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoD9B,CAAC;AAEF,MAAM,sBAAsB,GAAG;;;;;;;0DAO2B,CAAC;AAE3D,MAAM,UAAU,kBAAkB,CAAC,IAKlC;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChE,OAAO,sBAAsB;SAC1B,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC;SACtC,OAAO,CAAC,oBAAoB,EAAE,gBAAgB,CAAC;SAC/C,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;SAChD,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;SAChD,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAUhC;IACC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9E,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QACtE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,uBAAuB,IAAI,CAAC,iBAAiB,CAAC,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAC3F,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CACR,sBAAsB;aACnB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aAC5F,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,aAAa,CAAC;aAC/C,OAAO,EAAE,CACb,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAsB;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,MAAM,CAAC;QACtC,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,CAAqD;IACzE,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IACnD,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC;IACvE,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC;IACnE,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAC7B,wEAAwE,CAAC"}

package/dist/agent-runtime/reflection.d.ts

@@ -0,0 +1,81 @@
+/**
+ * agent-runtime/reflection — hierarchical reflection layer for AgentLoop.
+ *
+ * Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/loop/reflection.py
+ *
+ * Phase-1 reflector is heuristic-only (cheap):
+ *  - **Action reflection** — 4 identical actions in a row → stall.
+ *  - **Trajectory reflection** — ≥5 screenshots in the last 6 steps → stall
+ *    (the model is staring at the screen instead of acting).
+ *  - **Pre-action sanity** — block obviously-broken `nextAction` candidates
+ *    *before* dispatch (zero-cost; mirrors mobile-use `enable_pre_reflection`).
+ *  - **Model-based action reflection** is plumbed via `IActionReflector` —
+ *    composed by AgentLoop after `act()` returns, gated by confidence /
+ *    `actionResult` so the heuristic layer fires first.
+ */
+import type { Action } from "@beeos-ai/device-common";
+import type { IActionReflector } from "./reflectors/action-reflector.js";
+import type { ITrajectoryReflector } from "./reflectors/trajectory-reflector.js";
+import type { IModelAdapter, StepHistory } from "./adapters/base.js";
+/**
+ * Veto reason returned by {@link HierarchicalReflector.maybeBlockAction}.
+ *  - `reason` — short human-readable string, copied into the next StepHistory
+ *    rationale so the grounder sees *why* its previous attempt was blocked.
+ *  - `code` — stable identifier for telemetry / metrics tagging.
+ */
+export interface PreActionVeto {
+    code: "double_type_without_focus" | "double_tap_same_coords" | "blind_swipe" | "vlm_emitted_shell";
+    reason: string;
+}
+export interface HierarchicalReflectorOptions {
+    /** Optional VLM adapter — reserved for future model-based reflection hooks. */
+    adapter?: IModelAdapter;
+    /**
+     * Optional model-based reflector composed after `act()` returns. AgentLoop
+     * calls `actionReflector?.shouldReflect(...)` first; the heuristic veto
+     * still runs before any model invocation.
+     */
+    actionReflector?: IActionReflector;
+    /**
+     * Optional trajectory reflector (Phase 2.2). AgentLoop fires it every N
+     * steps to assess global progress; bail-out verdicts (`off_track` /
+     * `stuck`) are converted into a reflection veto so the PlanExecutor can
+     * trigger a replan.
+     */
+    trajectoryReflector?: ITrajectoryReflector;
+}
+export declare class HierarchicalReflector {
+    private readonly adapter?;
+    readonly actionReflector?: IActionReflector;
+    readonly trajectoryReflector?: ITrajectoryReflector;
+    constructor(adapterOrOptions?: IModelAdapter | HierarchicalReflectorOptions);
+    /**
+     * Returns a non-empty veto reason when the loop should stop, or `null` if
+     * the run is healthy and may continue.
+     */
+    maybeVeto(history: StepHistory[], step: number, _maxSteps: number): string | null;
+    /**
+     * Pre-action sanity — block obviously-broken `nextAction` candidates
+     * **before** AgentLoop dispatches them to MCP. Returns a veto when the
+     * action looks like a known-bad pattern; the AgentLoop should then skip
+     * the dispatch and record the step with `actionResult: "skipped"`.
+     *
+     * Rules (cover ~80% of common grounder mistakes — see plan Task 1.3):
+     *  1. Two consecutive `type` actions with no intervening `tap` /
+     *     `long_press` → focus probably wasn't established (mobile-use
+     *     `pre_reflection` rule).
+     *  2. Two consecutive `tap` actions on the same point (≤ 8px tolerance)
+     *     → looks like a double-click misclassified as two taps.
+     *  3. Two consecutive `swipe` actions in the same direction without an
+     *     intervening `screenshot` → blind scrolling without re-grounding.
+     *  4. The grounder emitted `execute_command` (the VLM should never produce
+     *     shell). This is exactly the "Baidu image" failure mode that
+     *     motivated this rule set.
+     *
+     * Heuristics are conservative — when in doubt, return `null` (let the
+     * action through). Repeated stalls are still caught by `maybeVeto`.
+     */
+    maybeBlockAction(history: StepHistory[], nextAction: Action): PreActionVeto | null;
+}
+export declare function actionsEqual(a: Action, b: Action): boolean;

package/dist/agent-runtime/reflection.js

@@ -0,0 +1,189 @@
+/**
+ * agent-runtime/reflection — hierarchical reflection layer for AgentLoop.
+ *
+ * Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/loop/reflection.py
+ *
+ * Phase-1 reflector is heuristic-only (cheap):
+ *  - **Action reflection** — 4 identical actions in a row → stall.
+ *  - **Trajectory reflection** — ≥5 screenshots in the last 6 steps → stall
+ *    (the model is staring at the screen instead of acting).
+ *  - **Pre-action sanity** — block obviously-broken `nextAction` candidates
+ *    *before* dispatch (zero-cost; mirrors mobile-use `enable_pre_reflection`).
+ *  - **Model-based action reflection** is plumbed via `IActionReflector` —
+ *    composed by AgentLoop after `act()` returns, gated by confidence /
+ *    `actionResult` so the heuristic layer fires first.
+ */
+export class HierarchicalReflector {
+    adapter;
+    actionReflector;
+    trajectoryReflector;
+    constructor(adapterOrOptions) {
+        if (adapterOrOptions && "name" in adapterOrOptions) {
+            this.adapter = adapterOrOptions;
+        }
+        else {
+            const opts = adapterOrOptions ?? {};
+            this.adapter = opts.adapter;
+            this.actionReflector = opts.actionReflector;
+            this.trajectoryReflector = opts.trajectoryReflector;
+        }
+        void this.adapter; // reserved for future model-based reflection
+    }
+    /**
+     * Returns a non-empty veto reason when the loop should stop, or `null` if
+     * the run is healthy and may continue.
+     */
+    maybeVeto(history, step, _maxSteps) {
+        if (step <= 2 || history.length === 0)
+            return null;
+        const last = history[history.length - 1].action;
+        if (!last)
+            return null;
+        // Action reflection — 4 identical actions in a row.
+        const recent = history
+            .slice(-4)
+            .map((h) => h.action)
+            .filter((a) => a !== null);
+        if (recent.length === 4 && recent.every((a) => actionsEqual(a, recent[0]))) {
+            return `stalled: action ${recent[0].type} repeated 4 consecutive times`;
+        }
+        // Trajectory reflection — screenshot-heavy stall.
+        const recentTypes = history
+            .slice(-6)
+            .map((h) => h.action?.type)
+            .filter((t) => Boolean(t));
+        if (recentTypes.filter((t) => t === "screenshot").length >= 5) {
+            return "stalled: screenshot-heavy trajectory with no actuation";
+        }
+        return null;
+    }
+    /**
+     * Pre-action sanity — block obviously-broken `nextAction` candidates
+     * **before** AgentLoop dispatches them to MCP. Returns a veto when the
+     * action looks like a known-bad pattern; the AgentLoop should then skip
+     * the dispatch and record the step with `actionResult: "skipped"`.
+     *
+     * Rules (cover ~80% of common grounder mistakes — see plan Task 1.3):
+     *  1. Two consecutive `type` actions with no intervening `tap` /
+     *     `long_press` → focus probably wasn't established (mobile-use
+     *     `pre_reflection` rule).
+     *  2. Two consecutive `tap` actions on the same point (≤ 8px tolerance)
+     *     → looks like a double-click misclassified as two taps.
+     *  3. Two consecutive `swipe` actions in the same direction without an
+     *     intervening `screenshot` → blind scrolling without re-grounding.
+     *  4. The grounder emitted `execute_command` (the VLM should never produce
+     *     shell). This is exactly the "Baidu image" failure mode that
+     *     motivated this rule set.
+     *
+     * Heuristics are conservative — when in doubt, return `null` (let the
+     * action through). Repeated stalls are still caught by `maybeVeto`.
+     */
+    maybeBlockAction(history, nextAction) {
+        if (!nextAction)
+            return null;
+        // Rule 4 — VLM emitted shell. Always block.
+        if (nextAction.type === "execute_command") {
+            return {
+                code: "vlm_emitted_shell",
+                reason: "grounder emitted execute_command — VLMs should never produce shell; planner should route shell via tools",
+            };
+        }
+        if (history.length === 0)
+            return null;
+        const prev = lastNonNullAction(history);
+        if (!prev)
+            return null;
+        // Rule 1 — double-type without re-focus.
+        if (nextAction.type === "type" && prev.action.type === "type") {
+            return {
+                code: "double_type_without_focus",
+                reason: "two consecutive `type` actions without a tap/long_press between — focus probably wasn't established",
+            };
+        }
+        // Rule 2 — double-tap as two taps on (almost) the same point.
+        if (nextAction.type === "tap" &&
+            prev.action.type === "tap" &&
+            isPointTarget(nextAction.target) &&
+            isPointTarget(prev.action.target)) {
+            const dx = Math.abs(nextAction.target.x - prev.action.target.x);
+            const dy = Math.abs(nextAction.target.y - prev.action.target.y);
+            if (dx <= 8 && dy <= 8) {
+                return {
+                    code: "double_tap_same_coords",
+                    reason: `two consecutive tap on (${prev.action.target.x},${prev.action.target.y}) → looks like a misclassified double_tap`,
+                };
+            }
+        }
+        // Rule 3 — blind swipe (same direction, no screenshot in between).
+        if (nextAction.type === "swipe" &&
+            prev.action.type === "swipe" &&
+            isBoxTarget(nextAction.target) &&
+            isBoxTarget(prev.action.target)) {
+            const sameDir = sign(nextAction.target.x2 - nextAction.target.x1) ===
+                sign(prev.action.target.x2 - prev.action.target.x1) &&
+                sign(nextAction.target.y2 - nextAction.target.y1) ===
+                    sign(prev.action.target.y2 - prev.action.target.y1);
+            const sawScreenshotBetween = anyActionBetween(history, prev.idx, history.length, (a) => a.type === "screenshot");
+            if (sameDir && !sawScreenshotBetween) {
+                return {
+                    code: "blind_swipe",
+                    reason: "swipe in the same direction as the previous step with no intervening screenshot — re-ground before scrolling further",
+                };
+            }
+        }
+        return null;
+    }
+}
+/* -------------------------------------------------------------------------- */
+/* helpers                                                                    */
+/* -------------------------------------------------------------------------- */
+export function actionsEqual(a, b) {
+    if (a.type !== b.type)
+        return false;
+    if (!targetsEqual(a.target, b.target))
+        return false;
+    return jsonEquals(a.payload, b.payload);
+}
+function targetsEqual(a, b) {
+    if (!a && !b)
+        return true;
+    if (!a || !b)
+        return false;
+    if (a.kind !== b.kind)
+        return false;
+    return jsonEquals(a, b);
+}
+function jsonEquals(a, b) {
+    if (Object.is(a, b))
+        return true;
+    if (typeof a !== "object" || typeof b !== "object" || a === null || b === null)
+        return false;
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+function isPointTarget(t) {
+    return !!t && t.kind === "point";
+}
+function isBoxTarget(t) {
+    return !!t && t.kind === "box";
+}
+function sign(n) {
+    return n > 0 ? 1 : n < 0 ? -1 : 0;
+}
+function lastNonNullAction(history) {
+    for (let i = history.length - 1; i >= 0; i--) {
+        const a = history[i].action;
+        if (a)
+            return { idx: i, action: a };
+    }
+    return null;
+}
+function anyActionBetween(history, startExclusive, endExclusive, pred) {
+    for (let i = startExclusive + 1; i < endExclusive; i++) {
+        const a = history[i].action;
+        if (a && pred(a))
+            return true;
+    }
+    return false;
+}
+//# sourceMappingURL=reflection.js.map

package/dist/agent-runtime/reflection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflection.js","sourceRoot":"","sources":["../../src/agent-runtime/reflection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAyCH,MAAM,OAAO,qBAAqB;IACf,OAAO,CAAiB;IAChC,eAAe,CAAoB;IACnC,mBAAmB,CAAwB;IAEpD,YAAY,gBAA+D;QACzE,IAAI,gBAAgB,IAAI,MAAM,IAAK,gBAAkC,EAAE,CAAC;YACtE,IAAI,CAAC,OAAO,GAAG,gBAAiC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAI,gBAA6D,IAAI,EAAE,CAAC;YAClF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;YAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACtD,CAAC;QACD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,6CAA6C;IAClE,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,OAAsB,EAAE,IAAY,EAAE,SAAiB;QAC/D,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEnD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAChD,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,oDAAoD;QACpD,MAAM,MAAM,GAAG,OAAO;aACnB,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,mBAAmB,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,+BAA+B,CAAC;QAC1E,CAAC;QAED,kDAAkD;QAClD,MAAM,WAAW,GAAG,OAAO;aACxB,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC;aAC1B,MAAM,CAAC,CAAC,CAAC,EAAuB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC9D,OAAO,wDAAwD,CAAC;QAClE,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,gBAAgB,CAAC,OAAsB,EAAE,UAAkB;QACzD,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QAE7B,4CAA4C;QAC5C,IAAI,UAAU,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC1C,OAAO;gBACL,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EACJ,0GAA0G;aAC7G,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,yCAAyC;QACzC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9D,OAAO;gBACL,IAAI,EAAE,2BAA2B;gBACjC,MAAM,EACJ,qGAAqG;aACxG,CAAC;QACJ,CAAC;QAED,8DAA8D;QAC9D,IACE,UAAU,CAAC,IAAI,KAAK,KAAK;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK;YAC1B,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EACjC,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,wBAAwB;oBAC9B,MAAM,EAAE,2BAA2B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,2CAA2C;iBAC3H,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mEAAmE;QACnE,IACE,UAAU,CAAC,IAAI,KAAK,OAAO;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO;YAC5B,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;YAC9B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAC/B,CAAC;YACD,MAAM,OAAO,GACX,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,oBAAoB,GAAG,gBAAgB,CAC3C,OAAO,EACP,IAAI,CAAC,GAAG,EACR,OAAO,CAAC,MAAM,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAC/B,CAAC;YACF,IAAI,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBACrC,OAAO;oBACL,IAAI,EAAE,aAAa;oBACnB,MAAM,EACJ,sHAAsH;iBACzH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAEhF,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IACpD,OAAO,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,YAAY,CAAC,CAAmB,EAAE,CAAmB;IAC5D,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3B,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,CAAU,EAAE,CAAU;IACxC,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7F,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,CAAmB;IACxC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;AACnC,CAAC;AAED,SAAS,WAAW,CAClB,CAAmB;IAEnB,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;AACjC,CAAC;AAED,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAsB;IAEtB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC;YAAE,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAsB,EACtB,cAAsB,EACtB,YAAoB,EACpB,IAA4B;IAE5B,KAAK,IAAI,CAAC,GAAG,cAAc,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/agent-runtime/reflectors/action-reflector.d.ts

@@ -0,0 +1,97 @@
+/**
+ * agent-runtime/reflectors/action-reflector — model-based action reflection.
+ *
+ * Phase 1.4 of the device-agent reflector upgrade plan
+ * ([../../../../../../.cursor/plans/device-agent-reflector-skills_d5599350.plan.md]).
+ *
+ * Goal: catch *the same step* the grounder just emitted when it's likely
+ * wrong (low confidence, or the MCP `act` failed) and feed back a textual
+ * verdict the next predict call can read. Concretely:
+ *   - heuristic veto (`HierarchicalReflector.maybeVeto`) runs first and
+ *     terminates the loop with a hard stop on stalls;
+ *   - if the heuristics are quiet, we ask the action reflector whether to
+ *     take a closer look. The default {@link LLMActionReflector} fires on
+ *     `actionResult === "act_failed"` OR on low-confidence predictions
+ *     OR (as a defensive fallback) on a small whitelist of high-risk verbs
+ *     so we still help even when the adapter doesn't expose logprobs.
+ *   - the reflector verdict is **advisory**: a `passed: false` result is
+ *     spliced into the next StepHistory rationale ("act_failed: …") so the
+ *     grounder retries with that context. We do NOT terminate the loop —
+ *     stalls are still the heuristic layer's job.
+ *
+ * The cost model is documented in
+ * `agents/device-agent/docs/reflectors.md` (Phase 2.4). Per-task budget
+ * with default config is ≤ 4 cheap text glm-4-flash calls / task, ~600-1200
+ * input tokens / 50 output tokens each.
+ */
+import OpenAI from "openai";
+import type { Action } from "@beeos-ai/device-common";
+import type { ModelUsage, Prediction, StepHistory } from "../adapters/base.js";
+/** Verbs that warrant a reflection on Branch C of the logprobs spike. */
+export declare const HIGH_RISK_VERBS: ReadonlySet<Action["type"]>;
+export type ActionReflectorGating = "logprob" | "logprob-degraded" | "whitelist";
+export interface ActionReflectCtx {
+    /** Goal of the surrounding task. */
+    goal: string;
+    /** Up to N previous step entries (most recent last). */
+    history: StepHistory[];
+    /** The prediction we want to reflect on. */
+    prediction: Prediction;
+    /** Last observation summary (no base64 — text only). */
+    observationSummary: string;
+}
+export interface ActionReflectVerdict {
+    /**
+     * `true`  — reflector agrees with the action; nothing to do.
+     * `false` — reflector flagged the action; the AgentLoop should record
+     *           `act_failed: <reason>` and let the grounder retry.
+     */
+    passed: boolean;
+    reason: string;
+    usage: ModelUsage;
+}
+export interface IActionReflector {
+    /**
+     * Should the loop invoke {@link reflect} on this prediction? Cheap to
+     * call — only returns true on the small subset that warrants the LLM
+     * round-trip.
+     */
+    shouldReflect(history: StepHistory[], lastPrediction: Prediction, actionResult: StepHistory["actionResult"]): boolean;
+    /** Run the actual reflection. */
+    reflect(ctx: ActionReflectCtx, signal?: AbortSignal): Promise<ActionReflectVerdict>;
+    /** Optional cleanup. */
+    close?(): Promise<void>;
+}
+export interface LLMActionReflectorOptions {
+    /** Override `process.env.PLANNER_BASE_URL` (planner endpoint reused). */
+    baseUrl?: string;
+    apiKey?: string;
+    /** Defaults to `glm-4-flash`. */
+    modelName?: string;
+    /** Confidence threshold; predictions below this fire the reflector. Defaults 0.6. */
+    confidenceThreshold?: number;
+    /**
+     * Spike branch (see {@link ActionReflectorGating}). Defaults to "whitelist"
+     * which is the safe fallback when the AutoGLM-Phone-9B endpoint doesn't
+     * expose logprobs.
+     */
+    gating?: ActionReflectorGating;
+    /** Sampling temperature. Defaults to 0. */
+    temperature?: number;
+    /** Max output tokens. Defaults to 80. */
+    maxTokens?: number;
+    /** Reused OpenAI client for tests. */
+    client?: OpenAI;
+}
+export declare class LLMActionReflector implements IActionReflector {
+    readonly name = "llm-action-reflector";
+    private readonly client;
+    private readonly modelName;
+    private readonly confidenceThreshold;
+    private readonly gating;
+    private readonly maxTokens;
+    private readonly temperature;
+    constructor(opts?: LLMActionReflectorOptions);
+    shouldReflect(_history: StepHistory[], lastPrediction: Prediction, actionResult: StepHistory["actionResult"]): boolean;
+    reflect(ctx: ActionReflectCtx, signal?: AbortSignal): Promise<ActionReflectVerdict>;
+}