jev-layer 0.2.0 → 0.3.0

package/src/mcp-server.mjs

@@ -1,15 +1,18 @@
 #!/usr/bin/env node
 import { createInterface } from "node:readline";
+import { readFile } from "node:fs/promises";
 import { appendExecutionReceipt, appendRoutingCase, buildExecutionReceipt, replayCasePath } from "./receipts.mjs";
 import { configuredProvider, configuredReplayPath, loadConfig } from "./config.mjs";
 import { decideBrowserStep } from "./browser.mjs";
+import { decideComputerStep } from "./computer.mjs";
 import { superviseWork } from "./supervision.mjs";
 import { buildShadowCompactionReport } from "./shadow-compaction.mjs";
 import { recommendModelRoute } from "./model-routing.mjs";
 import { routeRequest } from "./route.mjs";
 
 const { config } = await loadConfig();
-const serverInfo = { name: "jev-layer", version: "0.1.0" };
+const { version } = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8"));
+const serverInfo = { name: "jev-layer", version };
 const CASES_PATH = replayCasePath(configuredReplayPath(config));
 const pendingDecisions = new Map();
 const tools = [
@@ -23,6 +26,7 @@ const tools = [
         intent: { type: "string" },
         harness: { type: "string" },
         context: { type: "object" },
+        assets: { type: "object", description: "Inline PNG/JPEG assets keyed by reference; images are passed outside the text projection." },
         actor_permissions: { type: "array", items: { type: "string" } },
         capabilities: { type: "array", items: { type: "object" } },
         policy: { type: "object" },
@@ -41,7 +45,41 @@ const tools = [
         goal: { type: "string" },
         harness: { type: "string" },
         start_url: { type: ["string", "null"] },
-        observation: { type: "object" },
+        observation: { type: "object", description: "Host snapshot: url, visible_text, targets, tabs, scroll, and optional screenshot {format,data_base64,width,height}." },
+        progress: { type: "object" },
+        actor_permissions: { type: "array", items: { type: "string" } },
+        policy: { type: "object" },
+        provider: { type: "string", enum: ["demo", "typesafe", "openrouter"] },
+        enabled: { type: "boolean" },
+      },
+    },
+  },
+  {
+    name: "jev_computer_step",
+    description: "Recommend one host-declared desktop action, optionally grounding a click in a screenshot. Advisory only: host validates freshness, permissions, approval and execution.",
+    inputSchema: {
+      type: "object",
+      required: ["goal", "observation", "actions", "actor_permissions"],
+      properties: {
+        goal: { type: "string" },
+        harness: { type: "string" },
+        observation: { type: "object", description: "Host screenshot {format,data_base64,width,height}, optional application and visible_text." },
+        actions: { type: "array", maxItems: 32, items: {
+          type: "object", required: ["id", "operation"],
+          properties: {
+            id: { type: "string" },
+            operation: { type: "string", enum: ["click", "type", "key", "scroll", "handoff"] },
+            args: { type: "object", description: "Exact host-declared arguments; Jev does not generate typing or key input." },
+            target: { type: "string", description: "Unique visible click target to locate when no x/y are supplied." },
+            description: { type: "string" },
+            risk: { type: "string", enum: ["low", "medium", "high", "critical"] },
+            available: { type: "boolean" },
+            availability: { type: "object" },
+            permissions: { type: "array", items: { type: "string" } },
+            policy: { type: "object" },
+          },
+        } },
+        actor_permissions: { type: "array", items: { type: "string" }, description: "Must explicitly include computer; never grants host permission." },
         policy: { type: "object" },
         provider: { type: "string", enum: ["demo", "typesafe", "openrouter"] },
         enabled: { type: "boolean" },
@@ -148,6 +186,7 @@ async function handle(message) {
   const args = params.arguments && typeof params.arguments === "object" ? params.arguments : {};
   if (params.name === "jev_route") return route(args, id);
   if (params.name === "jev_browser_step") return browserStep(args, id);
+  if (params.name === "jev_computer_step") return computerStep(args, id);
   if (params.name === "jev_supervise") return supervise(args, id);
   if (params.name === "jev_model_route") return modelRoute(args, id);
   if (params.name === "jev_shadow_compaction") return shadowCompaction(args, id);
@@ -175,17 +214,22 @@ async function browserStep(args, id) {
     provider: configuredProvider(config, requestedProvider),
   });
   routed.decision.browser_action = routed.action
-    ? {
-        id: routed.action.id,
-        operation: routed.action.operation,
-        target_id: routed.action.target_id ?? null,
-        option_id: routed.action.option_id ?? null,
-        consequential: routed.action.consequential,
-      }
+    ? { ...routed.action, target_id: routed.action.target_id ?? null, option_id: routed.action.option_id ?? null }
     : null;
   return persistDecision(routed.request, routed.decision, id);
 }
 
+async function computerStep(args, id) {
+  const { provider: requestedProvider, enabled, ...input } = args;
+  const routed = await decideComputerStep(input, {
+    config,
+    enabled: enabled ?? (process.env.JEV_COMPUTER_USE === "1" || config.features?.computer_use),
+    provider: configuredProvider(config, requestedProvider),
+  });
+  routed.decision.computer_action = routed.action;
+  return persistDecision(routed.request ?? { harness: input.harness ?? "computer", intent: input.goal }, routed.decision, id);
+}
+
 async function supervise(args, id) {
   const { provider: requestedProvider, enabled, ...input } = args;
   const result = await superviseWork({
@@ -217,7 +261,9 @@ async function shadowCompaction(args, id) {
 }
 
 async function persistDecision(request, decision, id) {
-  pendingDecisions.set(decision.correlation_id, { request, decision });
+  const { assets, ...retainedRequest } = request;
+  if (Object.keys(assets ?? {}).length) retainedRequest.assets_omitted = true;
+  pendingDecisions.set(decision.correlation_id, { request: retainedRequest, decision });
   try {
     await appendRoutingCase({ path: CASES_PATH, request, decision });
     decision.receipt = { ...decision.receipt, replay_case_id: decision.correlation_id, replay_persisted: true };

package/src/providers/typesafe.mjs

@@ -1,7 +1,57 @@
+import { normalizeImageAssets } from "../images.mjs";
+
+function containsReference(value, reference) {
+  if (typeof value === "string") return value.includes(reference);
+  return value !== null && typeof value === "object"
+    && Object.values(value).some((child) => containsReference(child, reference));
+}
+
+function openRouterImageInput(state, questions, images) {
+  const parts = Object.fromEntries(Object.entries(images).map(([reference, image]) => [reference, {
+    type: "image_url",
+    image_url: { url: `data:image/${image.format === "png" ? "png" : "jpeg"};base64,${image.data_base64}` },
+  }]));
+  const pattern = new RegExp(Object.keys(parts).sort((a, b) => b.length - a.length).map((key) => key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "g");
+  const used = new Set();
+  function expand(value) {
+    if (typeof value === "string") {
+      if (Object.hasOwn(parts, value)) {
+        used.add(value);
+        return parts[value];
+      }
+      const content = [];
+      let offset = 0;
+      for (const match of value.matchAll(pattern)) {
+        if (match.index > offset) content.push(value.slice(offset, match.index));
+        content.push(parts[match[0]]);
+        used.add(match[0]);
+        offset = match.index + match[0].length;
+      }
+      if (!content.length) return value;
+      if (offset < value.length) content.push(value.slice(offset));
+      return content;
+    }
+    if (Array.isArray(value)) return value.map(expand);
+    if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, expand(child)]));
+    return value;
+  }
+  const input = {
+    state: expand(state),
+    questions: Object.fromEntries(Object.entries(questions).map(([key, question]) => [key, {
+      ...question,
+      instructions: expand(question.instructions),
+      ...(question.criteria !== undefined ? { criteria: expand(question.criteria) } : {}),
+    }])),
+  };
+  if (used.size !== Object.keys(parts).length) throw new Error("every image asset must be referenced in state, instructions, or criteria");
+  return input;
+}
+
 export class TypeSafeProvider {
   name = "typesafe";
+  supportsImages = true;
 
-  constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-latest", timeoutMs = 2_000, fetchImpl = globalThis.fetch } = {}) {
+  constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-1.14.0", timeoutMs = 2_000, fetchImpl = globalThis.fetch } = {}) {
     this.apiKey = apiKey;
     this.endpoint = endpoint;
     this.model = model;
@@ -9,7 +59,7 @@ export class TypeSafeProvider {
     this.fetchImpl = fetchImpl;
   }
 
-  async decide({ state, candidates }) {
+  async decide({ state, candidates, assets }) {
     const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
       name: candidate.name,
       kind: candidate.kind,
@@ -18,6 +68,7 @@ export class TypeSafeProvider {
     }]));
     return this.evaluate({
       state,
+      assets,
       questions: {
         tool: {
           type: "choice",
@@ -28,27 +79,44 @@ export class TypeSafeProvider {
     });
   }
 
-  async evaluate({ state, questions }) {
+  async evaluate({ state, questions, assets }) {
     if (!this.apiKey) throw new Error("TYPESAFE_API_KEY is not configured");
     if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable");
+    const images = normalizeImageAssets(assets);
+    const hasImages = Object.keys(images).length > 0;
+    const hasBox = Object.values(questions).some((question) => question?.type === "box");
+    for (const reference of Object.keys(images)) {
+      if (!containsReference(state, reference)
+        && !Object.values(questions).some((question) => containsReference(question?.instructions, reference) || containsReference(question?.criteria, reference))) {
+        throw new Error("every image asset must be referenced in state, instructions, or criteria");
+      }
+    }
+    const body = JSON.stringify({ state, model: this.model, questions, ...(hasImages ? { assets: images } : {}) });
+    if (Buffer.byteLength(body) > 32 * 1024 * 1024) throw new Error("TypeSafe request exceeds the 32 MiB body limit");
     const response = await this.fetchImpl(this.endpoint, {
       method: "POST",
       headers: {
         Authorization: `Bearer ${this.apiKey}`,
         "Content-Type": "application/json",
+        ...(hasImages || hasBox ? { "TypeSafe-Danger-Zone-Images": "true" } : {}),
       },
-      body: JSON.stringify({ state, model: this.model, questions }),
+      body,
       signal: AbortSignal.timeout(this.timeoutMs),
     }).catch((error) => {
       if (error?.name === "TimeoutError") throw new Error(`TypeSafe request timed out after ${this.timeoutMs}ms`);
+      if (hasImages) throw new Error("TypeSafe image request failed");
       throw error;
     });
 
     if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`TypeSafe returned HTTP ${response.status}: ${body.slice(0, 240)}`);
+      // Image requests must not copy an upstream echo of image data into receipts.
+      const detail = hasImages ? "" : `: ${(await response.text()).slice(0, 240)}`;
+      throw new Error(`TypeSafe returned HTTP ${response.status}${detail}`);
     }
-    const raw = await response.json();
+    const raw = await response.json().catch((error) => {
+      if (hasImages) throw new Error("TypeSafe image response is not valid JSON");
+      throw error;
+    });
     if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
       throw new Error("TypeSafe response has no answers object");
     }
@@ -57,10 +125,12 @@ export class TypeSafeProvider {
 }
 
 export class OpenRouterDecisionsProvider {
+  supportsImages = true;
+
   constructor({
     apiKey = process.env.OPENROUTER_API_KEY,
     endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions",
-    model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13",
+    model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.14",
     timeoutMs = 5_000,
     fetchImpl = globalThis.fetch,
     httpReferer = process.env.OPENROUTER_HTTP_REFERER,
@@ -75,7 +145,7 @@ export class OpenRouterDecisionsProvider {
     this.httpReferer = httpReferer;
     this.appTitle = appTitle;
   }
-  async decide({ state, candidates }) {
+  async decide({ state, candidates, assets }) {
     const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
       name: candidate.name,
       kind: candidate.kind,
@@ -84,6 +154,7 @@ export class OpenRouterDecisionsProvider {
     }]));
     return this.evaluate({
       state,
+      assets,
       questions: {
         tool: {
           type: "choice",
@@ -94,9 +165,14 @@ export class OpenRouterDecisionsProvider {
     });
   }
 
-  async evaluate({ state, questions }) {
+  async evaluate({ state, questions, assets }) {
     if (!this.apiKey) throw new Error("OPENROUTER_API_KEY is not configured");
     if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable");
+    const images = normalizeImageAssets(assets);
+    const hasImages = Object.keys(images).length > 0;
+    const input = hasImages ? openRouterImageInput(state, questions, images) : { state, questions };
+    const body = JSON.stringify({ model: this.model, ...input });
+    if (Buffer.byteLength(body) > 32 * 1024 * 1024) throw new Error("OpenRouter Decisions request exceeds the 32 MiB body limit");
 
     const response = await this.fetchImpl(this.endpoint, {
       method: "POST",
@@ -106,18 +182,22 @@ export class OpenRouterDecisionsProvider {
         ...(this.httpReferer ? { "HTTP-Referer": this.httpReferer } : {}),
         ...(this.appTitle ? { "X-Title": this.appTitle } : {}),
       },
-      body: JSON.stringify({ model: this.model, questions, state }),
+      body,
       signal: AbortSignal.timeout(this.timeoutMs),
     }).catch((error) => {
       if (error?.name === "TimeoutError") throw new Error(`OpenRouter Decisions request timed out after ${this.timeoutMs}ms`);
+      if (hasImages) throw new Error("OpenRouter Decisions image request failed");
       throw error;
     });
 
     if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`OpenRouter Decisions returned HTTP ${response.status}: ${body.slice(0, 240)}`);
+      const detail = hasImages ? "" : `: ${(await response.text()).slice(0, 240)}`;
+      throw new Error(`OpenRouter Decisions returned HTTP ${response.status}${detail}`);
     }
-    const raw = await response.json();
+    const raw = await response.json().catch((error) => {
+      if (hasImages) throw new Error("OpenRouter Decisions image response is not valid JSON");
+      throw error;
+    });
     if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
       throw new Error("OpenRouter Decisions response has no answers object");
     }

package/src/receipts.mjs

@@ -167,6 +167,7 @@ function replayRequest(request) {
     actor_permissions: Array.isArray(request.actor_permissions) ? request.actor_permissions.filter((value) => typeof value === "string") : undefined,
     policy: request.policy && typeof request.policy === "object" ? request.policy : {},
     capabilities: Array.isArray(request.capabilities) ? request.capabilities.map(replayCapability) : [],
+    ...(request.assets_omitted || Object.keys(request.assets ?? {}).length ? { assets_omitted: true } : {}),
   };
 }
 

package/src/route.mjs

@@ -3,11 +3,12 @@ import { projectState } from "./context-filter.mjs";
 import { discoverCapabilities } from "./discovery.mjs";
 import { decisionCandidates, DEFAULT_POLICY, deterministicCandidate, filterCapabilities, normalizeCapabilities } from "./registry.mjs";
 import { resolveProvider } from "./providers/index.mjs";
+import { normalizeImageAssets } from "./images.mjs";
 
 export async function routeRequest(input, options = {}) {
   const discovered = options.discovery ? discoverCapabilities(options.discovery) : [];
   const request = normalizeRequest(discovered.length ? { ...input, capabilities: [...(Array.isArray(input?.capabilities) ? input.capabilities : []), ...discovered] } : input);
-  if (options.engine === "jevrouter") return routeViaJevRouter(request, options);
+  if (options.engine === "jevrouter" && !input.assets && !input.assets_omitted) return routeViaJevRouter(request, options);
 
   const started = performance.now();
   const effectivePolicy = { ...DEFAULT_POLICY, ...request.policy, ...(options.policy ?? {}) };
@@ -28,6 +29,14 @@ export async function routeRequest(input, options = {}) {
   try {
     provider = resolveProvider(options.provider ?? "demo", options);
     base.provider = provider.name;
+    if (request.assets_omitted) throw new Error("visual replay requires the original images; stored screenshots are intentionally omitted");
+    const assets = normalizeImageAssets(request.assets);
+    if (Object.keys(assets).length) {
+      if (options.engine === "jevrouter" || provider.supportsImages !== true) throw new Error("provider does not support image decisions");
+      projection.state.visual_assets = Object.keys(assets);
+      projection.context_bytes = byteLength(projection.state);
+    }
+    request.assets = assets;
   } catch (error) {
     return fallback(base, assessments, "provider_error", error instanceof Error ? error.message : String(error), projection.state);
   }
@@ -66,7 +75,7 @@ export async function routeRequest(input, options = {}) {
 
   let raw;
   try {
-    raw = await provider.decide({ state: projection.state, candidates: eligible });
+    raw = await provider.decide({ state: projection.state, candidates: eligible, assets: request.assets });
   } catch (error) {
     return fallback(base, assessments, "provider_error", error instanceof Error ? error.message : String(error), projection.state);
   }

package/test/browser.test.mjs

@@ -6,6 +6,7 @@ import { test } from "node:test";
 import { buildBrowserRequest, browserActions, decideBrowserStep, runBrowserFastPath } from "../src/browser.mjs";
 import { readRoutingCases } from "../src/receipts.mjs";
 import { routeRequest } from "../src/route.mjs";
+import { buildExecutionReceipt, buildRoutingCase } from "../src/receipts.mjs";
 function providerFor(choices) {
   let call = 0;
   return {
@@ -197,3 +198,25 @@ test("native select uses the optional host select executor", async () => {
   assert.equal(selectCalls, 1);
   assert.equal(executeCalls, 0);
 });
+
+test("screenshots stay out of browser text context and durable replay", async () => {
+  const imageData = "private-screen-pixels";
+  const request = buildBrowserRequest({
+    goal: "inspect the screen",
+    observation: {
+      screenshot: { format: "png", data_base64: imageData, width: 280, height: 280 },
+      targets: [{ id: "inspect", role: "button", name: "Inspect" }],
+    },
+  });
+  assert.equal(JSON.stringify(request.context).includes(imageData), false);
+  const decision = await routeRequest(request, { provider: "demo" });
+  assert.equal(decision.status, "fallback");
+  const recorded = buildRoutingCase({ request, decision });
+  assert.equal(JSON.stringify(recorded).includes(imageData), false);
+  assert.equal(recorded.request.assets_omitted, true);
+  assert.equal(typeof recorded.request.context.browser.observation.screenshot.sha256, "string");
+  const execution = buildExecutionReceipt({ request, decision, host: {
+    status: "completed", result: { observation: { screenshot: { format: "png", data_base64: imageData } } },
+  } });
+  assert.equal(JSON.stringify(execution).includes(imageData), false);
+});

package/test/computer.test.mjs

@@ -0,0 +1,195 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { test } from "node:test";
+import { decideComputerStep } from "../src/computer.mjs";
+import { PNG } from "./image-fixture.mjs";
+
+const SCREENSHOT = { format: "png", data_base64: PNG, width: 400, height: 300 };
+const SCROLL = { id: "scroll-down", operation: "scroll", args: { direction: "down", amount: 120 } };
+const CLICK = { id: "save", operation: "click", target: "the Save button", args: { button: "left" }, risk: "low" };
+const BOX = { left: 0.2, right: 0.6, top: 0.1, bottom: 0.5 };
+
+function input(overrides = {}) {
+  return { goal: "Inspect the application", observation: { screenshot: SCREENSHOT, visible_text: "Save", application: "Editor" }, actions: [SCROLL], actor_permissions: ["computer"], ...overrides };
+}
+
+function grounding(visibility = 0.99, confidence = 0.95, box = BOX) {
+  return { answers: { target_visible: { type: "noul", noul: visibility }, target_box: { type: "box", box, confidence } } };
+}
+
+function provider({ choice, confidence = 1, result = grounding(), error = false } = {}) {
+  return {
+    name: "computer-test", supportsImages: true, choices: [], groundings: [],
+    async decide(request) {
+      this.choices.push(request);
+      const selected = choice ?? request.candidates[0].id;
+      return { answers: { tool: { type: "choice", choice: selected, confidence, probabilities: { [selected]: confidence } } } };
+    },
+    async evaluate(request) {
+      this.groundings.push(request);
+      if (error) throw new Error("offline");
+      return result;
+    },
+  };
+}
+
+function assertFallback(result, type) {
+  assert.equal(result.action, null);
+  assert.equal(result.decision.status, "fallback");
+  if (type) assert.equal(result.decision.fallback.type, type);
+  assert.equal(result.decision.execution.enabled, false);
+  assert.equal(result.decision.correlation_id, result.decision.receipt.correlation_id);
+}
+
+test("computer advice is opt-in and an explicit disable overrides the environment", async () => {
+  const previous = process.env.JEV_COMPUTER_USE;
+  const engine = provider();
+  try {
+    delete process.env.JEV_COMPUTER_USE;
+    assertFallback(await decideComputerStep(input(), { provider: engine }), "computer_use_disabled");
+    process.env.JEV_COMPUTER_USE = "1";
+    assertFallback(await decideComputerStep(input(), { enabled: false, provider: engine }), "computer_use_disabled");
+    assert.equal(engine.choices.length, 0);
+    const enabled = await decideComputerStep(input(), { provider: engine });
+    assert.equal(enabled.decision.status, "selected");
+    assert.deepEqual(enabled.action.args, SCROLL.args);
+    assert.deepEqual(enabled.decision.execution, { enabled: false, status: "not_started" });
+  } finally {
+    if (previous === undefined) delete process.env.JEV_COMPUTER_USE;
+    else process.env.JEV_COMPUTER_USE = previous;
+  }
+});
+
+test("malformed screenshots and non-closed action arguments fail open before provider calls", async () => {
+  const engine = provider();
+  for (const candidate of [
+    input({ observation: {} }),
+    input({ observation: { screenshot: { ...SCREENSHOT, data_base64: "file:///screen.png" } } }),
+    input({ observation: { screenshot: { ...SCREENSHOT, data_base64: Buffer.from("not an image").toString("base64") } } }),
+    input({ observation: { screenshot: { ...SCREENSHOT, width: 800 } } }),
+    input({ actions: [{ ...SCROLL, operation: "shell", args: { command: "anything" } }] }),
+    input({ actions: [{ ...SCROLL, args: { direction: "down", command: "anything" } }] }),
+    input({ actions: [{ ...CLICK, target: undefined }] }),
+    input({ actions: [{ ...CLICK, args: { x: 400, y: 20 } }] }),
+    input({ actions: [{ ...CLICK, args: { x: 20 } }] }),
+    input({ actions: [SCROLL, SCROLL] }),
+    input({ actions: Array.from({ length: 33 }, (_, index) => ({ ...SCROLL, id: `scroll-${index}` })) }),
+    input({ policy: { min_target_visibility: Number.NaN } }),
+    input({ policy: { min_box_confidence: 0 } }),
+  ]) assertFallback(await decideComputerStep(candidate, { enabled: true, provider: engine }), "invalid_input");
+  assert.equal(engine.choices.length, 0);
+  assert.equal(engine.groundings.length, 0);
+});
+
+test("registry filters enforce explicit computer permission, availability, extra permissions, and risk", async () => {
+  const engine = provider();
+  for (const candidate of [
+    input({ actor_permissions: undefined }),
+    input({ actor_permissions: [] }),
+    input({ actions: [{ ...SCROLL, permissions: ["administrator"] }] }),
+    input({ actions: [{ ...SCROLL, available: false }] }),
+    input({ actions: [{ ...SCROLL, availability: { available: false, reason: "offline" } }] }),
+    input({ actions: [{ ...SCROLL, risk: "high" }], policy: { max_risk: "low" } }),
+  ]) {
+    const result = await decideComputerStep(candidate, { enabled: true, provider: engine });
+    assert.equal(result.decision.status, "no_decision");
+    assert.equal(result.action, null);
+    assert.equal(result.decision.execution.enabled, false);
+    assert.ok(result.decision.candidates.every((item) => item.filtered));
+  }
+  assert.equal(engine.choices.length, 0);
+});
+
+test("consequential actions require confirmation even at low risk", async () => {
+  for (const action of [
+    { id: "click", operation: "click", args: { x: 17, y: 29, button: "right" }, risk: "low" },
+    { id: "type", operation: "type", args: { text: "host supplied text" }, risk: "low" },
+    { id: "key", operation: "key", args: { keys: ["Control", "S"] }, risk: "low" },
+    { ...SCROLL, consequential: true },
+    { ...SCROLL, requires_confirmation: true },
+    { ...SCROLL, policy: { requires_confirmation: true } },
+  ]) {
+    const engine = provider();
+    const result = await decideComputerStep(input({ actions: [action], policy: { confirmation_risk_levels: [] } }), { enabled: true, provider: engine });
+    assert.equal(result.decision.status, "needs_confirmation");
+    assert.equal(result.decision.candidates[0].requires_confirmation, true);
+    assert.equal(result.decision.execution.enabled, false);
+    assert.equal(engine.groundings.length, 0);
+  }
+});
+
+test("only the chosen described click is grounded, once, and mapped to bound screenshot pixels", async () => {
+  const engine = provider({ choice: "save" });
+  const result = await decideComputerStep(input({ actions: [SCROLL, CLICK, { ...CLICK, id: "other", target: "Cancel button" }] }), { enabled: true, provider: engine });
+  assert.equal(result.decision.status, "needs_confirmation");
+  assert.deepEqual(result.action.args, { button: "left", x: 160, y: 90 });
+  assert.equal(result.action.target, CLICK.target);
+  assert.deepEqual(result.action.grounding, { box: BOX, visibility: 0.99, confidence: 0.95, coordinate_space: "screenshot_pixels" });
+  assert.equal(result.action.screenshot.sha256, `sha256:${createHash("sha256").update(Buffer.from(PNG, "base64")).digest("hex")}`);
+  assert.equal(result.action.screenshot.requires_current, true);
+  assert.equal(result.action.screenshot.width, 400);
+  assert.equal(result.action.screenshot.height, 300);
+  assert.equal(engine.groundings.length, 1);
+  assert.equal(JSON.stringify(result.request.context).includes(PNG), false);
+  assert.equal(JSON.stringify(result.decision).includes(PNG), false);
+  assert.deepEqual(CLICK.args, { button: "left" });
+});
+
+test("absent, malformed, and uncertain grounding yields no action while retaining routing correlation", async () => {
+  const badResults = [
+    [grounding(0), "target_not_visible"],
+    [grounding(0.899), "target_not_visible"],
+    [grounding(0.99, 0.799), "low_grounding_confidence"],
+    [grounding(Number.NaN), "malformed_grounding"],
+    [grounding(Infinity), "malformed_grounding"],
+    [grounding("0.99"), "malformed_grounding"],
+    [grounding(0.99, Infinity), "malformed_grounding"],
+    [{ answers: { target_box: grounding().answers.target_box } }, "malformed_grounding"],
+    [grounding(0.99, 0.95, { ...BOX, left: -0.1 }), "malformed_grounding"],
+    [grounding(0.99, 0.95, { ...BOX, right: 1.1 }), "malformed_grounding"],
+    [grounding(0.99, 0.95, { ...BOX, left: BOX.right }), "malformed_grounding"],
+    [grounding(0.99, 0.95, { ...BOX, top: BOX.bottom + 0.1 }), "malformed_grounding"],
+    [grounding(0.99, 0.95, { ...BOX, bottom: Number.NaN }), "malformed_grounding"],
+  ];
+  for (const [raw, type] of badResults) {
+    const result = await decideComputerStep(input({ actions: [CLICK] }), { enabled: true, provider: provider({ result: raw }) });
+    assertFallback(result, type);
+    assert.equal(result.decision.selected, CLICK.id);
+    assert.equal(result.decision.receipt.provider, "computer-test");
+    assert.equal(result.decision.receipt.candidate_count, 1);
+  }
+});
+
+test("host grounding thresholds are inclusive and edge pixels remain inside the screenshot", async () => {
+  const engine = provider({ result: grounding(0.95, 0.9, { left: 0.999, right: 1, top: 0.999, bottom: 1 }) });
+  const result = await decideComputerStep(input({ actions: [CLICK], policy: { min_target_visibility: 0.95, min_box_confidence: 0.9 } }), { enabled: true, provider: engine });
+  assert.deepEqual(result.action.args, { button: "left", x: 399, y: 299 });
+  const stricter = await decideComputerStep(input({ actions: [CLICK] }), { enabled: true, provider: engine, policy: { min_target_visibility: 0.96 } });
+  assertFallback(stricter, "target_not_visible");
+});
+
+test("route fallback, provider failures, unsupported images, and unknown selections never leak actions", async () => {
+  assertFallback(await decideComputerStep(input(), { enabled: true, provider: provider({ confidence: 0.2 }) }), "low_confidence");
+  assertFallback(await decideComputerStep(input(), { enabled: true, provider: provider({ choice: "not-host-supplied" }) }), "invalid_selection");
+  assertFallback(await decideComputerStep(input({ actions: [CLICK] }), { enabled: true, provider: provider({ error: true }) }), "provider_error");
+  const unavailable = provider();
+  unavailable.supportsImages = false;
+  assertFallback(await decideComputerStep(input(), { enabled: true, provider: unavailable }), "unsupported_images");
+  assert.equal(unavailable.choices.length, 0);
+  const noGrounding = provider();
+  delete noGrounding.evaluate;
+  assertFallback(await decideComputerStep(input({ actions: [CLICK] }), { enabled: true, provider: noGrounding }), "grounding_unavailable");
+  const broken = provider();
+  broken.decide = async () => { throw new Error("offline"); };
+  assertFallback(await decideComputerStep(input(), { enabled: true, provider: broken }), "provider_error");
+});
+
+test("handoff is advisory and nonselected described targets are never grounded", async () => {
+  const engine = provider({ choice: "handoff" });
+  const action = { id: "handoff", operation: "handoff", args: { reason: "Host planning needed" }, available: true, risk: "low" };
+  const result = await decideComputerStep(input({ actions: [CLICK, action] }), { enabled: true, provider: engine });
+  assert.equal(result.decision.status, "selected");
+  assert.equal(result.action.operation, "handoff");
+  assert.equal(engine.groundings.length, 0);
+  assert.equal(result.decision.execution.enabled, false);
+});

package/test/hermes-adapter.test.mjs

@@ -3,9 +3,10 @@ import { mkdtemp, readFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
 import test from "node:test";
 
-const adapter = new URL("../src/hermes-adapter.mjs", import.meta.url).pathname;
+const adapter = fileURLToPath(new URL("../src/hermes-adapter.mjs", import.meta.url));
 
 function run(lines, replayPath) {
   return new Promise((resolve, reject) => {

package/test/image-fixture.mjs

@@ -0,0 +1,2 @@
+// Complete 400x300 still RGB PNG, generated with zlib and valid chunk CRCs.
+export const PNG = "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEsCAIAAABi1XKVAAABdElEQVR4nO3BMQEAAADCoPVPbQwfoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+Bn+3AAEkkD9cAAAAAElFTkSuQmCC";