effing-use 0.1.1 → 0.2.1

package/src/browser/refs.ts

@@ -1,4 +1,13 @@
 import type { Page, Locator } from "playwright";
+import {
+  getFingerprint,
+  buildFingerprint,
+  chooseRebindIndex,
+  RAW_FP_SCRIPT,
+  type Fingerprint,
+  type RawFp,
+  type RebindCandidate,
+} from "./identity.js";
 
 export class EngineError extends Error {
   code: string;
@@ -30,17 +39,52 @@ export function resolveRef(page: Page, ref: string): Locator {
 export async function resolveLocator(
   page: Page,
   target: string,
+  sessionId?: string,
 ): Promise<Locator> {
   const t = target.trim();
-  // 0. eNN ref
+  // 0. eNN ref — with identity validation when sessionId is known
   if (/^e\d+$/i.test(t)) {
     const loc = resolveRef(page, t);
-    if ((await loc.count()) > 0) return loc;
-    throw new EngineError(
-      "E_NOT_FOUND",
-      `No element matches "${t}".`,
-      "Call browser_observe kind=snapshot first to refresh refs, then use a fresh e-ref.",
-    );
+    if ((await loc.count()) === 0) {
+      throw new EngineError(
+        "E_NOT_FOUND",
+        `No element matches "${t}".`,
+        "Call browser_observe kind=snapshot first to refresh refs, then use a fresh e-ref.",
+      );
+    }
+    if (sessionId) {
+      const expected = getFingerprint(sessionId, t);
+      if (expected) {
+        // Validate against the SAME extractor that registered the snapshot
+        const raws = (await page.evaluate(RAW_FP_SCRIPT).catch(() => null)) as
+          | RawFp[]
+          | null;
+        const idx = Number(t.slice(1));
+        const actual = raws && raws[idx] ? buildFingerprint(raws[idx]) : null;
+        if (actual) {
+          const exact =
+            expected.role === actual.role &&
+            expected.accessibleName === actual.accessibleName &&
+            expected.textHash === actual.textHash;
+          if (!exact) {
+            // Conservative rebind — ambiguous → E_STALE, never guess
+            const rebound = await findRebound(page, expected, raws);
+            if (rebound) {
+              // Return rebound locator; caller can check rebound flag via session
+              (page as unknown as Record<string, unknown>).__rebound = true;
+              return rebound;
+            }
+            throw new EngineError(
+              "E_STALE",
+              `Ref ${t} is stale — element changed.`,
+              "Call browser_observe kind=snapshot to refresh refs.",
+              { expected, actual },
+            );
+          }
+        }
+      }
+    }
+    return loc;
   }
   // 1. role= selector passes through
   if (t.startsWith("role=")) return page.locator(t);
@@ -73,3 +117,144 @@ export async function resolveLocator(
     "Call browser_observe kind=snapshot first, then use an e-ref, role= selector, or CSS selector.",
   );
 }
+
+/**
+ * Find where the element `expected` used to describe lives now.
+ * Uses the shared extractor + conservative chooser: only an UNAMBIGUOUS match
+ * is returned; zero or several equally-good candidates → null → E_STALE
+ * upstream (v0.2 bug: first-match rebind clicked toggle-all).
+ */
+async function findRebound(
+  page: Page,
+  expected: Fingerprint,
+  raws: RawFp[] | null,
+): Promise<Locator | null> {
+  const list =
+    raws ?? ((await page.evaluate(RAW_FP_SCRIPT).catch(() => [])) as RawFp[]);
+  const candidates: RebindCandidate[] = list.map((r, i) => ({
+    index: i,
+    fp: buildFingerprint(r),
+  }));
+  const pick = chooseRebindIndex(expected, candidates);
+  if (pick === null) return null;
+  return page.locator(INTERACTIVE_SELECTOR).nth(pick);
+}
+
+/**
+ * Resolve a portable, standalone selector for a target (plan §6.2 preference:
+ * id → name → ARIA label → stable data-* → placeholder → recorded text).
+ * Never returns nth-index or session-scoped e-refs; null when nothing unique
+ * can be found (caller falls back to the raw target).
+ */
+export async function stableSelectorFor(
+  page: Page,
+  target: string,
+  sessionId?: string,
+): Promise<string | null> {
+  try {
+    const loc = await resolveLocator(page, target, sessionId);
+    const cands = await loc
+      .first()
+      .evaluate((el) => {
+        const out: string[] = [];
+        const tag = el.tagName.toLowerCase();
+        const esc = (s: string) => {
+          const c = (
+            globalThis as unknown as {
+              CSS?: { escape?: (x: string) => string };
+            }
+          ).CSS;
+          return c && c.escape ? c.escape(s) : s;
+        };
+        const av = (s: string): string | null => {
+          const bs = String.fromCharCode(92);
+          if (s.indexOf('"') !== -1 || s.indexOf(bs) !== -1) return null;
+          return '"' + s + '"';
+        };
+        const uniq = (s: string): boolean => {
+          try {
+            return document.querySelectorAll(s).length === 1;
+          } catch {
+            return false;
+          }
+        };
+        // 1. stable id (skip framework-generated/dynamic ids)
+        const id = el.getAttribute("id");
+        if (
+          id &&
+          !/^(bits|radix|headlessui|react-aria|ember|react-|:)/.test(id) &&
+          !/[-_]\d+$/.test(id) &&
+          uniq("#" + esc(id))
+        ) {
+          out.push("#" + esc(id));
+        }
+        // 2. name attribute
+        const name = el.getAttribute("name");
+        if (name) {
+          const a = av(name);
+          if (a && uniq("[name=" + a + "]")) out.push("[name=" + a + "]");
+        }
+        // 3. ARIA label on the element itself
+        const aria = el.getAttribute("aria-label");
+        if (aria) {
+          const a = av(aria);
+          if (a && uniq(tag + "[aria-label=" + a + "]"))
+            out.push(tag + "[aria-label=" + a + "]");
+        }
+        // 4. stable data-* (test attrs preferred; state/animation attrs skipped)
+        const attrs = Array.prototype.slice.call(el.attributes) as Array<{
+          name: string;
+          value: string;
+        }>;
+        const dataAttrs = attrs.filter(
+          (a) =>
+            a.name.startsWith("data-") &&
+            !/^data-(state|orientation|index|selected|active|highlighted|value|radix|size|style|theme)/.test(
+              a.name,
+            ),
+        );
+        dataAttrs.sort((a, b) => {
+          const score = (n: string) =>
+            /^data-(test|testid|cy)/.test(n) ? 0 : 1;
+          return score(a.name) - score(b.name);
+        });
+        for (const a of dataAttrs) {
+          const v = av(a.value);
+          if (!v) continue;
+          const s = tag + "[" + a.name + "=" + v + "]";
+          if (uniq(s)) {
+            out.push(s);
+            break;
+          }
+        }
+        // 5. placeholder (form controls with no name/id)
+        const ph = el.getAttribute("placeholder");
+        if (ph) {
+          const a = av(ph);
+          if (a && uniq(tag + "[placeholder=" + a + "]"))
+            out.push(tag + "[placeholder=" + a + "]");
+        }
+        // 6. recorded text (Playwright exact text engine)
+        const text = (el.textContent || "")
+          .trim()
+          .replace(/\s+/g, " ")
+          .slice(0, 60);
+        if (text && (tag === "button" || tag === "a")) {
+          const a = av(text);
+          if (a) out.push("text=" + a);
+        }
+        return out;
+      })
+      .catch(() => [] as string[]);
+    for (const c of cands) {
+      const n = await page
+        .locator(c)
+        .count()
+        .catch(() => 0);
+      if (n === 1) return c;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}

package/src/browser/session.ts

@@ -13,6 +13,7 @@ interface SessionEntry {
   page: Page;
   consoleLogs: Array<{ type: string; text: string; at: string }>;
   networkLogs: Array<{ method: string; url: string; status: number }>;
+  mustObserve: boolean;
 }
 
 const sessions = new Map<string, SessionEntry>();
@@ -48,6 +49,7 @@ export async function getPage(
     page,
     consoleLogs: [],
     networkLogs: [],
+    mustObserve: false,
   };
   page.on("console", (msg) => {
     entry.consoleLogs.push({
@@ -91,6 +93,20 @@ export function getNetworkLogs(
   return sessions.get(sessionId)?.networkLogs ?? [];
 }
 
+export function getMustObserve(sessionId = "default"): boolean {
+  return sessions.get(sessionId)?.mustObserve ?? false;
+}
+
+export function setMustObserve(sessionId = "default", v: boolean): void {
+  const s = sessions.get(sessionId);
+  if (s) s.mustObserve = v;
+}
+
+export function clearMustObserve(sessionId = "default"): void {
+  const s = sessions.get(sessionId);
+  if (s) s.mustObserve = false;
+}
+
 export async function closeSession(sessionId = "default"): Promise<void> {
   const s = sessions.get(sessionId);
   if (s) {

package/src/browser/state.ts

@@ -0,0 +1,100 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { Config } from "../config.js";
+
+export type StateData = {
+  notes: string[];
+  lastActions: string[];
+};
+
+const stateCache = new Map<string, StateData>();
+
+function sanitizeSegment(s: string): string {
+  return s.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64) || "default";
+}
+
+function statePath(config: Config, sessionId: string): string {
+  return join(config.outputDir, "state", `${sanitizeSegment(sessionId)}.md`);
+}
+
+function getState(sessionId: string): StateData {
+  let s = stateCache.get(sessionId);
+  if (!s) {
+    s = { notes: [], lastActions: [] };
+    stateCache.set(sessionId, s);
+  }
+  return s;
+}
+
+export async function appendNote(
+  config: Config,
+  sessionId: string,
+  note: string,
+): Promise<void> {
+  const s = getState(sessionId);
+  const line = note.slice(0, 300);
+  s.notes.push(line);
+  const max = config.stateMaxLines ?? 40;
+  while (s.notes.length > max) s.notes.shift();
+  await persist(config, sessionId);
+}
+
+export async function appendAction(
+  config: Config,
+  sessionId: string,
+  entry: string,
+): Promise<void> {
+  const s = getState(sessionId);
+  s.lastActions.push(entry.slice(0, 200));
+  while (s.lastActions.length > 10) s.lastActions.shift();
+  await persist(config, sessionId);
+}
+
+async function persist(config: Config, sessionId: string): Promise<void> {
+  const s = getState(sessionId);
+  const path = statePath(config, sessionId);
+  await mkdir(join(config.outputDir, "state"), { recursive: true });
+  const content = [
+    `# state:${sessionId}`,
+    `## notes`,
+    ...s.notes.map((n) => `- ${n}`),
+    `## lastActions`,
+    ...s.lastActions.map((a) => `- ${a}`),
+  ].join("\n");
+  await writeFile(path, content, "utf-8").catch(() => undefined);
+}
+
+export async function readState(
+  config: Config,
+  sessionId: string,
+): Promise<{ notes: string; lastActions: string[]; path: string }> {
+  const s = getState(sessionId);
+  // Try to load from disk if cache empty
+  if (s.notes.length === 0 && s.lastActions.length === 0) {
+    try {
+      const raw = await readFile(statePath(config, sessionId), "utf-8");
+      const lines = raw.split("\n");
+      let section: "notes" | "actions" | null = null;
+      for (const l of lines) {
+        if (l.startsWith("## notes")) section = "notes";
+        else if (l.startsWith("## lastActions")) section = "actions";
+        else if (l.startsWith("- ") && section === "notes")
+          s.notes.push(l.slice(2));
+        else if (l.startsWith("- ") && section === "actions")
+          s.lastActions.push(l.slice(2));
+      }
+    } catch {
+      /* no file yet */
+    }
+  }
+  const notes = s.notes.join("\n").slice(0, 1500);
+  return {
+    notes,
+    lastActions: [...s.lastActions],
+    path: statePath(config, sessionId),
+  };
+}
+
+export function clearState(sessionId: string): void {
+  stateCache.delete(sessionId);
+}

package/src/cli.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env bun
+// CLI face — same engine via HTTP transport (model-agnostic).
+// Usage: effing-use observe --kind snapshot --mode delta --session dev
+//        effing-use act --action click --target e5 --expect 'url~/dashboard'
+//        effing-use extract --kind state
+
+const HELP = `effing-use CLI — same engine as MCP, over HTTP
+Usage:
+  effing-use observe --kind <snapshot|screenshot|url|title|console|network|tabs|focused> [--mode full|delta] [--scope <css>] [--session <id>]
+  effing-use act --action <action> [--target <ref>] [--value <val>] [--expect <expr>] [--approve] [--session <id>]
+  effing-use act --action batch --file steps.json
+  effing-use extract --kind <text|html|table|query|pdf|trace_start|trace_stop|state> [--selector <css>] [--session <id>]
+  effing-use --help
+
+Env: EFFING_USE_URL (default http://localhost:3123/mcp)
+`;
+
+function parseArgs(argv: string[]): Record<string, string | boolean> {
+  const out: Record<string, string | boolean> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--help" || a === "-h") out.help = true;
+    else if (a.startsWith("--")) {
+      const k = a.slice(2);
+      const v = argv[i + 1];
+      if (v && !v.startsWith("--")) {
+        out[k] = v;
+        i++;
+      } else out[k] = true;
+    }
+  }
+  return out;
+}
+
+async function main() {
+  const [, , verb, ...rest] = process.argv;
+  const args = parseArgs(rest);
+
+  if (!verb || args.help || verb === "--help" || verb === "-h") {
+    process.stdout.write(HELP);
+    process.exit(0);
+  }
+
+  const url = process.env.EFFING_USE_URL ?? "http://localhost:3123/mcp";
+  // For CLI we call the HTTP MCP endpoint via JSON-RPC
+  // Minimal client: POST tools/call
+  const toolMap: Record<string, string> = {
+    observe: "browser_observe",
+    act: "browser_act",
+    extract: "browser_extract",
+  };
+  const tool = toolMap[verb];
+  if (!tool) {
+    process.stderr.write(`Unknown verb "${verb}". Use observe|act|extract.\n`);
+    process.exit(1);
+  }
+
+  // Build arguments object for the tool
+  const toolArgs: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(args)) {
+    if (k === "help") continue;
+    // coerce
+    if (k === "approve") toolArgs[k] = v === true || v === "true";
+    else if (k === "limit") toolArgs[k] = Number(v);
+    else toolArgs[k] = v;
+  }
+  // Map CLI names to tool schema names
+  if (
+    toolArgs.kind === undefined &&
+    verb === "act" &&
+    toolArgs.action === undefined
+  ) {
+    // allow --kind alias? no
+  }
+  // handle --file for batch
+  if (toolArgs.file) {
+    try {
+      const raw = await Bun.file(String(toolArgs.file)).text();
+      const steps = JSON.parse(raw);
+      toolArgs.steps = Array.isArray(steps) ? steps : (steps.steps ?? []);
+    } catch (e) {
+      process.stderr.write(
+        `Failed to read --file: ${e instanceof Error ? e.message : String(e)}\n`,
+      );
+      process.exit(1);
+    }
+    delete toolArgs.file;
+  }
+  // sessionId alias
+  if (toolArgs.session && !toolArgs.sessionId) {
+    toolArgs.sessionId = toolArgs.session;
+    delete toolArgs.session;
+  }
+
+  const body = JSON.stringify({
+    jsonrpc: "2.0",
+    id: 1,
+    method: "tools/call",
+    params: { name: tool, arguments: toolArgs },
+  });
+
+  let res: Response;
+  try {
+    res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Accept: "application/json, text/event-stream",
+      },
+      body,
+    });
+  } catch {
+    process.stderr.write(
+      JSON.stringify({
+        ok: false,
+        code: "E_NO_SERVER",
+        message: `Cannot reach ${url}`,
+        hint: "start: bun src/http.ts",
+      }) + "\n",
+    );
+    process.exit(2);
+  }
+
+  if (!res.ok) {
+    // Try to parse SSE or JSON
+    const text = await res.text().catch(() => "");
+    // SSE format: data: {...}
+    const m = /data:\s*(\{.*\})/.exec(text);
+    const payload = m ? m[1] : text;
+    try {
+      const j = JSON.parse(payload);
+      const content =
+        j.result?.content?.[0]?.text ?? j.error?.message ?? payload;
+      // content is often JSON string of the tool result
+      try {
+        const inner = JSON.parse(content);
+        process.stdout.write(JSON.stringify(inner) + "\n");
+        process.exit(inner.ok === false ? 1 : 0);
+      } catch {
+        process.stdout.write(content + "\n");
+        process.exit(0);
+      }
+    } catch {
+      process.stderr.write(text + "\n");
+      process.exit(1);
+    }
+  }
+
+  const text = await res.text();
+  // Handle SSE stream
+  const lines = text.split("\n").filter((l) => l.startsWith("data:"));
+  const last = lines[lines.length - 1]?.slice(5).trim() ?? text;
+  try {
+    const j = JSON.parse(last);
+    const content = j.result?.content?.[0]?.text ?? last;
+    try {
+      const inner = JSON.parse(content);
+      process.stdout.write(JSON.stringify(inner) + "\n");
+      process.exit(inner.ok === false ? 1 : 0);
+    } catch {
+      process.stdout.write(content + "\n");
+      process.exit(0);
+    }
+  } catch {
+    process.stdout.write(text + "\n");
+    process.exit(0);
+  }
+}
+
+main().catch((e) => {
+  process.stderr.write(String(e) + "\n");
+  process.exit(1);
+});

package/src/config.ts

@@ -15,6 +15,11 @@ const ConfigSchema = v.object({
   outputDir: v.optional(v.string(), ".browser-use"),
   outputMaxChars: v.optional(v.number(), 4000),
   allowEval: v.optional(v.boolean(), false),
+  deltaDefault: v.optional(v.boolean(), true),
+  effectMaxChars: v.optional(v.number(), 800),
+  stateMaxLines: v.optional(v.number(), 40),
+  recordRedact: v.optional(v.boolean(), true),
+  effingUseUrl: v.optional(v.string(), "http://localhost:3123/mcp"),
 });
 
 export type Config = v.InferOutput<typeof ConfigSchema>;
@@ -28,5 +33,10 @@ export function loadConfig(): Config {
     outputDir: process.env.OUTPUT_DIR ?? ".browser-use",
     outputMaxChars: Number(process.env.OUTPUT_MAX_CHARS ?? 4000),
     allowEval: process.env.ALLOW_EVAL === "true",
+    deltaDefault: process.env.DELTA_DEFAULT !== "false",
+    effectMaxChars: Number(process.env.EFFECT_MAX_CHARS ?? 800),
+    stateMaxLines: Number(process.env.STATE_MAX_LINES ?? 40),
+    recordRedact: process.env.RECORD_REDACT !== "false",
+    effingUseUrl: process.env.EFFING_USE_URL ?? "http://localhost:3123/mcp",
   });
 }

package/src/http.ts

@@ -11,6 +11,7 @@ const transport = new HttpTransport(server, { path: "/mcp" });
 
 Bun.serve({
   port,
+  hostname: "0.0.0.0",
   // Bun closes idle connections after 10s by default (idleTimeout), and the
   // timer applies even while a response is being streamed. The MCP
   // Streamable-HTTP SSE notification stream sits idle between server->client

package/src/tools/act.ts

@@ -3,7 +3,7 @@ import { tool } from "tmcp/utils";
 import * as v from "valibot";
 import { loadConfig } from "../config.js";
 import { getPage, getContext } from "../browser/session.js";
-import { doAct, doBatch, doGoal, type ActAction } from "../browser/engine.js";
+import { doAct, doBatch, type ActAction } from "../browser/engine.js";
 import { EngineError } from "../browser/refs.js";
 
 export const ACT_ACTIONS = [
@@ -34,6 +34,11 @@ export const ACT_ACTIONS = [
   "tab_select",
   "tab_close",
   "resize",
+  "note",
+  "record_start",
+  "record_stop",
+  "compile",
+  "replay",
 ] as const;
 
 const StepSchema = v.object({
@@ -53,6 +58,8 @@ export const actTool = defineTool(
       value: v.optional(v.string()),
       sessionId: v.optional(v.string()),
       steps: v.optional(v.array(StepSchema)),
+      expect: v.optional(v.string()),
+      approve: v.optional(v.boolean()),
     }),
     annotations: {
       readOnlyHint: false,
@@ -61,13 +68,13 @@ export const actTool = defineTool(
       openWorldHint: true,
     },
   },
-  async ({ action, target, value, sessionId, steps }) => {
+  async ({ action, target, value, sessionId, steps, expect, approve }) => {
     const config = loadConfig();
     const sid = sessionId ?? "default";
     try {
       const page = await getPage(config, sid);
       const context = await getContext(config, sid);
-      const opts = { context, sessionId: sid };
+      const opts = { context, sessionId: sid, expect, approve };
       if (action === "batch") {
         if (!steps || steps.length === 0)
           return tool.text(
@@ -94,10 +101,6 @@ export const actTool = defineTool(
           JSON.stringify({ ok: true, action, url, title, ...result }),
         );
       }
-      if (action === "goal") {
-        const result = await doGoal(page, config, value ?? target ?? "", opts);
-        return tool.text(JSON.stringify({ ok: true, action, ...result }));
-      }
       const result = await doAct(
         page,
         config,

package/src/tools/extract.ts

@@ -20,6 +20,7 @@ export const extractTool = defineTool(
         "pdf",
         "trace_start",
         "trace_stop",
+        "state",
       ]),
       selector: v.optional(v.string()),
       mode: v.optional(v.string()),

package/src/tools/observe.ts

@@ -26,6 +26,8 @@ export const observeTool = defineTool(
       limit: v.optional(v.number()),
       sessionId: v.optional(v.string()),
       inline: v.optional(v.boolean()),
+      mode: v.optional(v.picklist(["full", "delta"])),
+      scope: v.optional(v.string()),
     }),
     annotations: {
       readOnlyHint: true,
@@ -34,7 +36,7 @@ export const observeTool = defineTool(
       openWorldHint: false,
     },
   },
-  async ({ kind, target, limit, sessionId }) => {
+  async ({ kind, target, limit, sessionId, mode, scope }) => {
     const config = loadConfig();
     const sid = sessionId ?? "default";
     try {
@@ -43,6 +45,8 @@ export const observeTool = defineTool(
       const result = await doObserve(page, config, kind, target, limit, {
         context,
         sessionId: sid,
+        mode: mode as "full" | "delta" | undefined,
+        scope,
       });
       return tool.text(JSON.stringify({ ok: true, kind, ...result }));
     } catch (e) {