mcp-scraper 0.2.6 → 0.2.7

package/README.md

@@ -114,8 +114,9 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
 ### Browser-agent tools
 
 - `browser_open` — open a live cloud browser session. Returns a `session_id`, a human `watch_url`, and the raw `live_view_url` when available.
-- `browser_screenshot` — capture a screenshot plus visible text and clickable element coordinates.
-- `browser_read` — read the current page text and elements without an image.
+- `browser_screenshot` — capture a screenshot plus visible text and clickable element center coordinates and DOM bounds.
+- `browser_read` — read the current page text and elements with center coordinates and DOM bounds, without an image.
+- `browser_locate` — locate exact visible DOM elements or text ranges and return screenshot-pixel bounds.
 - `browser_goto`
 - `browser_click`
 - `browser_type`
@@ -125,9 +126,13 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
 - `browser_replay_stop` — stop a replay. Returns the final `view_url` and `download_url`.
 - `browser_list_replays` — list replay videos for a session.
 - `browser_replay_download` — download and save the replay MP4 locally under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
+- `browser_replay_mark` — while recording, locate a DOM target and return a replay-timed annotation object.
+- `browser_replay_annotate` — download a replay MP4, render timed boxes, circles, underlines, arrows, and labels using annotation objects from `browser_replay_mark` or exact bounds from `browser_locate`, and save a new annotated MP4 locally.
 - `browser_close`
 - `browser_list_sessions`
 
+For accurate annotated videos, do not guess annotation times from a script. Start the replay, navigate until each target is visible and stable, call `browser_replay_mark` for each callout, then stop the replay and pass the returned annotations to `browser_replay_annotate` with the returned `source_width` and `source_height`.
+
 For US local SERP tools (`harvest_paa` and `search_serp`), keep `proxyMode` at the default `location` unless you are debugging. Location mode uses fresh residential proxy IDs across retries and treats CAPTCHA, proxy tunnel failure, and wrong-location evidence as retryable before returning.
 
 The MCPB bundle and `mcp-scraper-combined` expose both sections through one local MCP server. The split `mcp-scraper` entrypoint exposes only the web-intelligence tools, and the split `browser-agent` entrypoint exposes only the browser-agent tools.

package/dist/bin/api-server.cjs

@@ -17082,7 +17082,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.2.6";
+    PACKAGE_VERSION = "0.2.7";
   }
 });
 
@@ -18351,18 +18351,26 @@ async function recordAction(input) {
 async function recordReplayStart(input) {
   const db = getDb();
   await db.execute({
-    sql: `INSERT INTO browser_agent_replays (replay_id, session_id, view_url, label)
-          VALUES (?, ?, ?, ?)`,
-    args: [input.replayId, input.sessionId, input.viewUrl, input.label]
+    sql: `INSERT INTO browser_agent_replays (replay_id, session_id, view_url, label, started_at)
+          VALUES (?, ?, ?, ?, ?)`,
+    args: [input.replayId, input.sessionId, input.viewUrl, input.label, (/* @__PURE__ */ new Date()).toISOString()]
   });
 }
 async function recordReplayStop(replayId, viewUrl) {
   const db = getDb();
   await db.execute({
-    sql: `UPDATE browser_agent_replays SET stopped_at = datetime('now'), view_url = COALESCE(?, view_url) WHERE replay_id = ?`,
-    args: [viewUrl, replayId]
+    sql: `UPDATE browser_agent_replays SET stopped_at = ?, view_url = COALESCE(?, view_url) WHERE replay_id = ?`,
+    args: [(/* @__PURE__ */ new Date()).toISOString(), viewUrl, replayId]
   });
 }
+async function getActiveReplayRow(sessionId) {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT * FROM browser_agent_replays WHERE session_id = ? AND stopped_at IS NULL ORDER BY started_at DESC LIMIT 1`,
+    args: [sessionId]
+  });
+  return res.rows[0] ?? null;
+}
 async function listReplayRows(sessionId) {
   const db = getDb();
   const res = await db.execute({
@@ -18482,7 +18490,13 @@ async function readPage(cdpWsUrl) {
           role: el2.getAttribute("role") || el2.tagName.toLowerCase(),
           name,
           x: Math.round(r.left + r.width / 2),
-          y: Math.round(r.top + r.height / 2)
+          y: Math.round(r.top + r.height / 2),
+          left: Math.round(r.left),
+          top: Math.round(r.top),
+          width: Math.round(r.width),
+          height: Math.round(r.height),
+          right: Math.round(r.right),
+          bottom: Math.round(r.bottom)
         });
       }
       const text = (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").trim().slice(0, 6e3);
@@ -18492,13 +18506,168 @@ async function readPage(cdpWsUrl) {
       url,
       title,
       text: data.text,
-      elements: data.els.map((e, i) => ({ ref: i + 1, role: e.role, name: e.name, x: e.x, y: e.y }))
+      elements: data.els.map((e, i) => ({
+        ref: i + 1,
+        role: e.role,
+        name: e.name,
+        x: e.x,
+        y: e.y,
+        left: e.left,
+        top: e.top,
+        width: e.width,
+        height: e.height,
+        right: e.right,
+        bottom: e.bottom
+      }))
     };
   } finally {
     await browser.close().catch(() => {
     });
   }
 }
+async function locatePageTargets(cdpWsUrl, targets) {
+  const browser = await import_playwright4.chromium.connectOverCDP(cdpWsUrl);
+  try {
+    const context = browser.contexts()[0] ?? await browser.newContext();
+    const page = context.pages()[0] ?? await context.newPage();
+    const url = page.url();
+    const title = await page.title().catch(() => "");
+    await page.evaluate(() => {
+      ;
+      globalThis.__name = (target) => target;
+    }).catch(() => {
+    });
+    const data = await page.evaluate((rawTargets) => {
+      const normalize = (value) => value.replace(/\s+/g, " ").trim();
+      const isVisibleRect = (r) => r.width >= 1 && r.height >= 1 && r.bottom >= 0 && r.right >= 0 && r.top <= window.innerHeight && r.left <= window.innerWidth;
+      const roleOf = (el2) => el2.getAttribute("role") || el2.tagName.toLowerCase();
+      const nameOf = (el2) => {
+        const e = el2;
+        return normalize(
+          e.getAttribute("aria-label") || e.placeholder || e.innerText || e.getAttribute("value") || e.getAttribute("title") || ""
+        ).slice(0, 160);
+      };
+      const textOf = (el2) => normalize(el2.innerText || el2.textContent || "").slice(0, 500);
+      const unionRects = (rects) => {
+        const visible = rects.filter(isVisibleRect);
+        if (!visible.length) return null;
+        const left = Math.min(...visible.map((r) => r.left));
+        const top = Math.min(...visible.map((r) => r.top));
+        const right = Math.max(...visible.map((r) => r.right));
+        const bottom = Math.max(...visible.map((r) => r.bottom));
+        return {
+          left,
+          top,
+          right,
+          bottom,
+          width: right - left,
+          height: bottom - top,
+          x: left,
+          y: top,
+          toJSON: () => ({})
+        };
+      };
+      const elementPayload = (el2, rect) => ({
+        role: roleOf(el2),
+        tag: el2.tagName.toLowerCase(),
+        name: nameOf(el2),
+        text: textOf(el2),
+        x: Math.round(rect.left + rect.width / 2),
+        y: Math.round(rect.top + rect.height / 2),
+        left: Math.round(rect.left),
+        top: Math.round(rect.top),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        right: Math.round(rect.right),
+        bottom: Math.round(rect.bottom)
+      });
+      const selectorMatches = (selector) => {
+        const nodes = Array.from(document.querySelectorAll(selector));
+        return nodes.map((el2) => ({ el: el2, rect: el2.getBoundingClientRect() })).filter((item) => isVisibleRect(item.rect));
+      };
+      const textMatches = (needle, match) => {
+        const wanted = normalize(needle);
+        const wantedLower = wanted.toLowerCase();
+        if (!wantedLower) return [];
+        const matches = [];
+        const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+        let node = walker.nextNode();
+        while (node) {
+          const raw = node.textContent || "";
+          const normalizedRaw = normalize(raw);
+          const rawLower = raw.toLowerCase();
+          const normalizedLower = normalizedRaw.toLowerCase();
+          const ok = match === "exact" ? normalizedLower === wantedLower : normalizedLower.includes(wantedLower);
+          if (ok) {
+            const start = match === "exact" ? 0 : rawLower.indexOf(wantedLower);
+            const parent = node.parentElement;
+            if (parent && start >= 0) {
+              try {
+                const range = document.createRange();
+                range.setStart(node, Math.min(start, raw.length));
+                range.setEnd(node, Math.min(raw.length, start + needle.length));
+                const rect = unionRects(Array.from(range.getClientRects()));
+                range.detach();
+                if (rect) matches.push({ el: parent, rect });
+              } catch {
+                const rect = parent.getBoundingClientRect();
+                if (isVisibleRect(rect)) matches.push({ el: parent, rect });
+              }
+            }
+          }
+          node = walker.nextNode();
+        }
+        if (matches.length) return matches;
+        const fallback = Array.from(document.querySelectorAll("body *")).map((el2) => ({ el: el2, rect: el2.getBoundingClientRect(), text: textOf(el2) })).filter((item) => isVisibleRect(item.rect)).filter((item) => {
+          const value = item.text.toLowerCase();
+          return match === "exact" ? value === wantedLower : value.includes(wantedLower);
+        }).sort((a, b) => a.rect.width * a.rect.height - b.rect.width * b.rect.height);
+        return fallback.map(({ el: el2, rect }) => ({ el: el2, rect }));
+      };
+      const viewport = {
+        width: window.innerWidth,
+        height: window.innerHeight,
+        devicePixelRatio: window.devicePixelRatio || 1
+      };
+      const results = rawTargets.map((target) => {
+        const index = Math.max(0, Math.floor(target.index ?? 0));
+        const match = target.match === "exact" ? "exact" : "contains";
+        try {
+          const selector = target.selector?.trim();
+          const text = target.text?.trim();
+          const candidates = selector ? selectorMatches(selector) : text ? textMatches(text, match) : [];
+          const selected = candidates[index];
+          return {
+            name: target.name ?? null,
+            selector: selector || null,
+            text: text || null,
+            match,
+            index,
+            found: Boolean(selected),
+            element: selected ? elementPayload(selected.el, selected.rect) : null,
+            ...selected ? {} : { error: "target not found in current viewport" }
+          };
+        } catch (err) {
+          return {
+            name: target.name ?? null,
+            selector: target.selector ?? null,
+            text: target.text ?? null,
+            match,
+            index,
+            found: false,
+            element: null,
+            error: err instanceof Error ? err.message : String(err)
+          };
+        }
+      });
+      return { viewport, targets: results };
+    }, targets);
+    return { url, title, viewport: data.viewport, targets: data.targets };
+  } finally {
+    await browser.close().catch(() => {
+    });
+  }
+}
 async function replayStart(runtimeSessionId) {
   const k = client();
   const res = await k.browsers.replays.start(runtimeSessionId);
@@ -18561,6 +18730,22 @@ function failure(err) {
   const msg = err instanceof Error ? err.message : String(err);
   return { error: sanitizeVendorName(msg) };
 }
+function parseAgentDate(value) {
+  if (!value) return null;
+  const parsed = Date.parse(value);
+  if (Number.isFinite(parsed)) return parsed;
+  const legacyParsed = Date.parse(`${value.replace(" ", "T")}Z`);
+  return Number.isFinite(legacyParsed) ? legacyParsed : null;
+}
+function replayClock(row) {
+  if (!row) return null;
+  const startedAtMs = parseAgentDate(row.started_at);
+  return {
+    replay_id: row.replay_id,
+    started_at: row.started_at,
+    replay_elapsed_seconds: startedAtMs == null ? null : Math.max(0, (Date.now() - startedAtMs) / 1e3)
+  };
+}
 function replayDownloadUrl(sessionId, replayId) {
   return `/agent/sessions/${encodeURIComponent(sessionId)}/replays/${encodeURIComponent(replayId)}/download`;
 }
@@ -18671,13 +18856,15 @@ function buildBrowserAgentRoutes() {
       }
       await charge(row.id, user.id, t0);
       await recordAction({ sessionId: row.id, type: "screenshot", params: null, ok: true });
+      const activeReplay = await getActiveReplayRow(row.id);
       return c.json({
         image_base64: shot.base64,
         mime_type: shot.mimeType,
         url: page?.url ?? null,
         title: page?.title ?? null,
         elements: page?.elements ?? [],
-        text: page?.text ?? null
+        text: page?.text ?? null,
+        replay: replayClock(activeReplay)
       });
     } catch (err) {
       await charge(row.id, user.id, t0);
@@ -18694,7 +18881,8 @@ function buildBrowserAgentRoutes() {
       const page = await readPage(row.cdp_ws_url);
       await charge(row.id, user.id, t0);
       await recordAction({ sessionId: row.id, type: "read", params: null, ok: true });
-      return c.json(page);
+      const activeReplay = await getActiveReplayRow(row.id);
+      return c.json({ ...page, replay: replayClock(activeReplay) });
     } catch (err) {
       await charge(row.id, user.id, t0);
       return c.json(failure(err), 502);
@@ -18723,6 +18911,27 @@ function buildBrowserAgentRoutes() {
       return c.json(failure(err), 502);
     }
   });
+  app2.post("/sessions/:id/locate", async (c) => {
+    const user = c.get("user");
+    const row = await loadOpenSession(c.req.param("id"), user.id);
+    if (!row) return c.json({ error: "not found" }, 404);
+    const body = await c.req.json().catch(() => ({}));
+    const rawTargets = Array.isArray(body.targets) ? body.targets : [];
+    const targets = rawTargets.filter((value) => value && typeof value === "object").slice(0, 20).map((value) => value);
+    if (!targets.length) return c.json({ error: "targets is required" }, 400);
+    const t0 = Date.now();
+    try {
+      const result = await locatePageTargets(row.cdp_ws_url, targets);
+      await charge(row.id, user.id, t0);
+      await recordAction({ sessionId: row.id, type: "locate", params: { targets: targets.length }, ok: true });
+      const activeReplay = await getActiveReplayRow(row.id);
+      return c.json({ ...result, replay: replayClock(activeReplay) });
+    } catch (err) {
+      await charge(row.id, user.id, t0);
+      await recordAction({ sessionId: row.id, type: "locate", params: { targets: targets.length }, ok: false, error: String(err) });
+      return c.json(failure(err), 502);
+    }
+  });
   app2.post("/sessions/:id/type", async (c) => {
     const user = c.get("user");
     const row = await loadOpenSession(c.req.param("id"), user.id);