mcp-scraper 0.5.0 → 0.7.0

package/dist/worker-KJZ3ZN2N.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\nimport { runWithCostContext } from './cost-context.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const result = await runWithCostContext(\n      { op: opts.serpOnly ? 'serp' : 'paa', userId: Number(job.user_id) },\n      () => harvest({\n        ...opts,\n        kernelApiKey: browserServiceApiKey(),\n        headless: true,\n        format: 'json',\n        outputDir: '/tmp/paa-output-api',\n        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n      }),\n    )\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACIA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AAC/C;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,SAAS,MAAM;AAAA,MACnB,EAAE,IAAI,KAAK,WAAW,SAAS,OAAO,QAAQ,OAAO,IAAI,OAAO,EAAE;AAAA,MAClE,MAAM,QAAQ;AAAA,QACZ,GAAG;AAAA,QACH,cAAc,qBAAqB;AAAA,QACnC,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/docs/spec-browser-routing.md

@@ -1,108 +0,0 @@
-# Spec: Headless-first browser routing + smart proxy (scrape / SERP / Maps)
-
-## Context & goal
-We confirmed (Kernel docs + live probes) that: Kernel defaults to **headful (8 GB, $0.0001333/s)**; we never sent `headless`, so we've been paying **8×** on every browser op. **Headless (1 GB, $0.0000166667/s)** works for most scraping; **proxies of any type are FREE**; **idle is free**. Goal: route each operation to the cheapest mode that still works, escalate only when needed, and always get full (local-pack) SERP/Maps results.
-
-## Test evidence (decisions are grounded in these)
-- SERP headless + location proxy → 9 organic + 3 local-pack, 1 attempt, 11s. ✅
-- SERP headless **no proxy** → 9 organic, **0 local-pack**, 4s. ✅ organic only (local pack needs a regional proxy IP).
-- SERP headless + proxy occasionally CAPTCHAs on attempt 1, recovers on a fresh-IP retry. ✅
-- Website plain HTTP (no Kernel) scraped dallas.gov, 6s. ✅ (don't need Kernel for normal sites.)
-- Website Kernel headless + rotating proxy → 2 pages. ✅
-- **maps_search headless** (proxy and no-proxy) → 10 results, fast. ✅
-- **maps_place headless → "Target crashed" (×2)**; headful → OK (5 reviews, 23s). ❌ headless for place.
-
-## Routing policy (the decision table to implement)
-| Op (cost-context `op`) | Primary mode | Proxy | Escalation / fallback | Notes |
-|---|---|---|---|---|
-| `page_scrape` (extract_url) | plain HTTP (no Kernel) | none | **headless** Kernel on HTTP fail | concurrency 1 |
-| `extract_site` (bulk) | plain HTTP, concurrency = plan limit (3 on Starter) | none | **headless** Kernel rotating on block | already HTTP-first |
-| `harvest` (serp/paa) | **headless** | **rotating location** | fresh-IP retry ×N, then **headful** Kernel | retry already = new session+IP |
-| `maps_search` | **headless** | **rotating location** | fresh-IP retry ×N, then **headful** | |
-| `maps_place` | **HEADFUL (required)** | rotating location | n/a (already headful) | **headless crashes** |
-| `yt_channel`, `fb_search` | **headless** | existing | — | proven headless |
-| `fb_ad`, `instagram_*` | headful (unchanged) | existing | — | **untested headless — keep headful until probed** |
-| `browser_session` (interactive) | headful (required) | existing | — | Live View/replays need headful |
-
-Proxy rule for SERP + Maps: **always rotating location proxy**; if the user passes a `location`, resolve a region proxy but **still rotate to a new proxy per attempt**.
-
-## Implementation (exact changes)
-
-### 1. Shared headless-op config — `src/api/cost-rates.ts`
-Add:
-```ts
-export const HEADLESS_OPS = new Set<string>(['harvest', 'maps_search', 'yt_channel', 'fb_search'])
-```
-(Do NOT include `maps_place`, `browser_session`, `fb_ad`, `instagram_*`.) This is the single source of truth; expand only after a tool is probed headless-safe.
-
-### 2. BrowserDriver headless decision + per-attempt override — `src/driver/BrowserDriver.ts`
-- Import `HEADLESS_OPS` from `../api/cost-rates.js` (already imports `currentCostContext`).
-- Add to `DriverConfig` (`src/types.ts`): `headlessMode?: 'headless' | 'headful'`.
-- Replace the current inline block (the `HEADLESS_SAFE_OPS` set + `useHeadless`) with:
-```ts
-const ctx = currentCostContext()
-let useHeadless: boolean
-if (config.headlessMode === 'headful') useHeadless = false
-else if (config.headlessMode === 'headless') useHeadless = true
-else useHeadless = ctx?.forceHeadless === true
-  || (process.env.KERNEL_HEADLESS_DISABLED !== 'true' && !!ctx?.op && HEADLESS_OPS.has(ctx.op))
-this.kernelHeadlessSent = useHeadless ? true : null
-// create payload: ...(useHeadless ? { headless: true } : {})
-```
-`config.headlessMode` (set by the retry loop) wins over the op default, enabling headful escalation.
-
-### 3. Headful escalation for SERP/PAA — `src/harvest.ts`
-- Add a constant: `const HEADLESS_ATTEMPTS = MAX_ATTEMPTS` (i.e. all normal attempts headless), and extend the loop with ONE extra headful attempt.
-- In the attempt loop (`for (let i = 0; i < maxAttempts; i++)`, ~line 252), when building the per-attempt `DriverConfig` that is passed to `driver.launch(config)` (the config assembled around line 166/271), set:
-```ts
-const escalateHeadful = i >= maxAttempts   // last, extra attempt
-config.headlessMode = escalateHeadful ? 'headful' : 'headless'
-```
-- Bump the loop bound so there is a final headful attempt: iterate `maxAttempts + 1`, where the last iteration forces `headful`. Only run the headful attempt if prior attempts failed with CAPTCHA (`outcome === 'captcha'`); if a headless attempt already succeeded, break as today.
-
-### 4. Headful escalation for maps_search — `src/maps-search-rotation.ts`
-Mirror §3 in `runMapsSearchWithRotation` (~line 129): attempts 1..N headless (`headlessMode:'headless'`), final attempt `headlessMode:'headful'` if previous attempts failed. `maps_place` (`src/api/maps-routes.ts` `/place` → `MapsExtractor`) needs **no change** — its op is `maps_place`, absent from `HEADLESS_OPS`, so it stays headful automatically.
-
-### 5. Website Kernel fallback → headless — `src/api/kernel-fetch.ts`
-Pin the website-fallback browser to headless (proven safe, 8× cheaper). Change the create to default headless unless an explicit headful override is in context:
-```ts
-const ctx = currentCostContext()
-const useHeadless = ctx?.forceHeadless !== false   // default headless for page fetches
-const kb = await client.browsers.create({ stealth: true, ...(useHeadless ? { headless: true } : {}), timeout_seconds: 60 })
-```
-`rotating-proxy-crawl.ts` already passes `headless` explicitly (keep its `DEFAULT_HEADLESS = true`).
-
-### 6. extract_url auto-fallback stays on — `src/api/server.ts` (`/extract-url`)
-Confirm the handler always passes `browserServiceApiKey()` into `extractKpo` so `tryPlainFetch → fetchWithKernel` fallback runs without requiring `browserFallback:true` (the probe showed it already falls back). If it is gated, ungate it so concurrency-1 normal scrapes get the HTTP→headless-Kernel backup by default.
-
-### 7. Proxy defaults → rotating location for SERP — `src/schemas.ts`
-- Change `DEFAULT_PROXY_MODE` to `'location'` (maps is already `'location'`).
-- In the proxy resolver (`resolveKernelProxyId` / per-attempt proxy selection used by harvest + maps rotation): ensure each attempt requests a **new** proxy id even when a `location` is fixed (rotate within the region). If no `location` is supplied, `'location'` mode should fall back to rotating generic proxies rather than erroring — verify and patch the resolver.
-
-### 8. Telemetry — already in place
-`kernel_session_log.headless_sent` + `cost_probe_runs.mode` already record the tier. After this change, `costs`/`probe-report` will show the real (now mostly-headless) spend. No change needed beyond re-running probes.
-
-## Files changed (summary)
-- `src/api/cost-rates.ts` — add `HEADLESS_OPS`.
-- `src/types.ts` — add `headlessMode?` to `DriverConfig`.
-- `src/driver/BrowserDriver.ts` — headless decision honoring `config.headlessMode` + `HEADLESS_OPS`.
-- `src/harvest.ts` — headless attempts + final headful escalation.
-- `src/maps-search-rotation.ts` — same escalation for maps_search.
-- `src/api/kernel-fetch.ts` — default website fallback to headless.
-- `src/api/server.ts` — ensure extract_url passes kernel key for auto-fallback.
-- `src/schemas.ts` — `DEFAULT_PROXY_MODE = 'location'`; resolver rotates proxy per attempt.
-
-## Kill-switch / rollback
-`KERNEL_HEADLESS_DISABLED=true` (env) reverts ALL ops to headful instantly. Per-op revert = remove the op from `HEADLESS_OPS`.
-
-## Verification (run after implementing, before/after compare)
-1. `npm run build` green.
-2. Deploy to prod.
-3. `npx tsx scripts/cost-probe.ts --api-key sk_live_…` (default) and `--headless` — expect: serp/paa/maps_search/yt_channel/fb_search now headless by default; maps_place still headful; website ops HTTP/headless.
-4. `python3 .claude/skills/mcp-metrics/scripts/mcp_metrics.py probe-report` — confirm `mgn_hl` (now real) is positive/healthy and `try` counts show escalation only on CAPTCHA.
-5. `… costs 1` — confirm live spend drops toward the headless column.
-6. Owner pulls the Kernel invoice to confirm the per-second tier flipped to headless for the affected ops.
-
-## Open items to test BEFORE widening
-- Probe `fb_ad`, `instagram_profile`, `instagram_media`, `fb_video_transcribe` in headless before adding to `HEADLESS_OPS` (maps_place proved untested headless can crash).
-- Decide SERP pricing: 0.1 cr barely covers a slow proxy SERP even headless; fast no-proxy profits.

package/docs/spec-kernel-computer-controls.md

@@ -1,166 +0,0 @@
-# Spec — Tighten Kernel Computer Controls usage (human-like input on the hosted browser)
-
-## Correction to the original premise
-An earlier assessment claimed "we drive the Kernel browser over CDP with raw Playwright — we never use the Computer Controls API." **That was wrong** (a bad grep — searched raw HTTP paths + `page.mouse`, missed the SDK method names). Ground truth in `src/services/browser-agent/browser-agent-service.ts`:
-
-| Action | Implementation today | API used |
-|---|---|---|
-| screenshot | `k.browsers.computer.captureScreenshot` | ✅ Computer Controls |
-| click | `k.browsers.computer.clickMouse` | ✅ Computer Controls |
-| type | `k.browsers.computer.typeText` | ✅ Computer Controls |
-| scroll | `k.browsers.computer.scroll` | ✅ Computer Controls |
-| press | `k.browsers.computer.pressKey` | ✅ Computer Controls |
-| moveMouse | `k.browsers.computer.moveMouse({smooth:true})` | ✅ defined… but never called |
-| goto / read / locate / captureFanout | `playwrightChromium.connectOverCDP` | Playwright/CDP (DOM access — required, kept) |
-
-So the interactive agent's **input + screenshot already follow Kernel's API**. The DOM-read path uses Playwright-over-CDP because Computer Controls cannot read the DOM / element coordinates — that path stays (and is the same free Playwright code used off-Kernel).
-
-## Dual-path principle (the rule, stated)
-- **On Kernel (hosted browser):** drive *input* (move/click/type/scroll/press/screenshot) through **Computer Controls** (`k.browsers.computer.*`) — native OS input, no CDP input fingerprint, bezier curves, inter-keystroke delays. Read the DOM via Playwright-over-CDP (no Computer Controls equivalent).
-- **Off Kernel (free local/HTTP path, `BrowserDriver` non-hosted, future local mode):** keep Playwright. It's free and there's no Computer Controls layer to use.
-
-This spec changes only the Kernel input path. It does not touch Playwright scraping (`BrowserDriver`, reddit/serp/maps/fb deterministic tools) — their bottleneck is IP/challenge, not input cadence, and they barely click.
-
-## The two real gaps (verified)
-1. **Clicks teleport.** `click()` calls `clickMouse(x,y)` with no preceding move. `moveMouse({smooth:true})` exists (bezier) but is **called from nowhere** (`grep` confirms one definition, zero call sites). So every click is an instant cursor jump to the target — a behavioral bot signal Kernel's docs flag ("identical cursor paths, rapid mouse movements"). The bezier capability is built and unused.
-2. **Typing has no human cadence by default.** `typeText(id, text, delayMs?)` only passes `delay` if the *caller* supplies `body.delay` (`browser-agent-routes.ts:486`). The model never sends it → `delay` is `undefined` → instant uniform typing ("uniform typing speeds" — another flagged signal).
-
-## Changes
-
-### 1. `src/services/browser-agent/browser-agent-service.ts`
-
-**Add a constant + jitter helper near the top of the action section (after `client()`):**
-```ts
-const HUMAN_TYPE_DELAY_MS = 55
-function jitter(base: number, spread: number): number {
-  return Math.max(1, Math.round(base + (Math.random() * 2 - 1) * spread))
-}
-```
-
-**`click()` — move (bezier) to the target before clicking.** Replace the body:
-```ts
-export async function click(
-  runtimeSessionId: string,
-  x: number,
-  y: number,
-  opts: { button?: 'left' | 'right' | 'middle'; numClicks?: number } = {},
-): Promise<void> {
-  const k = client()
-  await k.browsers.computer.moveMouse(runtimeSessionId, { x, y, smooth: true })
-  await k.browsers.computer.clickMouse(runtimeSessionId, {
-    x,
-    y,
-    button: opts.button ?? 'left',
-    ...(opts.numClicks ? { num_clicks: opts.numClicks } : {}),
-  })
-}
-```
-(Optional optimization — collapse the two round-trips into one with `computer.batch` if the installed SDK exposes it: `await k.browsers.computer.batch(runtimeSessionId, { actions: [{ type: 'move_mouse', x, y, smooth: true }, { type: 'click_mouse', x, y, button: opts.button ?? 'left', num_clicks: opts.numClicks ?? 1 }] })`. Verify the SDK method/shape before using; if unavailable, keep the two sequential calls above.)
-
-**`typeText()` — default a jittered human delay when the caller gives none:**
-```ts
-export async function typeText(runtimeSessionId: string, text: string, delayMs?: number): Promise<void> {
-  const k = client()
-  const delay = typeof delayMs === 'number' ? delayMs : jitter(HUMAN_TYPE_DELAY_MS, 30)
-  await k.browsers.computer.typeText(runtimeSessionId, { text, delay })
-}
-```
-
-No route changes required — `browser-agent-routes.ts` already forwards `x/y`, `button`, `num_clicks`, and an optional `delay`; the new behavior is purely in the service defaults. The MCP tool schemas/descriptions need no change (the human-like behavior is automatic; callers still just pass x/y/text).
-
-### 2. No change to
-- `goto` / `readPage` / `locatePageTargets` / `captureFanout` — DOM access over CDP is required and free-path-shared. Leave as-is.
-- `BrowserDriver` and all deterministic scrapers (reddit, serp, maps, fb, instagram, youtube) — Playwright stays.
-- The free/local browser path — Playwright stays.
-
-## Verification
-1. `grep -n "moveMouse" src/services/browser-agent/browser-agent-service.ts` → now **2** hits (definition + the new call in `click`).
-2. `npm run build` clean.
-3. Deploy prod; open a session and drive it:
-   - `POST /agent/sessions {url:"https://example.com"}` → `POST /agent/sessions/:id/screenshot` (note an element's x/y) → `POST /agent/sessions/:id/click {x,y}` → screenshot again: the click lands (move+click path works).
-   - `POST /agent/sessions/:id/type {text:"hello world"}` (no `delay`) succeeds — typing now carries the default jittered cadence server-side.
-   - `DELETE /agent/sessions/:id`.
-4. Behavioral check on a detection-sensitive target (manual): drive a known bot-walled login form; clicks should approach via curve and typing should be paced (observable in the live-view / replay).
-
-## Release
-- One file changed (`browser-agent-service.ts`). The behavior ships server-side → live on Vercel deploy; **no npm republish required** (the stdio package is a thin client; tool schemas/descriptions are unchanged).
-- Commit on `main`: `feat(browser): human-like input on Kernel — bezier move-before-click + default typing cadence`.
-
----
-
-# Phase 2 — Move the DOM-read path to the Playwright Execution API (in-VM + Patchright)
-
-## Why (revised from "defer")
-The read path (`goto`, `readPage`, `locatePageTargets`, `captureFanout`) uses `playwrightChromium.connectOverCDP(cdpWsUrl)` from our serverless function — **outside** the Kernel VM. Kernel's recommended-practices table explicitly says *"Prefer our Playwright Execution API or Computer Controls API over self-hosted Playwright/Puppeteer,"* and the bot-detection guide states the Execution API *"runs in the same VM as the browser, ensuring headers, user-agent strings, and environment match. Kernel automatically applies Patchright to remove automation fingerprints, including headless indicators."* So external CDP is the residual detection signal, and this is the documented fix — not optional polish.
-
-## The API
-`POST /browsers/{id}/playwright/execute` — body `{ code: string, timeout_sec?: 1..300 }`. The `code` runs **in-VM** inside a function with `page`, `context`, `browser` in scope; `return` a value. Response `{ success, result, error, stdout, stderr }`. SDK call (verify exact name against installed `@onkernel/sdk`): `k.browsers.playwright.execute(runtimeSessionId, { code, timeout_sec })`.
-
-## Mechanic
-Our read functions already do their work via `page.evaluate(FN)` over a `connectOverCDP` page. The migration is **mechanical**: take the existing in-page extraction JS (the function bodies passed to `page.evaluate`) and ship them as the `code` string to `execute`, returning the same shape — but now running in-VM with Patchright instead of over external CDP.
-
-### `src/services/browser-agent/browser-agent-service.ts`
-Add a helper:
-```ts
-async function runInVm<T>(runtimeSessionId: string, code: string, timeoutSec = 30): Promise<T> {
-  const k = client()
-  const res = await k.browsers.playwright.execute(runtimeSessionId, { code, timeout_sec: timeoutSec })
-  if (!res.success) throw new Error(res.error || 'playwright/execute failed')
-  return res.result as T
-}
-```
-Then rewrite each read function to call `runInVm` with its existing extraction logic as a code string instead of `connectOverCDP`:
-- **`goto(runtimeSessionId, url)`** → `code`: `` await page.goto(${JSON.stringify(url)}, { waitUntil: 'domcontentloaded', timeout: 45000 }); return { url: page.url(), title: await page.title() }; ``  (signature changes from `cdpWsUrl` to `runtimeSessionId`).
-- **`readPage`** → move the current element/text-collection `page.evaluate` body into the `code` string; `return` the same `{ url, title, text, elements }` object.
-- **`locatePageTargets`** → same: the selector/text-bounds logic becomes the `code` body, `return` the bounds array.
-- **`captureFanout`** → port its capture logic; if it relies on response listeners over CDP, evaluate whether it can run in-VM or must stay on CDP (the fan-out hook listens to network events — this one may legitimately stay on CDP; verify before migrating).
-
-### Routes (`src/api/browser-agent-routes.ts`)
-The read/locate/goto routes currently pass `row.cdp_ws_url` to the service. Change those call sites to pass `row.runtime_session_id`. No other route logic changes.
-
-### Keep CDP only where required
-If `captureFanout` needs live network-event listeners (CDP-only), leave it on `connectOverCDP`. Everything else moves in-VM.
-
-## Dual-path still holds
-- **On Kernel:** input → Computer Controls (Phase 1); DOM read → Playwright Execution API (Phase 2). Zero external CDP except any network-listener edge case.
-- **Off Kernel (free/local):** unchanged local Playwright.
-
-## Verification (Phase 2)
-1. `grep -c connectOverCDP src/services/browser-agent/browser-agent-service.ts` → drops to 0 (or 1 if captureFanout stays).
-2. Build clean.
-3. Drive a session: `goto` → `read` → `locate` return the same shapes as before (regression check against current output).
-4. On a detection-sensitive page, confirm reads no longer attach an external CDP client (the session shows only in-VM execution + Computer Controls input).
-
-## Release (Phase 2)
-- Same file (+ route call-site edits). Server-side → ships on deploy; no npm republish.
-- Separate commit from Phase 1 so each is independently revertible: `feat(browser): read DOM via Kernel Playwright Execution API (in-VM, Patchright) instead of external CDP`.
-
-## Sequencing
-Ship **Phase 1 first** (tiny, pure win), validate, then **Phase 2** (mechanical but touches 3–4 functions + their routes; needs a regression pass on read/locate output shape). Don't bundle — Phase 2 has real surface area.
-
-## Still out of scope
-- Migrating the **deterministic scrapers** (`BrowserDriver` reddit/serp/maps/fb) from `connectOverCDP` to Playwright Execution API. Same benefit would apply, but their bottleneck was IP/challenge (solved) and they're a larger blast radius. Flag as Phase 3 if detection on those tools resurfaces.
-- **Batch beyond move+click** — only if a concrete need appears.
-
----
-
-# Other Kernel APIs — assessment + findings (2026-06-30)
-
-Reviewed the full Kernel API reference (browsers, playwright, computer-controls, filesystem, processes). Findings:
-
-## TESTED & REJECTED — browser curl for Reddit
-**Hypothesis:** `client.browsers.curl(id, {url})` (`POST /browsers/{id}/curl`) sends an HTTP request through Chrome's stack (TLS fingerprint + cookies + proxy), so curling `old.reddit/...json` through a residential-proxied browser would return the comment tree cheaply, no render.
-**Result: FAILED.** Both residential-proxied and no-proxy curl returned **HTTP 403 / Reddit block page** (no JSON). Reason: curl does NOT execute JavaScript, and Reddit's wall is a **JS challenge** that only a *rendered* browser solves. `reddit_thread` therefore MUST stay headful-render (the render is what beats the challenge; the ~$0.0009 cost is justified). **Do not re-attempt curl for Reddit.**
-**Curl is still valid** for genuine JSON/REST APIs with no JS challenge (inherits the browser's authenticated session + residential IP). Keep as a back-pocket tier; not a current build item.
-
-## REJECTED — do NOT run ffmpeg/media in the Kernel VM (cost-wrong)
-Initially flagged process-exec/filesystem for in-VM ffmpeg muxing. **Retracted — it's the wrong place cost-wise.** A Kernel browser VM bills $0.0001333/s (headful); muxing is pure CPU work unrelated to browsing, so running it there is the most expensive option. And we don't need it for product value:
-- **Transcription already runs outside Kernel** via fal.ai wizper, which takes an `audio_url` and fetches it itself (`media-transcription.ts`) — no download, no ffmpeg, no Kernel. Cheap and correct.
-- **Muxing** (`VideoMixer.ts`) is an optional convenience that gracefully degrades to returning separate track URLs when ffmpeg is absent.
-- The `instagram_media` ENOENT was the **Vercel serverless read-only filesystem** (`mkdir ~/Downloads/...`), NOT a Kernel gap. Fix = write to `/tmp` or return track URLs; if muxing must be reliable, run ffmpeg on a cheap non-serverless worker — never in Kernel.
-**Conclusion: keep all media transcode/transcription OUTSIDE Kernel.** Process-exec/filesystem APIs have no compelling MCP Scraper use today.
-
-## LOW / skip unless needed
-- computer-controls extras: `drag` (slider captchas), `clipboard` read/write, `set_cursor`, `get_cursor_position` — situational agent polish.
-- browsers: create/get/list/delete/update (already used), ad-hoc extension upload (niche), load-extensions.
-- filesystem watch/stream, process async/PTY/signals — only for long-running in-VM jobs.

package/docs/spec-mcp-tools.md

@@ -1,146 +0,0 @@
-# MCP Tool Spec — Current vs Proposed (every tool)
-
-Implementation blueprint. Each tool lists **current values** (verbatim from source) and **proposed values**, with exact `file:line` targets. Inventory: 46 registered tools — 25 core (`src/mcp/paa-mcp-server.ts`), 2 SERP-snapshot (`src/mcp/mcp-routes.ts`), 19 browser (`src/mcp/browser-agent-mcp-server.ts`).
-
-Source of truth for costs: `src/api/rates.ts` → `MC_COSTS` (MC_PER_CREDIT = 1000). Site prices already derive from this via `scripts/gen-rates.ts` (the rate catalog). Credit $-value: Starter $0.0015, Growth $0.00125, Scale $0.00104.
-
----
-
-## PART 0 — Cross-cutting changes (do these once; they touch many tools)
-
-### 0.1 Delete dead code: `MCP_TOOL_MANIFEST`
-- **File:** `src/mcp/mcp-tool-manifest.ts` (~39 KB). **Current:** exported, imported by **nothing** (never in any commit). **Proposed:** delete the file, OR (ambitious) wire its `tokenBudgetClass`/`producesHandlesFor` as the progressive-disclosure layer (see 0.3). Default recommendation: **delete**.
-
-### 0.2 Remove stale proxy boilerplate (server now self-heals proxy選択)
-The agent no longer needs to reason about proxy at all: `DEFAULT_PROXY_MODE='none'`, the dead `configured` proxy is neutralized, and harvest does a 2-strike location→none fallback. So the proxy prose is dead weight + factually wrong ("default configured = highest success").
-
-- **`src/mcp/mcp-tool-schemas.ts`** — the `proxyMode` / `proxyZip` / `debug` `.describe()` strings (lines 11–13, 134–136, 151–153, 861–863, 874–877; ~15 fields, ~580 tokens). **Current:** each says *"Default configured uses the service proxy … highest general success rate … none only for direct-network debugging."* **Proposed:** collapse each `proxyMode` describe to one line: `'Leave unset. The service auto-selects the most reliable routing (clean egress by default; city/ZIP residential only when a location is given, with automatic fallback). Set "location" only to force local-pack targeting.'` Drop the `proxyZip`/`debug` describe to a single short sentence each.
-- **Tool descriptions carrying the same prose** (rewrite, see PART 4): `harvest_paa`, `search_serp`, `capture_serp_snapshot`, `maps_search`, `directory_workflow`.
-
-### 0.3 Progressive disclosure (optional, high-leverage)
-- **Current:** all 46 tool descriptions load into agent context every session.
-- **Proposed:** advertise a **core** set; defer the rest behind a `tools/list` tier or a `workflow_suggest`-style index. Core = `harvest_paa, search_serp, extract_url, extract_site, map_site_urls, maps_search, maps_place_intel, youtube_harvest, youtube_transcribe, facebook_page_intel, instagram_profile_content, credits_info`. Deferred = all `workflow_*`, `rank_tracker_workflow`, `directory_workflow`, `capture_serp_*`, `audit_site`, `facebook_ad_*`, `browser_replay_*`, `browser_list_*`. Reuse the deleted manifest's `tokenBudgetClass` field as the tier key.
-
-### 0.4 Cost catalog — DONE this session
-`MC_COSTS` is the single source; `gen-rates.ts` regenerates `public/rates.generated.js` on build; all site prices read `window.MCP_RATES`. No further action except the cost changes in PART 1.
-
----
-
-## PART 1 — Cost table (all metered tools)
-
-`cr` = credits. "Margin" = at Starter $0.0015/cr vs measured marginal cost. Measured costs from the 2026-06-29 clean sweep.
-
-| Tool | MC_COSTS key | Current cr | Measured cost | Margin (Starter) | **Proposed cr** | Why |
-|---|---|---|---|---|---|---|
-| harvest_paa | `paa` | 0.3 / question | $0.00018/q | ~2.5× @10q, ↑ at scale | **0.3 / q** (keep) | set this session |
-| search_serp | `serp` | 0.1 / search | $0.00004 | 3.8× | **0.1** (keep) | cheap, fine |
-| extract_url | `page_scrape` | 0.1 / page | ~$0 (HTTP) | huge | **0.1** (keep) | |
-| extract_site | `page_scrape` | 0.1 / page | ~$0 (HTTP) | huge | **0.1** (keep) | |
-| map_site_urls | `url_map` | 2 / op | ~$0 (HTTP) | huge | **0.5** | overpriced; lower for goodwill |
-| audit_site | `page_scrape`×pages | 0.1 / page | ~$0 + LLM | high | **0.1/page** (keep) | LLM cost is small |
-| maps_search | `maps_search` | 2 / search | $0.00013 | 23× | **0.5** | 23× is needless; DataForSEO is $0.0006 |
-| maps_place_intel | `maps_place` + `maps_review` | 2 + 0.1/review | $0.0015 (headful) | **2.0× — under 4×** | **3 base + 0.2/review** | headful, can't cheapen; raise to clear 4× |
-| youtube_harvest | `yt_channel` | 0.2 / call | $0.00011 | ~6× | **0.2** (keep) + consider 0.01/video | flat ok at small N |
-| youtube_transcribe | `yt_transcription` | 0.2 / min | ~$0 (free captions) | huge | **0.2/min** (keep) | |
-| facebook_page_intel | `fb_ad` | 0.2 / call | $0.00010 | ~5× | **0.2** (keep) | set this session |
-| facebook_ad_search | `fb_search` | 0.1 / search | soft-blocks (refunds) | n/a | **0.1** (keep) | only failing tool; see PART 2 |
-| facebook_ad_transcribe | `fb_transcribe` | 0.2 / min | ~$0 | high | **0.2/min** (keep) | |
-| facebook_video_transcribe | `fb_transcribe` | 0.2 / min | browser + fal | medium | **0.2/min** (keep) | |
-| instagram_profile_content | `instagram_profile` | 0.2 / scan | $0.00015 | ~5× | **0.2** (keep) | set this session |
-| instagram_media_download | `instagram_media` | 0.2 / post | browser | ~5× | **0.2** (keep) | |
-| instagram (transcribe) | `instagram_transcribe` | 0.05 | fal | high | **0.05** (keep) | |
-| capture_serp_snapshot | (serp path) | ~0.1 | browser | ok | **0.1** (align to serp) | |
-| directory_workflow | maps_search × cities | 2 × cities | maps×N | inherits maps | **0.5 × cities** | follows maps_search drop |
-| browser_* (19 tools) | `browser_minute` | 12 / min | ~$0.008/min headful (idle free) | ~2.25× active | **12/min** (keep) | set this session |
-| workflow_* / rank_tracker | per underlying call | varies | varies | — | **bill underlying** (keep) | planning tools = free |
-
-**Net cost changes to make:** `maps_search` 2 → **0.5**, `url_map` 2 → **0.5**, `maps_place` base 2 → **3**, `maps_review` 0.1 → **0.2**. All in `src/api/rates.ts` `MC_COSTS`. Everything else stays as set this session.
-
----
-
-## PART 2 — Per-tool spec (current → proposed)
-
-For each: cost (PART 1), key params (current default/cap → proposed), behavior, description action. `file:line` = `src/mcp/paa-mcp-server.ts` unless noted.
-
-### Research / SERP
-**harvest_paa** (174) — params: `maxQuestions` default 30 / max 200 (schemas.ts:7); `proxyMode` default none. Behavior: headful PAA, `none` egress, 2-strike fallback, **partial-return on timeout** (softDeadlineMs). **Proposed:** keep params; **rewrite description** (PART 4) to drop proxy prose + note "returns partial results if it hits the time budget." Cap guidance already good.
-
-**search_serp** (182) — `pages` default 1 / max 2. **Proposed:** rewrite description (drop proxy prose). Keep params.
-
-**capture_serp_snapshot / capture_serp_page_snapshots** (`mcp-routes.ts` 73, 81) — **Proposed:** rewrite `capture_serp_snapshot` description (drop proxy prose). These are Phoenix-product paths → candidates for **deferred** tier (0.3).
-
-### Web extraction
-**extract_url** (190) — params: `screenshot`, `extractBranding`, `downloadMedia`, `depositToVault` (all default false). **Proposed:** no change; description is good (selection-oriented).
-
-**map_site_urls** (198) — `maxUrls` max 10000, >500 → CSV file. **Proposed:** cost 2→0.5; keep caps.
-
-**extract_site** (206) — `maxPages` max 10000, >25 → folder mode. `rotateProxies` optional (schemas.ts:39 — this proxy hint is **legitimate**, keep it). **Proposed:** no cost change (0.1/page); keep description.
-
-**audit_site** (214) — `maxPages` cap. **Proposed:** no change; **deferred** tier candidate.
-
-### YouTube
-**youtube_harvest** (222) — `maxVideos` default 50 / max 500 (schemas.ts:57). **Proposed:** keep 0.2 flat; **note** in cost model that 500-video harvests are the context/cost outlier — consider per-video 0.01 later. Keep description.
-
-**youtube_transcribe** (230) — **Proposed:** no change.
-
-### Facebook
-**facebook_page_intel** (238) — `maxAds` default 50 / max 200 (schemas.ts:71). **Proposed:** keep 0.2.
-**facebook_ad_search** (246) — `maxResults` default 10 / max 20. **THE ONE BROKEN TOOL:** soft-blocks (FB rejects egress IP for ad-library search; refunds correctly). **Proposed:** (a) try a clean residential/BYO proxy specifically for this path; (b) if unreliable, mark description "best-effort" and keep auto-refund. Needs its own investigation task.
-**facebook_ad_transcribe** (254) / **facebook_video_transcribe** (262) — **Proposed:** no change.
-
-### Instagram
-**instagram_profile_content** (270) — `maxItems` default 50 / max **2000**, `maxScrolls` 10/250. **Proposed:** keep cost 0.2; **keep maxItems cap (no evidence it needs lowering)**.
-**instagram_media_download** (278) — `includeTranscript` default false (adds cost). **Proposed:** no change.
-
-### Maps
-**maps_place_intel** (286) — `includeReviews` false, `maxReviews` default 50 / max **500**. Behavior: **headful** (headless crashes). **Proposed:** cost **3 base + 0.2/review**; **keep maxReviews cap (no evidence it needs lowering)** — customer pays per review. Keep description.
-**maps_search** (294) — `maxResults` default 10 / max 50; `proxyMode` default **location** (DEFAULT_MAPS_PROXY_MODE). **Proposed:** cost 2→**0.5**; **rewrite description** to drop the "omit proxyMode for residential" prose (server handles it); keep location default but note auto-fallback. ⚠️ Maps still uses `location` proxy → watch for the same flagged-pool CAPTCHA; if it appears, flip to `none` like SERP.
-
-### Workflows / planning (no web, no/low credits)
-**directory_workflow** (302) — `state` TN, `minPopulation` 100000, `maxCities` 25/100, `maxResultsPerCity` 50, `concurrency` 5/5. **Proposed:** cost follows maps_search (0.5×cities); rewrite description to drop proxy prose; **deferred** tier.
-**workflow_list / workflow_suggest / workflow_run / workflow_step / workflow_status / workflow_artifact_read** (310–356) — planning/orchestration, no credits. **Proposed:** no change; **deferred** tier (except keep `workflow_suggest` discoverable as the router).
-**rank_tracker_workflow** (358) — local planning, no credits. **Proposed:** no change; **deferred** tier.
-**credits_info** (366) — read-only, server-driven prices. **Proposed:** no change; **core** tier.
-
-### Browser (19 tools, `browser-agent-mcp-server.ts`)
-`browser_profile_connect, browser_profile_list, browser_open, browser_screenshot, browser_read, browser_locate, browser_goto, browser_click, browser_type, browser_scroll, browser_press, browser_replay_start, browser_replay_stop, browser_list_replays, browser_replay_download, browser_replay_mark, browser_replay_annotate, browser_close, browser_list_sessions`.
-- **Cost:** all session-metered via `browser_minute` = **12 cr/min** (idle free — Standby). Accurate post-change.
-- **Params:** `timeout_seconds` default 600 (browser-agent-tool-schemas.ts:24) — describe already correct ("idles into zero-cost standby … longer timeout is cheap").
-- **Proposed:** **core** = `browser_open, browser_screenshot, browser_read, browser_goto, browser_click, browser_type, browser_scroll, browser_press, browser_close`. **deferred** = all `browser_replay_*`, `browser_list_*`, `browser_profile_*`. No description/cost changes needed.
-
----
-
-## PART 3 — Param caps: NO CHANGES
-
-Earlier drafts proposed lowering `maps_place_intel maxReviews` (500) and `instagram_profile_content maxItems` (2000). **Retracted — there is no evidence either cap causes a problem.** No measured failure, timeout, or cost issue at the high end. The customer pays per unit, and long jobs are handled by partial-return/timeout budgets (already done for PAA), not by capping. **All caps stay as-is.** Only lower a cap if we *observe* a specific failure mode at the high end.
-
----
-
-## PART 4 — Description rewrites (exact old → new)
-
-Remove proxy prose; keep selection guidance. Targets in `src/mcp/paa-mcp-server.ts` (+ mcp-routes.ts).
-
-**harvest_paa (176)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence."* → replace with: *"Routing is automatic — leave proxyMode unset. If a deep harvest exceeds the time budget it returns the questions gathered so far (partial), billed per returned question."*
-
-**search_serp (184)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."* → *"Routing is automatic — leave proxyMode unset; set location only to force local-pack targeting."*
-
-**maps_search (296)** — strike: *"Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting."* → *"Routing is automatic and location-aware — leave proxyMode unset; pass proxyZip only to force a specific ZIP."*
-
-**directory_workflow (304)** — strike the sentence *"Maps workflows default to location-targeted proxying … use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting."* → *"Routing is automatic per city."*
-
-**capture_serp_snapshot (`mcp-routes.ts:73`)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."* → *"Routing is automatic — leave proxyMode unset."*
-
----
-
-## PART 5 — Execution order, files, release
-
-1. **Costs** — `src/api/rates.ts` `MC_COSTS`: `maps_search` 2000→**500**, `url_map` 2000→**500**, `maps_place` 2000→**3000**, `maps_review` 100→**200**.
-2. **Proxy describe trim** — `mcp-tool-schemas.ts` proxyMode/proxyZip/debug fields (0.2).
-3. **Description rewrites** — PART 4 (`paa-mcp-server.ts`, `mcp-routes.ts`).
-4. **Delete** `src/mcp/mcp-tool-manifest.ts` (0.1).
-5. **(Optional, separate PR)** progressive-disclosure tier (0.3).
-6. **Build** (`npm run build` — regenerates rate catalog + bundles), **deploy Vercel prod** (covers REST billing + remote-MCP descriptions + site prices).
-7. **`npm publish`** — REQUIRED this time: tool descriptions are baked into the stdio package, so npm-installed clients need the new version to see trimmed/accurate descriptions. Bump version in the 3 release files first (per release workflow), vercel prod before publish. (No param caps changed.)
-9. **Open task:** `facebook_ad_search` soft-block (clean/BYO proxy or best-effort label).
-
-**Why npm matters here (unlike the rate change):** rates/logic run server-side and reach thin clients automatically, but **tool descriptions and input-schema caps are advertised by the MCP package itself** → republish required for stdio installs; remote-MCP users get it on deploy.

package/docs/spec-server-instructions.md

@@ -1,94 +0,0 @@
-# Spec — server-instructions.ts routing-map rewrite
-
-Implements the approved improvements from the MCP-design audit + reviewer feedback. **One file changes:** `src/mcp/server-instructions.ts` (the `SERVER_INSTRUCTIONS` string — the only content loaded into the model's context before it fetches any tool schema). No other source files. Requires a build + Vercel deploy (remote-MCP gets it immediately) + npm republish (the string ships in the stdio package).
-
-## Approved improvements (the full list)
-1. **Add `reddit_thread` to the map; stop routing Reddit to the browser.** Current line 15 says "Anything without a dedicated tool (e.g. Reddit…) -> browser_* agent" — **stale**, we shipped `reddit_thread` in 0.3.25. A model reading the current map will drive a browser for Reddit instead of calling the dedicated tool.
-2. **Make discovery→action ID seams explicit and schema-accurate** (reviewer #1). Encode, per chain, what the discovery tool *returns* and what the action tool *takes* — using the real params, not inferred ones.
-3. **Workflow triggers + boundary** (reviewer #2): plain-language "when to reach for a workflow vs hand-chaining," and state `query_fanout_workflow`'s browser prerequisite.
-4. **Reorder the no-dedicated-tool branch to check login FIRST** (reviewer #3), so a logged-in site isn't short-circuited into `browser_open` without `browser_profile_connect`.
-5. **Keep** the `rotateProxies` NOTE (reviewer #4 — defensible cross-cutting hint).
-
-Not in this spec (explicitly out): annotations (audited → correct as-is, not a bug), server namespace rename (breaking, deferred decision), tool-count tiering (soft, not needed). `deep_research_workflow` removal already shipped.
-
-## Verified schema facts backing each seam (source of truth)
-| Discovery tool | Returns (output schema) | Action tool | Takes (input schema) | Seam |
-|---|---|---|---|---|
-| `map_site_urls` / `search_serp` | `urls` / `organicResults[].url` | `extract_url` | `url` | url → url |
-| `search_serp` | `organicResults[].url` (incl. reddit.com) | `reddit_thread` | `url` | reddit url → reddit_thread |
-| `maps_search` | `results[].{name, placeUrl, cid}` | `maps_place_intel` | **`businessName` + `location`** (NOT an id) | name → name; **independent, callable directly** |
-| `youtube_harvest` | `videos[].videoId` | `youtube_transcribe` | `videoId` OR `url` | videoId → videoId |
-| `facebook_ad_search` | `advertisers[].{pageId, libraryId}` | `facebook_page_intel` | `pageId` / `libraryId` / `query` | id → id |
-| `facebook_page_intel` | `ads[].videoUrl` | `facebook_ad_transcribe` | `videoUrl` | videoUrl → videoUrl |
-| (organic FB url) | — | `facebook_video_transcribe` | `url` (FB reel/video/post) | url direct |
-| `instagram_profile_content` | `post`/`reel`/`tv` url arrays | `instagram_media_download` | `url` | url → url |
-| (must precede) `browser_open` on chatgpt.com/claude.ai | — | `query_fanout_workflow` | runs against the open AI session | browser-session prerequisite |
-
-`facebook_ad_transcribe` does NOT consume anything from `facebook_ad_search` — the videoUrl comes from `facebook_page_intel`. `maps_place_intel` does NOT consume a `place_id` — it takes a business name + location.
-
-## Exact change
-Replace the entire `SERVER_INSTRUCTIONS` template literal in `src/mcp/server-instructions.ts` (lines 1–29) with:
-
-```ts
-export const SERVER_INSTRUCTIONS = `
-This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
-this routing map, then load its schema before calling. Where a tool needs a value another tool
-produces, the seam is noted so you can chain them.
-
-ROUTING
-- One web page -> extract_url (takes a url). Whole site (crawl + SEO report) -> extract_site (takes a
-  url). Just the URL list -> map_site_urls (takes a url). map_site_urls and search_serp return urls you
-  can feed straight into extract_url.
-- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail ->
-  harvest_paa. search_serp returns organicResults[].url (often including reddit.com threads) to feed
-  extract_url or reddit_thread.
-- Google Maps: find places -> maps_search (returns results[] with name, placeUrl, cid). One place
-  deep-dive + reviews -> maps_place_intel (takes businessName + location, NOT an id — call it directly
-  with a name from a maps_search result or from the user).
-- YouTube: find or list videos -> youtube_harvest (returns videos[].videoId). Transcribe one ->
-  youtube_transcribe (takes a videoId from youtube_harvest, or a url).
-- Facebook: find advertisers -> facebook_ad_search (returns advertisers[] with pageId, libraryId). Get
-  one advertiser's ads -> facebook_page_intel (takes pageId, libraryId, or query; returns ads[].videoUrl).
-  Transcribe an ad video -> facebook_ad_transcribe (takes a videoUrl from facebook_page_intel).
-  Transcribe an organic Facebook reel/video/post -> facebook_video_transcribe (takes the Facebook url
-  directly).
-- Instagram: profile inventory -> instagram_profile_content (returns post/reel/tv urls). One post or
-  reel -> instagram_media_download (takes a url from instagram_profile_content, or a url the user gives).
-- Reddit: a reddit.com thread/post URL -> reddit_thread. It returns the post plus its comment tree and
-  handles Reddit's bot wall itself (no login needed). Find threads first with search_serp.
-- No dedicated tool (an arbitrary site, or a logged-in dashboard) -> the browser_* agent. FIRST decide
-  whether the site needs a login:
-  - Needs a login (ChatGPT, Claude, any account) -> save it first: browser_profile_connect returns an
-    mcpscraper.dev sign-in link for the user; one profile holds MANY logins (call it again with the same
-    profile + a new domain to add accounts). Poll browser_profile_list until AUTHENTICATED, then
-    browser_open with that profile. browser_profile_list also shows what a profile is connected to.
-  - No login -> browser_open, then navigate/read.
-
-WORKFLOWS (multi-step orchestrations — prefer these over hand-chaining primitives when the whole job is one of these)
-- Build a local business directory for a niche across a state/region -> directory_workflow.
-- Stand up a rank-tracking blueprint (database + cron + ingestion plan) -> rank_tracker_workflow.
-- Find which sources an AI answer cites for a query (AEO) -> query_fanout_workflow (open a browser
-  session on chatgpt.com or claude.ai FIRST, then run it against that session).
-
-NOTES
-- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
-  It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
-- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
-  cheapest tool that answers the question (plain search/extract before browser agents).
-- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
-  for full detail rather than expecting the whole payload inline.
-\`.trim()
-```
-
-## Build / deploy / release
-1. Edit the one file above.
-2. `npm run build` (typecheck + bundle; regenerates rate catalog — no functional change here).
-3. `vercel deploy --prod --yes` → remote-MCP clients get the new routing map immediately.
-4. **Bump 0.3.25 → 0.3.26** in the 3 version files (`package.json`, `package-lock.json`, `src/version.ts`) and `npm publish` — `SERVER_INSTRUCTIONS` is compiled into the stdio package, so npm-installed clients need the new version to see the corrected map.
-5. Commit on `main` (now the source of truth): `fix(mcp): routing map — add reddit_thread, explicit chaining seams, workflow triggers, login-first fallback`.
-
-## Verification
-- `grep -c reddit_thread src/mcp/server-instructions.ts` → ≥1 (was 0).
-- `grep -c "without a dedicated tool" src/mcp/server-instructions.ts` → 0 (old Reddit-to-browser line gone).
-- After deploy, the live `/mcp` server-instructions block reflects the new map (inspect via an MCP client's server info, or the SERVER_INSTRUCTIONS in the published tarball).
-- Behavioral smoke (manual, in an MCP client): "get the comments on <reddit thread url>" routes to `reddit_thread`, not `browser_open`.