mcp-scraper 0.3.24 → 0.3.26

package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js}

@@ -4,14 +4,15 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-JTWQGPKF.js";
+} from "./chunk-WYCER2HW.js";
 import {
   harvest
-} from "./chunk-IGOEAKC7.js";
+} from "./chunk-G7PQ64LM.js";
 import {
   browserServiceApiKey
-} from "./chunk-ZAP7MYVO.js";
-import "./chunk-LFATOGDF.js";
+} from "./chunk-3PRO376E.js";
+import "./chunk-K443GQY5.js";
+import "./chunk-BP27CZ5Q.js";
 import {
   claimPendingJob,
   completeJob,
@@ -19,7 +20,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-3KYRG7O7.js";
+} from "./chunk-TRNSXR46.js";
 
 // src/api/webhook.ts
 async function deliverWebhook(url, payload, retries = 3) {
@@ -124,4 +125,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-UM5UM6O5.js.map
+//# sourceMappingURL=worker-UC6D2756.js.map

package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const result = await harvest({\n      ...opts,\n      kernelApiKey: browserServiceApiKey(),\n      headless: true,\n      format: 'json',\n      outputDir: '/tmp/paa-output-api',\n      onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n    })\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACGA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AAC/C;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,GAAG;AAAA,MACH,cAAc,qBAAqB;AAAA,MACnC,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,IAClE,CAAC;AACD,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}
+{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const result = await harvest({\n      ...opts,\n      kernelApiKey: browserServiceApiKey(),\n      headless: true,\n      format: 'json',\n      outputDir: '/tmp/paa-output-api',\n      onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n    })\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACGA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AAC/C;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,GAAG;AAAA,MACH,cAAc,qBAAqB;AAAA,MACnC,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,IAClE,CAAC;AACD,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/docs/kernel-proxy-future-enhancements.md

@@ -0,0 +1,80 @@
+# Kernel / proxy retry — future enhancement notes
+
+**Status:** ideas only — NOT implemented, NO code from here is in the tree.
+**Origin:** salvaged from an abandoned automated branch `codex/kernel-proxy-retry-cleanup`
+(a stale `~/Projects/mcp-scraper` clone, since deleted). That branch was evaluated against the
+current Desktop code and found **superseded** — Desktop independently built the same retry +
+error-classification machinery and went further (fresh disposable proxy per attempt, state
+escalation, rollout to maps/Facebook/site-crawl). **Do not `git merge` that branch** — it
+diverged onto an older base and a merge would regress the current proxy layer.
+
+The two items below are the *only* things Desktop genuinely lacked. They are documented here as
+"if we want to build X, here's how" — to be rebuilt from scratch onto current code, never pasted.
+
+---
+
+## A. SERP-timeout salvage (best candidate)
+
+**Problem it solves:** when a Google page load hits a navigation timeout (e.g.
+`domcontentloaded` slow) but the page is *actually* a usable `/search` results page with no
+CAPTCHA, the current code throws and triggers a wasted retry. We can instead detect "good page,
+just slow" and continue.
+
+**Where:** `src/driver/BrowserDriver.ts`, in the `goto` flow.
+
+**How to build:**
+1. Add a predicate `currentGoogleSerpIsUsable(page)` — returns true when the current URL is a
+   Google `/search` page, results containers are present, and no CAPTCHA/consent interstitial is
+   detected.
+2. In `goto`, catch the navigation-timeout case (an `isNavigationTimeout(err)` guard). Before
+   rethrowing, if `currentGoogleSerpIsUsable(page)` is true, log a "salvaged slow SERP" telemetry
+   line and proceed instead of throwing.
+3. Keep the throw for every other timeout (non-SERP pages, CAPTCHA present, empty page).
+
+**Tools it improves:** `search_serp`, `harvest_paa`, `capture_serp_*` — fewer discarded captures
+and fewer retry credits burned on pages that were fine.
+
+**Watch out for:** only salvage when results are genuinely present — a timed-out blank/consent
+page must still fail so the normal retry/rotation kicks in.
+
+---
+
+## B. Opt-in proxy connectivity-check bypass (situational)
+
+**Problem it solves:** the browser-service residential proxies run an internal connectivity ping
+(an S3-style reachability check) before use. When that ping falsely fails, proxy creation is
+rejected even though the proxy would work — surfacing as `proxy_unavailable`.
+
+**Where:** `src/kernel-proxy-resolver.ts`, in the residential-proxy creation path.
+
+**How to build:**
+1. Gate behind an env flag `KERNEL_PROXY_BYPASS_CONNECTIVITY_CHECK` (default off).
+2. When set, pass a `bypass_hosts` / skip-connectivity-check option into the proxy-create call so
+   the internal ping is skipped.
+3. Add a `shouldRetryWithoutBypass` fallback: if a bypassed proxy then fails for real, retry once
+   with the check re-enabled so we don't mask a genuinely dead proxy.
+
+**Build ONLY if** we actually observe proxy-create failures from the connectivity ping in
+production. Otherwise leave it — it's complexity with no payoff until that failure mode is real.
+
+---
+
+## C. Minor niceties (low priority, build only if convenient)
+
+- **Per-attempt hard timeout:** a fresh `AbortController` per retry attempt with its own
+  `perAttemptTimeoutMs` (~120s), chained to the parent signal. Desktop bounds attempt *count* but
+  has no per-attempt hard cap.
+- **Typed `KernelProxyUnavailableError`:** Desktop detects proxy-unavailable via regex on the
+  message; a dedicated error class would be cleaner to branch on.
+- **Request-level retry knobs:** expose `maxAttempts`, `retryOnCaptcha`, `retryOnProxyFailure`,
+  `allowLocationFallback` as request schema fields instead of hardcoded constants, for per-call
+  tuning / debugging.
+
+---
+
+## What NOT to take
+
+Everything else from the codex branch — the retry loop, the `proxy_tunnel_failed` /
+`proxy_unavailable` / `location_mismatch` error classification, `LocationMismatchError`, the
+503/retryable mappings — is **already in Desktop** (`src/harvest.ts`, `src/errors.ts`,
+`src/api/harvest-problems.ts`) and in a more advanced form. Do not reintroduce it.

package/docs/spec-browser-routing.md

@@ -0,0 +1,108 @@
+# Spec: Headless-first browser routing + smart proxy (scrape / SERP / Maps)
+
+## Context & goal
+We confirmed (Kernel docs + live probes) that: Kernel defaults to **headful (8 GB, $0.0001333/s)**; we never sent `headless`, so we've been paying **8×** on every browser op. **Headless (1 GB, $0.0000166667/s)** works for most scraping; **proxies of any type are FREE**; **idle is free**. Goal: route each operation to the cheapest mode that still works, escalate only when needed, and always get full (local-pack) SERP/Maps results.
+
+## Test evidence (decisions are grounded in these)
+- SERP headless + location proxy → 9 organic + 3 local-pack, 1 attempt, 11s. ✅
+- SERP headless **no proxy** → 9 organic, **0 local-pack**, 4s. ✅ organic only (local pack needs a regional proxy IP).
+- SERP headless + proxy occasionally CAPTCHAs on attempt 1, recovers on a fresh-IP retry. ✅
+- Website plain HTTP (no Kernel) scraped dallas.gov, 6s. ✅ (don't need Kernel for normal sites.)
+- Website Kernel headless + rotating proxy → 2 pages. ✅
+- **maps_search headless** (proxy and no-proxy) → 10 results, fast. ✅
+- **maps_place headless → "Target crashed" (×2)**; headful → OK (5 reviews, 23s). ❌ headless for place.
+
+## Routing policy (the decision table to implement)
+| Op (cost-context `op`) | Primary mode | Proxy | Escalation / fallback | Notes |
+|---|---|---|---|---|
+| `page_scrape` (extract_url) | plain HTTP (no Kernel) | none | **headless** Kernel on HTTP fail | concurrency 1 |
+| `extract_site` (bulk) | plain HTTP, concurrency = plan limit (3 on Starter) | none | **headless** Kernel rotating on block | already HTTP-first |
+| `harvest` (serp/paa) | **headless** | **rotating location** | fresh-IP retry ×N, then **headful** Kernel | retry already = new session+IP |
+| `maps_search` | **headless** | **rotating location** | fresh-IP retry ×N, then **headful** | |
+| `maps_place` | **HEADFUL (required)** | rotating location | n/a (already headful) | **headless crashes** |
+| `yt_channel`, `fb_search` | **headless** | existing | — | proven headless |
+| `fb_ad`, `instagram_*` | headful (unchanged) | existing | — | **untested headless — keep headful until probed** |
+| `browser_session` (interactive) | headful (required) | existing | — | Live View/replays need headful |
+
+Proxy rule for SERP + Maps: **always rotating location proxy**; if the user passes a `location`, resolve a region proxy but **still rotate to a new proxy per attempt**.
+
+## Implementation (exact changes)
+
+### 1. Shared headless-op config — `src/api/cost-rates.ts`
+Add:
+```ts
+export const HEADLESS_OPS = new Set<string>(['harvest', 'maps_search', 'yt_channel', 'fb_search'])
+```
+(Do NOT include `maps_place`, `browser_session`, `fb_ad`, `instagram_*`.) This is the single source of truth; expand only after a tool is probed headless-safe.
+
+### 2. BrowserDriver headless decision + per-attempt override — `src/driver/BrowserDriver.ts`
+- Import `HEADLESS_OPS` from `../api/cost-rates.js` (already imports `currentCostContext`).
+- Add to `DriverConfig` (`src/types.ts`): `headlessMode?: 'headless' | 'headful'`.
+- Replace the current inline block (the `HEADLESS_SAFE_OPS` set + `useHeadless`) with:
+```ts
+const ctx = currentCostContext()
+let useHeadless: boolean
+if (config.headlessMode === 'headful') useHeadless = false
+else if (config.headlessMode === 'headless') useHeadless = true
+else useHeadless = ctx?.forceHeadless === true
+  || (process.env.KERNEL_HEADLESS_DISABLED !== 'true' && !!ctx?.op && HEADLESS_OPS.has(ctx.op))
+this.kernelHeadlessSent = useHeadless ? true : null
+// create payload: ...(useHeadless ? { headless: true } : {})
+```
+`config.headlessMode` (set by the retry loop) wins over the op default, enabling headful escalation.
+
+### 3. Headful escalation for SERP/PAA — `src/harvest.ts`
+- Add a constant: `const HEADLESS_ATTEMPTS = MAX_ATTEMPTS` (i.e. all normal attempts headless), and extend the loop with ONE extra headful attempt.
+- In the attempt loop (`for (let i = 0; i < maxAttempts; i++)`, ~line 252), when building the per-attempt `DriverConfig` that is passed to `driver.launch(config)` (the config assembled around line 166/271), set:
+```ts
+const escalateHeadful = i >= maxAttempts   // last, extra attempt
+config.headlessMode = escalateHeadful ? 'headful' : 'headless'
+```
+- Bump the loop bound so there is a final headful attempt: iterate `maxAttempts + 1`, where the last iteration forces `headful`. Only run the headful attempt if prior attempts failed with CAPTCHA (`outcome === 'captcha'`); if a headless attempt already succeeded, break as today.
+
+### 4. Headful escalation for maps_search — `src/maps-search-rotation.ts`
+Mirror §3 in `runMapsSearchWithRotation` (~line 129): attempts 1..N headless (`headlessMode:'headless'`), final attempt `headlessMode:'headful'` if previous attempts failed. `maps_place` (`src/api/maps-routes.ts` `/place` → `MapsExtractor`) needs **no change** — its op is `maps_place`, absent from `HEADLESS_OPS`, so it stays headful automatically.
+
+### 5. Website Kernel fallback → headless — `src/api/kernel-fetch.ts`
+Pin the website-fallback browser to headless (proven safe, 8× cheaper). Change the create to default headless unless an explicit headful override is in context:
+```ts
+const ctx = currentCostContext()
+const useHeadless = ctx?.forceHeadless !== false   // default headless for page fetches
+const kb = await client.browsers.create({ stealth: true, ...(useHeadless ? { headless: true } : {}), timeout_seconds: 60 })
+```
+`rotating-proxy-crawl.ts` already passes `headless` explicitly (keep its `DEFAULT_HEADLESS = true`).
+
+### 6. extract_url auto-fallback stays on — `src/api/server.ts` (`/extract-url`)
+Confirm the handler always passes `browserServiceApiKey()` into `extractKpo` so `tryPlainFetch → fetchWithKernel` fallback runs without requiring `browserFallback:true` (the probe showed it already falls back). If it is gated, ungate it so concurrency-1 normal scrapes get the HTTP→headless-Kernel backup by default.
+
+### 7. Proxy defaults → rotating location for SERP — `src/schemas.ts`
+- Change `DEFAULT_PROXY_MODE` to `'location'` (maps is already `'location'`).
+- In the proxy resolver (`resolveKernelProxyId` / per-attempt proxy selection used by harvest + maps rotation): ensure each attempt requests a **new** proxy id even when a `location` is fixed (rotate within the region). If no `location` is supplied, `'location'` mode should fall back to rotating generic proxies rather than erroring — verify and patch the resolver.
+
+### 8. Telemetry — already in place
+`kernel_session_log.headless_sent` + `cost_probe_runs.mode` already record the tier. After this change, `costs`/`probe-report` will show the real (now mostly-headless) spend. No change needed beyond re-running probes.
+
+## Files changed (summary)
+- `src/api/cost-rates.ts` — add `HEADLESS_OPS`.
+- `src/types.ts` — add `headlessMode?` to `DriverConfig`.
+- `src/driver/BrowserDriver.ts` — headless decision honoring `config.headlessMode` + `HEADLESS_OPS`.
+- `src/harvest.ts` — headless attempts + final headful escalation.
+- `src/maps-search-rotation.ts` — same escalation for maps_search.
+- `src/api/kernel-fetch.ts` — default website fallback to headless.
+- `src/api/server.ts` — ensure extract_url passes kernel key for auto-fallback.
+- `src/schemas.ts` — `DEFAULT_PROXY_MODE = 'location'`; resolver rotates proxy per attempt.
+
+## Kill-switch / rollback
+`KERNEL_HEADLESS_DISABLED=true` (env) reverts ALL ops to headful instantly. Per-op revert = remove the op from `HEADLESS_OPS`.
+
+## Verification (run after implementing, before/after compare)
+1. `npm run build` green.
+2. Deploy to prod.
+3. `npx tsx scripts/cost-probe.ts --api-key sk_live_…` (default) and `--headless` — expect: serp/paa/maps_search/yt_channel/fb_search now headless by default; maps_place still headful; website ops HTTP/headless.
+4. `python3 .claude/skills/mcp-metrics/scripts/mcp_metrics.py probe-report` — confirm `mgn_hl` (now real) is positive/healthy and `try` counts show escalation only on CAPTCHA.
+5. `… costs 1` — confirm live spend drops toward the headless column.
+6. Owner pulls the Kernel invoice to confirm the per-second tier flipped to headless for the affected ops.
+
+## Open items to test BEFORE widening
+- Probe `fb_ad`, `instagram_profile`, `instagram_media`, `fb_video_transcribe` in headless before adding to `HEADLESS_OPS` (maps_place proved untested headless can crash).
+- Decide SERP pricing: 0.1 cr barely covers a slow proxy SERP even headless; fast no-proxy profits.

package/docs/spec-mcp-tools.md

@@ -0,0 +1,146 @@
+# MCP Tool Spec — Current vs Proposed (every tool)
+
+Implementation blueprint. Each tool lists **current values** (verbatim from source) and **proposed values**, with exact `file:line` targets. Inventory: 46 registered tools — 25 core (`src/mcp/paa-mcp-server.ts`), 2 SERP-snapshot (`src/mcp/mcp-routes.ts`), 19 browser (`src/mcp/browser-agent-mcp-server.ts`).
+
+Source of truth for costs: `src/api/rates.ts` → `MC_COSTS` (MC_PER_CREDIT = 1000). Site prices already derive from this via `scripts/gen-rates.ts` (the rate catalog). Credit $-value: Starter $0.0015, Growth $0.00125, Scale $0.00104.
+
+---
+
+## PART 0 — Cross-cutting changes (do these once; they touch many tools)
+
+### 0.1 Delete dead code: `MCP_TOOL_MANIFEST`
+- **File:** `src/mcp/mcp-tool-manifest.ts` (~39 KB). **Current:** exported, imported by **nothing** (never in any commit). **Proposed:** delete the file, OR (ambitious) wire its `tokenBudgetClass`/`producesHandlesFor` as the progressive-disclosure layer (see 0.3). Default recommendation: **delete**.
+
+### 0.2 Remove stale proxy boilerplate (server now self-heals proxy選択)
+The agent no longer needs to reason about proxy at all: `DEFAULT_PROXY_MODE='none'`, the dead `configured` proxy is neutralized, and harvest does a 2-strike location→none fallback. So the proxy prose is dead weight + factually wrong ("default configured = highest success").
+
+- **`src/mcp/mcp-tool-schemas.ts`** — the `proxyMode` / `proxyZip` / `debug` `.describe()` strings (lines 11–13, 134–136, 151–153, 861–863, 874–877; ~15 fields, ~580 tokens). **Current:** each says *"Default configured uses the service proxy … highest general success rate … none only for direct-network debugging."* **Proposed:** collapse each `proxyMode` describe to one line: `'Leave unset. The service auto-selects the most reliable routing (clean egress by default; city/ZIP residential only when a location is given, with automatic fallback). Set "location" only to force local-pack targeting.'` Drop the `proxyZip`/`debug` describe to a single short sentence each.
+- **Tool descriptions carrying the same prose** (rewrite, see PART 4): `harvest_paa`, `search_serp`, `capture_serp_snapshot`, `maps_search`, `directory_workflow`.
+
+### 0.3 Progressive disclosure (optional, high-leverage)
+- **Current:** all 46 tool descriptions load into agent context every session.
+- **Proposed:** advertise a **core** set; defer the rest behind a `tools/list` tier or a `workflow_suggest`-style index. Core = `harvest_paa, search_serp, extract_url, extract_site, map_site_urls, maps_search, maps_place_intel, youtube_harvest, youtube_transcribe, facebook_page_intel, instagram_profile_content, credits_info`. Deferred = all `workflow_*`, `rank_tracker_workflow`, `directory_workflow`, `capture_serp_*`, `audit_site`, `facebook_ad_*`, `browser_replay_*`, `browser_list_*`. Reuse the deleted manifest's `tokenBudgetClass` field as the tier key.
+
+### 0.4 Cost catalog — DONE this session
+`MC_COSTS` is the single source; `gen-rates.ts` regenerates `public/rates.generated.js` on build; all site prices read `window.MCP_RATES`. No further action except the cost changes in PART 1.
+
+---
+
+## PART 1 — Cost table (all metered tools)
+
+`cr` = credits. "Margin" = at Starter $0.0015/cr vs measured marginal cost. Measured costs from the 2026-06-29 clean sweep.
+
+| Tool | MC_COSTS key | Current cr | Measured cost | Margin (Starter) | **Proposed cr** | Why |
+|---|---|---|---|---|---|---|
+| harvest_paa | `paa` | 0.3 / question | $0.00018/q | ~2.5× @10q, ↑ at scale | **0.3 / q** (keep) | set this session |
+| search_serp | `serp` | 0.1 / search | $0.00004 | 3.8× | **0.1** (keep) | cheap, fine |
+| extract_url | `page_scrape` | 0.1 / page | ~$0 (HTTP) | huge | **0.1** (keep) | |
+| extract_site | `page_scrape` | 0.1 / page | ~$0 (HTTP) | huge | **0.1** (keep) | |
+| map_site_urls | `url_map` | 2 / op | ~$0 (HTTP) | huge | **0.5** | overpriced; lower for goodwill |
+| audit_site | `page_scrape`×pages | 0.1 / page | ~$0 + LLM | high | **0.1/page** (keep) | LLM cost is small |
+| maps_search | `maps_search` | 2 / search | $0.00013 | 23× | **0.5** | 23× is needless; DataForSEO is $0.0006 |
+| maps_place_intel | `maps_place` + `maps_review` | 2 + 0.1/review | $0.0015 (headful) | **2.0× — under 4×** | **3 base + 0.2/review** | headful, can't cheapen; raise to clear 4× |
+| youtube_harvest | `yt_channel` | 0.2 / call | $0.00011 | ~6× | **0.2** (keep) + consider 0.01/video | flat ok at small N |
+| youtube_transcribe | `yt_transcription` | 0.2 / min | ~$0 (free captions) | huge | **0.2/min** (keep) | |
+| facebook_page_intel | `fb_ad` | 0.2 / call | $0.00010 | ~5× | **0.2** (keep) | set this session |
+| facebook_ad_search | `fb_search` | 0.1 / search | soft-blocks (refunds) | n/a | **0.1** (keep) | only failing tool; see PART 2 |
+| facebook_ad_transcribe | `fb_transcribe` | 0.2 / min | ~$0 | high | **0.2/min** (keep) | |
+| facebook_video_transcribe | `fb_transcribe` | 0.2 / min | browser + fal | medium | **0.2/min** (keep) | |
+| instagram_profile_content | `instagram_profile` | 0.2 / scan | $0.00015 | ~5× | **0.2** (keep) | set this session |
+| instagram_media_download | `instagram_media` | 0.2 / post | browser | ~5× | **0.2** (keep) | |
+| instagram (transcribe) | `instagram_transcribe` | 0.05 | fal | high | **0.05** (keep) | |
+| capture_serp_snapshot | (serp path) | ~0.1 | browser | ok | **0.1** (align to serp) | |
+| directory_workflow | maps_search × cities | 2 × cities | maps×N | inherits maps | **0.5 × cities** | follows maps_search drop |
+| browser_* (19 tools) | `browser_minute` | 12 / min | ~$0.008/min headful (idle free) | ~2.25× active | **12/min** (keep) | set this session |
+| workflow_* / rank_tracker | per underlying call | varies | varies | — | **bill underlying** (keep) | planning tools = free |
+
+**Net cost changes to make:** `maps_search` 2 → **0.5**, `url_map` 2 → **0.5**, `maps_place` base 2 → **3**, `maps_review` 0.1 → **0.2**. All in `src/api/rates.ts` `MC_COSTS`. Everything else stays as set this session.
+
+---
+
+## PART 2 — Per-tool spec (current → proposed)
+
+For each: cost (PART 1), key params (current default/cap → proposed), behavior, description action. `file:line` = `src/mcp/paa-mcp-server.ts` unless noted.
+
+### Research / SERP
+**harvest_paa** (174) — params: `maxQuestions` default 30 / max 200 (schemas.ts:7); `proxyMode` default none. Behavior: headful PAA, `none` egress, 2-strike fallback, **partial-return on timeout** (softDeadlineMs). **Proposed:** keep params; **rewrite description** (PART 4) to drop proxy prose + note "returns partial results if it hits the time budget." Cap guidance already good.
+
+**search_serp** (182) — `pages` default 1 / max 2. **Proposed:** rewrite description (drop proxy prose). Keep params.
+
+**capture_serp_snapshot / capture_serp_page_snapshots** (`mcp-routes.ts` 73, 81) — **Proposed:** rewrite `capture_serp_snapshot` description (drop proxy prose). These are Phoenix-product paths → candidates for **deferred** tier (0.3).
+
+### Web extraction
+**extract_url** (190) — params: `screenshot`, `extractBranding`, `downloadMedia`, `depositToVault` (all default false). **Proposed:** no change; description is good (selection-oriented).
+
+**map_site_urls** (198) — `maxUrls` max 10000, >500 → CSV file. **Proposed:** cost 2→0.5; keep caps.
+
+**extract_site** (206) — `maxPages` max 10000, >25 → folder mode. `rotateProxies` optional (schemas.ts:39 — this proxy hint is **legitimate**, keep it). **Proposed:** no cost change (0.1/page); keep description.
+
+**audit_site** (214) — `maxPages` cap. **Proposed:** no change; **deferred** tier candidate.
+
+### YouTube
+**youtube_harvest** (222) — `maxVideos` default 50 / max 500 (schemas.ts:57). **Proposed:** keep 0.2 flat; **note** in cost model that 500-video harvests are the context/cost outlier — consider per-video 0.01 later. Keep description.
+
+**youtube_transcribe** (230) — **Proposed:** no change.
+
+### Facebook
+**facebook_page_intel** (238) — `maxAds` default 50 / max 200 (schemas.ts:71). **Proposed:** keep 0.2.
+**facebook_ad_search** (246) — `maxResults` default 10 / max 20. **THE ONE BROKEN TOOL:** soft-blocks (FB rejects egress IP for ad-library search; refunds correctly). **Proposed:** (a) try a clean residential/BYO proxy specifically for this path; (b) if unreliable, mark description "best-effort" and keep auto-refund. Needs its own investigation task.
+**facebook_ad_transcribe** (254) / **facebook_video_transcribe** (262) — **Proposed:** no change.
+
+### Instagram
+**instagram_profile_content** (270) — `maxItems` default 50 / max **2000**, `maxScrolls` 10/250. **Proposed:** keep cost 0.2; **keep maxItems cap (no evidence it needs lowering)**.
+**instagram_media_download** (278) — `includeTranscript` default false (adds cost). **Proposed:** no change.
+
+### Maps
+**maps_place_intel** (286) — `includeReviews` false, `maxReviews` default 50 / max **500**. Behavior: **headful** (headless crashes). **Proposed:** cost **3 base + 0.2/review**; **keep maxReviews cap (no evidence it needs lowering)** — customer pays per review. Keep description.
+**maps_search** (294) — `maxResults` default 10 / max 50; `proxyMode` default **location** (DEFAULT_MAPS_PROXY_MODE). **Proposed:** cost 2→**0.5**; **rewrite description** to drop the "omit proxyMode for residential" prose (server handles it); keep location default but note auto-fallback. ⚠️ Maps still uses `location` proxy → watch for the same flagged-pool CAPTCHA; if it appears, flip to `none` like SERP.
+
+### Workflows / planning (no web, no/low credits)
+**directory_workflow** (302) — `state` TN, `minPopulation` 100000, `maxCities` 25/100, `maxResultsPerCity` 50, `concurrency` 5/5. **Proposed:** cost follows maps_search (0.5×cities); rewrite description to drop proxy prose; **deferred** tier.
+**workflow_list / workflow_suggest / workflow_run / workflow_step / workflow_status / workflow_artifact_read** (310–356) — planning/orchestration, no credits. **Proposed:** no change; **deferred** tier (except keep `workflow_suggest` discoverable as the router).
+**rank_tracker_workflow** (358) — local planning, no credits. **Proposed:** no change; **deferred** tier.
+**credits_info** (366) — read-only, server-driven prices. **Proposed:** no change; **core** tier.
+
+### Browser (19 tools, `browser-agent-mcp-server.ts`)
+`browser_profile_connect, browser_profile_list, browser_open, browser_screenshot, browser_read, browser_locate, browser_goto, browser_click, browser_type, browser_scroll, browser_press, browser_replay_start, browser_replay_stop, browser_list_replays, browser_replay_download, browser_replay_mark, browser_replay_annotate, browser_close, browser_list_sessions`.
+- **Cost:** all session-metered via `browser_minute` = **12 cr/min** (idle free — Standby). Accurate post-change.
+- **Params:** `timeout_seconds` default 600 (browser-agent-tool-schemas.ts:24) — describe already correct ("idles into zero-cost standby … longer timeout is cheap").
+- **Proposed:** **core** = `browser_open, browser_screenshot, browser_read, browser_goto, browser_click, browser_type, browser_scroll, browser_press, browser_close`. **deferred** = all `browser_replay_*`, `browser_list_*`, `browser_profile_*`. No description/cost changes needed.
+
+---
+
+## PART 3 — Param caps: NO CHANGES
+
+Earlier drafts proposed lowering `maps_place_intel maxReviews` (500) and `instagram_profile_content maxItems` (2000). **Retracted — there is no evidence either cap causes a problem.** No measured failure, timeout, or cost issue at the high end. The customer pays per unit, and long jobs are handled by partial-return/timeout budgets (already done for PAA), not by capping. **All caps stay as-is.** Only lower a cap if we *observe* a specific failure mode at the high end.
+
+---
+
+## PART 4 — Description rewrites (exact old → new)
+
+Remove proxy prose; keep selection guidance. Targets in `src/mcp/paa-mcp-server.ts` (+ mcp-routes.ts).
+
+**harvest_paa (176)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence."* → replace with: *"Routing is automatic — leave proxyMode unset. If a deep harvest exceeds the time budget it returns the questions gathered so far (partial), billed per returned question."*
+
+**search_serp (184)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."* → *"Routing is automatic — leave proxyMode unset; set location only to force local-pack targeting."*
+
+**maps_search (296)** — strike: *"Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting."* → *"Routing is automatic and location-aware — leave proxyMode unset; pass proxyZip only to force a specific ZIP."*
+
+**directory_workflow (304)** — strike the sentence *"Maps workflows default to location-targeted proxying … use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting."* → *"Routing is automatic per city."*
+
+**capture_serp_snapshot (`mcp-routes.ts:73`)** — strike: *"Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."* → *"Routing is automatic — leave proxyMode unset."*
+
+---
+
+## PART 5 — Execution order, files, release
+
+1. **Costs** — `src/api/rates.ts` `MC_COSTS`: `maps_search` 2000→**500**, `url_map` 2000→**500**, `maps_place` 2000→**3000**, `maps_review` 100→**200**.
+2. **Proxy describe trim** — `mcp-tool-schemas.ts` proxyMode/proxyZip/debug fields (0.2).
+3. **Description rewrites** — PART 4 (`paa-mcp-server.ts`, `mcp-routes.ts`).
+4. **Delete** `src/mcp/mcp-tool-manifest.ts` (0.1).
+5. **(Optional, separate PR)** progressive-disclosure tier (0.3).
+6. **Build** (`npm run build` — regenerates rate catalog + bundles), **deploy Vercel prod** (covers REST billing + remote-MCP descriptions + site prices).
+7. **`npm publish`** — REQUIRED this time: tool descriptions are baked into the stdio package, so npm-installed clients need the new version to see trimmed/accurate descriptions. Bump version in the 3 release files first (per release workflow), vercel prod before publish. (No param caps changed.)
+9. **Open task:** `facebook_ad_search` soft-block (clean/BYO proxy or best-effort label).
+
+**Why npm matters here (unlike the rate change):** rates/logic run server-side and reach thin clients automatically, but **tool descriptions and input-schema caps are advertised by the MCP package itself** → republish required for stdio installs; remote-MCP users get it on deploy.

package/docs/spec-server-instructions.md

@@ -0,0 +1,94 @@
+# Spec — server-instructions.ts routing-map rewrite
+
+Implements the approved improvements from the MCP-design audit + reviewer feedback. **One file changes:** `src/mcp/server-instructions.ts` (the `SERVER_INSTRUCTIONS` string — the only content loaded into the model's context before it fetches any tool schema). No other source files. Requires a build + Vercel deploy (remote-MCP gets it immediately) + npm republish (the string ships in the stdio package).
+
+## Approved improvements (the full list)
+1. **Add `reddit_thread` to the map; stop routing Reddit to the browser.** Current line 15 says "Anything without a dedicated tool (e.g. Reddit…) -> browser_* agent" — **stale**, we shipped `reddit_thread` in 0.3.25. A model reading the current map will drive a browser for Reddit instead of calling the dedicated tool.
+2. **Make discovery→action ID seams explicit and schema-accurate** (reviewer #1). Encode, per chain, what the discovery tool *returns* and what the action tool *takes* — using the real params, not inferred ones.
+3. **Workflow triggers + boundary** (reviewer #2): plain-language "when to reach for a workflow vs hand-chaining," and state `query_fanout_workflow`'s browser prerequisite.
+4. **Reorder the no-dedicated-tool branch to check login FIRST** (reviewer #3), so a logged-in site isn't short-circuited into `browser_open` without `browser_profile_connect`.
+5. **Keep** the `rotateProxies` NOTE (reviewer #4 — defensible cross-cutting hint).
+
+Not in this spec (explicitly out): annotations (audited → correct as-is, not a bug), server namespace rename (breaking, deferred decision), tool-count tiering (soft, not needed). `deep_research_workflow` removal already shipped.
+
+## Verified schema facts backing each seam (source of truth)
+| Discovery tool | Returns (output schema) | Action tool | Takes (input schema) | Seam |
+|---|---|---|---|---|
+| `map_site_urls` / `search_serp` | `urls` / `organicResults[].url` | `extract_url` | `url` | url → url |
+| `search_serp` | `organicResults[].url` (incl. reddit.com) | `reddit_thread` | `url` | reddit url → reddit_thread |
+| `maps_search` | `results[].{name, placeUrl, cid}` | `maps_place_intel` | **`businessName` + `location`** (NOT an id) | name → name; **independent, callable directly** |
+| `youtube_harvest` | `videos[].videoId` | `youtube_transcribe` | `videoId` OR `url` | videoId → videoId |
+| `facebook_ad_search` | `advertisers[].{pageId, libraryId}` | `facebook_page_intel` | `pageId` / `libraryId` / `query` | id → id |
+| `facebook_page_intel` | `ads[].videoUrl` | `facebook_ad_transcribe` | `videoUrl` | videoUrl → videoUrl |
+| (organic FB url) | — | `facebook_video_transcribe` | `url` (FB reel/video/post) | url direct |
+| `instagram_profile_content` | `post`/`reel`/`tv` url arrays | `instagram_media_download` | `url` | url → url |
+| (must precede) `browser_open` on chatgpt.com/claude.ai | — | `query_fanout_workflow` | runs against the open AI session | browser-session prerequisite |
+
+`facebook_ad_transcribe` does NOT consume anything from `facebook_ad_search` — the videoUrl comes from `facebook_page_intel`. `maps_place_intel` does NOT consume a `place_id` — it takes a business name + location.
+
+## Exact change
+Replace the entire `SERVER_INSTRUCTIONS` template literal in `src/mcp/server-instructions.ts` (lines 1–29) with:
+
+```ts
+export const SERVER_INSTRUCTIONS = `
+This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
+this routing map, then load its schema before calling. Where a tool needs a value another tool
+produces, the seam is noted so you can chain them.
+
+ROUTING
+- One web page -> extract_url (takes a url). Whole site (crawl + SEO report) -> extract_site (takes a
+  url). Just the URL list -> map_site_urls (takes a url). map_site_urls and search_serp return urls you
+  can feed straight into extract_url.
+- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail ->
+  harvest_paa. search_serp returns organicResults[].url (often including reddit.com threads) to feed
+  extract_url or reddit_thread.
+- Google Maps: find places -> maps_search (returns results[] with name, placeUrl, cid). One place
+  deep-dive + reviews -> maps_place_intel (takes businessName + location, NOT an id — call it directly
+  with a name from a maps_search result or from the user).
+- YouTube: find or list videos -> youtube_harvest (returns videos[].videoId). Transcribe one ->
+  youtube_transcribe (takes a videoId from youtube_harvest, or a url).
+- Facebook: find advertisers -> facebook_ad_search (returns advertisers[] with pageId, libraryId). Get
+  one advertiser's ads -> facebook_page_intel (takes pageId, libraryId, or query; returns ads[].videoUrl).
+  Transcribe an ad video -> facebook_ad_transcribe (takes a videoUrl from facebook_page_intel).
+  Transcribe an organic Facebook reel/video/post -> facebook_video_transcribe (takes the Facebook url
+  directly).
+- Instagram: profile inventory -> instagram_profile_content (returns post/reel/tv urls). One post or
+  reel -> instagram_media_download (takes a url from instagram_profile_content, or a url the user gives).
+- Reddit: a reddit.com thread/post URL -> reddit_thread. It returns the post plus its comment tree and
+  handles Reddit's bot wall itself (no login needed). Find threads first with search_serp.
+- No dedicated tool (an arbitrary site, or a logged-in dashboard) -> the browser_* agent. FIRST decide
+  whether the site needs a login:
+  - Needs a login (ChatGPT, Claude, any account) -> save it first: browser_profile_connect returns an
+    mcpscraper.dev sign-in link for the user; one profile holds MANY logins (call it again with the same
+    profile + a new domain to add accounts). Poll browser_profile_list until AUTHENTICATED, then
+    browser_open with that profile. browser_profile_list also shows what a profile is connected to.
+  - No login -> browser_open, then navigate/read.
+
+WORKFLOWS (multi-step orchestrations — prefer these over hand-chaining primitives when the whole job is one of these)
+- Build a local business directory for a niche across a state/region -> directory_workflow.
+- Stand up a rank-tracking blueprint (database + cron + ingestion plan) -> rank_tracker_workflow.
+- Find which sources an AI answer cites for a query (AEO) -> query_fanout_workflow (open a browser
+  session on chatgpt.com or claude.ai FIRST, then run it against that session).
+
+NOTES
+- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
+  It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
+- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
+  cheapest tool that answers the question (plain search/extract before browser agents).
+- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
+  for full detail rather than expecting the whole payload inline.
+\`.trim()
+```
+
+## Build / deploy / release
+1. Edit the one file above.
+2. `npm run build` (typecheck + bundle; regenerates rate catalog — no functional change here).
+3. `vercel deploy --prod --yes` → remote-MCP clients get the new routing map immediately.
+4. **Bump 0.3.25 → 0.3.26** in the 3 version files (`package.json`, `package-lock.json`, `src/version.ts`) and `npm publish` — `SERVER_INSTRUCTIONS` is compiled into the stdio package, so npm-installed clients need the new version to see the corrected map.
+5. Commit on `main` (now the source of truth): `fix(mcp): routing map — add reddit_thread, explicit chaining seams, workflow triggers, login-first fallback`.
+
+## Verification
+- `grep -c reddit_thread src/mcp/server-instructions.ts` → ≥1 (was 0).
+- `grep -c "without a dedicated tool" src/mcp/server-instructions.ts` → 0 (old Reddit-to-browser line gone).
+- After deploy, the live `/mcp` server-instructions block reflects the new map (inspect via an MCP client's server info, or the SERVER_INSTRUCTIONS in the published tarball).
+- Behavioral smoke (manual, in an MCP client): "get the comments on <reddit thread url>" routes to `reddit_thread`, not `browser_open`.