mcp-scraper 0.1.6 → 0.1.7

package/dist/{worker-UT4ZQU2T.js → worker-PBG6LGET.js}

@@ -4,10 +4,11 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-7HB7NDOY.js";
+} from "./chunk-ZK456YXN.js";
 import {
   harvest
-} from "./chunk-W4P2U5VF.js";
+} from "./chunk-LUBDFS67.js";
+import "./chunk-ZMOWIBMK.js";
 import {
   claimPendingJob,
   completeJob,
@@ -120,4 +121,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-UT4ZQU2T.js.map
+//# sourceMappingURL=worker-PBG6LGET.js.map

package/dist/{worker-UT4ZQU2T.js.map → worker-PBG6LGET.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const result = await harvest({\n      ...opts,\n      kernelApiKey: process.env.KERNEL_API_KEY,\n      headless: true,\n      format: 'json',\n      outputDir: '/tmp/paa-output-api',\n      onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n    })\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACEA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AAC/C;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,GAAG;AAAA,MACH,cAAc,QAAQ,IAAI;AAAA,MAC1B,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,IAClE,CAAC;AACD,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}
+{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const result = await harvest({\n      ...opts,\n      kernelApiKey: process.env.KERNEL_API_KEY,\n      headless: true,\n      format: 'json',\n      outputDir: '/tmp/paa-output-api',\n      onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n    })\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACEA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AAC/C;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,GAAG;AAAA,MACH,cAAc,QAAQ,IAAI;AAAA,MAC1B,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,IAClE,CAAC;AACD,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md

@@ -0,0 +1,58 @@
+# ADR 0001: In-page API interception for anti-bot scraping (Facebook Ad Library)
+
+- **Status:** Accepted
+- **Date:** 2026-06-03
+- **Deciders:** Andrew (operator), engineering
+- **Applies to:** `facebook_ad_search`, `facebook_page_intel`, and future scrapers of JS-heavy, anti-bot-protected sites
+
+## Context
+
+`facebook_ad_search` was unreliable — it usually returned `soft-block: no results (refunded)`, and when it did return, advertiser names came back `undefined` with `—` for library IDs. Two independent root causes:
+
+1. **DOM scraping a hostile SPA.** The Facebook Ad Library is a React app. The ads are *not* in the initial HTML — the page fires its own background GraphQL request and renders the JSON into DOM cards asynchronously. Our scraper waited for those cards to render, then re-parsed the rendered text with regexes (`See ad details … Sponsored`). That is fragile (breaks on any UI change) and slow, and it conflated "page still loading" with "no results."
+
+2. **A single static proxy.** Every Facebook scrape launched with the static `KERNEL_PROXY_ID`. Facebook flags/rate-limits a repeatedly-used IP and serves a near-empty / login-gated page to it. A real residential browser loaded the same search fine — confirming the block was IP-reputation, not the URL or a login requirement. `facebook_page_intel` worked intermittently for the same reason (same proxy, sometimes flagged).
+
+The "undefined names / — library IDs" was a third, smaller bug: a field-name mismatch between the API response (`pageName`, `sampleLibraryId`) and the MCP formatter (`name`, `libraryId`).
+
+## Decision
+
+**Stop scraping the rendered page. Intercept the JSON the page already fetches, on a fresh residential proxy, with the DOM scrape kept only as a fallback.**
+
+Concretely, for the Ad Library:
+
+1. **Intercept the in-page GraphQL response.** The SPA issues `AdLibrarySearchPaginationQuery` (a `doc_id`-based POST to `/api/graphql/`) and Facebook returns the ads as JSON. We attach a Playwright `page.on('response')` listener before navigation and parse the JSON directly (`data.ad_library_main.search_results_connection.edges[].node.collated_results[]`). No request is forged — the page makes the request itself, so it carries Facebook's own session/cookies and looks like a legitimate first-party call. Implemented in `src/extractor/FacebookAdGraphql.ts`.
+2. **Fresh residential proxy per request.** Both FB routes now launch via `kernelLaunchOptsResidential()`, which uses `resolveKernelProxyId` (the same residential rotation that fixed the PAA CAPTCHAs) instead of the flagged static `KERNEL_PROXY_ID`.
+3. **DOM scrape as fallback.** If no GraphQL response is captured (e.g., the query shape drifts), the existing regex DOM parse still runs on the already-loaded page.
+4. **Field aliasing for client compatibility.** The API now returns both `pageName`/`name` and `sampleLibraryId`/`libraryId`, so even already-installed MCP clients render correctly without a package update; the formatter was also fixed to read the canonical fields.
+
+This is the **same pattern already proven by the YouTube transcription InnerTube tier** (`CaptionFetcher.ts` does `page.evaluate(fetch('/youtubei/v1/player'…))` from inside the loaded page). This ADR names it as the preferred default for this class of target.
+
+### The reusable pattern
+
+> For a JS-heavy, anti-bot-protected site whose data arrives via an internal API call: load the page on a clean (residential) IP, capture or replay that internal request **from within the page's own session**, parse the JSON, and keep DOM scraping only as a fallback.
+
+Prefer **intercepting the response** (`page.on('response')`) over **replaying the request** when the page fires the call itself — it needs zero token/`doc_id`/`fb_dtsg` reconstruction and is the most robust form.
+
+## Consequences
+
+**Positive**
+- Parses structured JSON, not DOM → immune to CSS/markup changes (the usual scraper breakage).
+- Faster — no waiting for the grid to render/scroll before reading.
+- Unambiguous — an explicit ad list or an error, killing the "0 results = blocked?" guesswork.
+- Lower block rate — an in-session first-party request plus a clean residential IP looks legitimate.
+- Verified live: `Nike` returned 10 distinct, correctly-named advertisers with library IDs (Nike, Jordan, Nordstrom Rack, eBay, Whatnot…), end-to-end through the MCP, no soft-block.
+
+**Negative / risks**
+- The GraphQL `doc_id` and response shape are Facebook-internal and **drift every few months**. Mitigation: the DOM fallback, plus the capture probe technique below to re-derive the shape in minutes.
+- Still not 100% — the Ad Library is adversarial; residential IPs can occasionally be challenged.
+- Captured `doc_id` (`24922295957467452` as of 2026-06-03) is intentionally *not* hardcoded — we match on the `fb_api_req_friendly_name` (`AdLibrarySearchPaginationQuery`) so a `doc_id` rotation alone doesn't break us.
+
+**Maintenance — re-capturing the request shape when it drifts**
+Run a Kernel session on a residential proxy, attach `page.on('request'|'response')`, navigate to the Ad Library search URL, and log GraphQL `fb_api_req_friendly_name` / `doc_id` / `variables` / response keys. (A throwaway `fb-capture.ts` probe was used during this work; recreate it from this description.)
+
+## Alternatives considered
+
+- **Official Graph Ad Library API (`/ads_archive`).** Rejected: for the US it only returns political/issue ads; commercial ads (the use case) aren't exposed outside the EU.
+- **Replay the GraphQL request manually** (rebuild `lsd`/`jazoest`/`__dyn`/`doc_id`/variables). Rejected as the default: more brittle than intercepting the page's own response, though viable for pagination beyond what scrolling triggers.
+- **Just rotate proxies, keep DOM scraping.** Rejected: fixes the block but leaves the fragile, slow DOM parse and the "0 results" ambiguity.

package/docs/adr/README.md

@@ -0,0 +1,11 @@
+# Architecture Decision Records
+
+Short, durable records of *why* a non-obvious technical decision was made — the context, the choice, and the consequences — so future readers (and future us) don't re-litigate or accidentally undo it.
+
+Write one when a decision is hard to reverse, surprising, or encodes a constraint that isn't visible in the code (an anti-bot workaround, a vendor limitation, a deliberate fallback). Don't write one for routine changes.
+
+## Format
+`NNNN-kebab-title.md`, Nygard-style: **Status · Context · Decision · Consequences**. Status is `Proposed` → `Accepted` → (later) `Superseded by ADR-XXXX`. Number sequentially; never renumber.
+
+## Index
+- [0001 — In-page API interception for anti-bot scraping (Facebook Ad Library)](./0001-in-page-graphql-interception-for-anti-bot-scraping.md)

package/docs/mcp-tool-quality-spec.md

@@ -0,0 +1,238 @@
+# MCP Tool Quality Spec
+
+This spec defines the shipping bar for MCP Scraper tools. It exists because MCP behavior is model-facing: a tool can be technically callable and still fail if the AI cannot infer when to use it, how to fill inputs, or how to chain the result.
+
+## What Actually Steers The AI
+
+The model is primarily affected by what the MCP client receives from `tools/list` and `tools/call`.
+
+1. Tool name
+2. Tool title and annotations
+3. Tool description
+4. Input schema field names, descriptions, defaults, enums, and limits
+5. Output schema and `structuredContent`
+6. Tool result text and next-step guidance
+7. Error shape and retry guidance
+
+README files, website copy, and public skill markdown help humans and skill loaders, but they do not reliably affect runtime AI behavior unless the client explicitly injects those files into context.
+
+## Tool Boundary Rules
+
+Each tool must have one primary job.
+
+- Split tools when the user intent, cost model, result shape, or follow-up workflow differs.
+- Do not overload one tool with unrelated modes if a model could choose the wrong path.
+- Descriptions for adjacent tools must explicitly say when not to use each tool.
+- If a workflow has a natural sequence, encode it in the description and result guidance.
+
+Example:
+
+- `maps_search`: find multiple Google Maps candidates for a category, market, lead list, or "more than the 3-pack".
+- `maps_place_intel`: hydrate one known business or one selected candidate with full profile details and optional reviews.
+
+## Tool Naming
+
+Names must be short, stable, and action-oriented.
+
+- Use domain + action: `maps_search`, `maps_place_intel`, `facebook_ad_search`.
+- Avoid generic names such as `search`, `lookup`, `get_data`, or `run`.
+- Avoid names that imply broader capability than the tool has.
+- Do not rename a public tool without a compatibility plan.
+
+## Tool Descriptions
+
+Descriptions are model instructions. They must be concise but operational.
+
+Every description must include:
+
+- What the tool does.
+- When to use it.
+- When not to use it if there is an adjacent tool.
+- Important defaults and hard caps.
+- Cost-sensitive behavior when relevant.
+- The expected next tool when chaining is common.
+- Whether reports are saved locally.
+
+Bad:
+
+```text
+Search Google Maps.
+```
+
+Good:
+
+```text
+Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Returns up to 50 candidates. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.
+```
+
+## Input Schema
+
+Each input field must have a clear description.
+
+Required:
+
+- Required fields must be actually required in the schema.
+- Defaults must be encoded in the schema, not only in prose.
+- Hard caps must be encoded in the schema.
+- Fields that are often confused must say what not to put there.
+- Location and query fields must tell the model to split location from the topic when possible.
+- Enum fields must explain when to choose each option.
+
+For numeric limits:
+
+- Normal/default value belongs in `.default(...)`.
+- Maximum value belongs in `.max(...)`.
+- Description must say when to use higher values.
+
+## Output Schema And Structured Content
+
+Any tool whose output may be consumed by another tool should have `outputSchema` and return `structuredContent`.
+
+Required for chaining tools:
+
+- Return arrays/objects in `structuredContent`; do not force the model to parse Markdown.
+- Keep `content` as a human-readable report.
+- Ensure `structuredContent` validates against `outputSchema`.
+- Include IDs, URLs, names, positions, counts, and any fields needed for the next tool.
+
+Example:
+
+`maps_search` must return `structuredContent.results[]` with at least:
+
+- `position`
+- `name`
+- `placeUrl`
+- `cid`
+- `cidDecimal`
+- `rating`
+- `reviewCount`
+- `category`
+- `address`
+- `websiteUrl`
+- `directionsUrl`
+- `metadata`
+
+## Tool Annotations
+
+Every public tool should define annotations.
+
+Use:
+
+- `readOnlyHint: true` for research, scrape, search, transcript, and inspect tools.
+- `destructiveHint: false` unless the tool mutates or deletes user data.
+- `idempotentHint: false` for live web searches because results, billing, and anti-bot state can change.
+- `openWorldHint: true` for tools that access public web or external live systems.
+- A human-readable `title`.
+
+Annotations are hints, not a replacement for descriptions.
+
+## Result Text
+
+Human-readable text still matters. It is what users see and what some clients preserve in context.
+
+Every successful result should include:
+
+- Clear title.
+- Returned count versus requested count when relevant.
+- Key result table or summary.
+- Saved report path when report saving is enabled.
+- Next-step guidance for common follow-up actions.
+
+For chained tools, the result text should name the next tool explicitly.
+
+## Error Format
+
+Errors must help the model choose the next action.
+
+Every API error should include:
+
+- `error` or `error_code`
+- Human-readable message
+- Whether retry is reasonable when known
+- Enough context to avoid repeating the same bad call
+
+Common cases:
+
+- Auth failure: tell user API key is invalid or missing. Do not retry.
+- Insufficient balance: return balance, required credits, and top-up URL.
+- CAPTCHA/block: say it is temporary and retryable later.
+- Validation error: identify the bad or missing field.
+- Timeout/cancel: say whether the server attempted cleanup.
+
+## Cost And Concurrency
+
+Tools that cost credits or hold jobs must expose that in metadata or result text.
+
+Required:
+
+- Add cost entry to `CREDIT_COST_CATALOG`.
+- Add ledger operation for billable work.
+- Include refund path on failure where applicable.
+- Respect the account concurrency model.
+- Tool descriptions should warn when a tool is expensive or long-running.
+
+## Docs Surfaces
+
+For a new or changed public tool, update all applicable surfaces:
+
+- `src/mcp/paa-mcp-server.ts`
+- `src/mcp/mcp-tool-schemas.ts`
+- `src/mcp/mcp-response-formatter.ts`
+- API route and API schema
+- `README.md`
+- `public/skill.md`
+- `public/codex-skill.md`
+- `public/skills/mcp-scraper/skill.md`
+- Dashboard UI when users can trigger the workflow there
+- Live protocol tool list tests
+
+## Packaging And Deployment
+
+MCP changes often have two release surfaces.
+
+NPX package:
+
+- Bump `package.json` and `package-lock.json` when publishing npm.
+- Rebuild `dist` with `npx tsup`.
+- Verify `npm pack --dry-run` includes the rebuilt stdio binary.
+- Verify built output contains the new tool name, description, schema, and formatter behavior.
+
+Hosted API:
+
+- Deploy API routes before or with the MCP package.
+- A new MCP tool that calls a new hosted endpoint is broken until production has that endpoint.
+- Answer "was this prod?" by separating API deployment from npm package publication.
+
+## Required Tests
+
+Every new public MCP tool needs tests at the right level for risk.
+
+Minimum:
+
+- Schema default and hard cap tests.
+- Formatter test when result text or `structuredContent` matters.
+- Tool list/protocol test updated with the new tool.
+- Billing/ledger count tests updated for new billable operations.
+- Typecheck.
+- Unit/contract suite.
+
+For live-web tools:
+
+- One live smoke test or saved live evidence for the core workflow.
+- If anti-bot behavior is likely, capture failure mode and retry guidance.
+
+## Definition Of Done
+
+A public MCP tool change is not done until all are true:
+
+- Tool name and boundary are clear.
+- Tool description tells the model when to use it and when not to.
+- Input schema encodes defaults, limits, and field-level instructions.
+- Output is structured when the result will be chained.
+- Result text is useful to humans and names the next tool when appropriate.
+- Errors are actionable.
+- Billing and refunds are correct.
+- Dashboard, docs, skill text, and README are updated where relevant.
+- `dist` is rebuilt for NPX package changes.
+- Production API deployment is accounted for separately from npm publication.
+- Tests and smoke evidence prove the workflow.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -20,14 +20,15 @@
   },
   "files": [
     "dist",
+    "docs",
     "README.md"
   ],
   "scripts": {
     "build": "tsup && npm run build:ui && npm run build:blog",
     "build:ui": "esbuild public/app.jsx --bundle --loader:.jsx=jsx --outfile=public/app.js --target=es2020",
-    "build:blog": "tsx scripts/build-blog.ts",
-    "api": "tsx bin/api-server.ts",
-    "dev": "tsx src/cli.ts",
+    "build:blog": "node scripts/run-ts-sandbox-safe.mjs scripts/build-blog.ts",
+    "api": "node scripts/run-ts-sandbox-safe.mjs bin/api-server.ts",
+    "dev": "node scripts/run-ts-sandbox-safe.mjs src/cli.ts",
     "test": "vitest run tests/unit tests/contract",
     "test:live": "vitest run --config vitest.live.config.ts",
     "test:integration": "node scripts/run-integration-tests.mjs",