mcp-scraper 0.38.2 → 0.40.1

package/dist/site-extract-repository-XSPJTCIL.js

@@ -1,62 +0,0 @@
-import {
-  abandonExtractSettlement,
-  claimFailedExtractJobForRefinalize,
-  completeExtractJob,
-  countSuccessfulPages,
-  createExtractJob,
-  createOrGetExtractJob,
-  extractJobLimitInfo,
-  failExtractJob,
-  failStaleRunningExtractJob,
-  failUnfundedExtractJob,
-  finishExtractJob,
-  getExtractJob,
-  getExtractJobByIdempotencyKey,
-  getExtractedImageLinks,
-  getExtractedPages,
-  getExtractedUrls,
-  listExtractJobs,
-  listFundedPendingExtractJobs,
-  listStaleRunningExtractJobs,
-  listUnsettledExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  saveExtractPages,
-  setExtractJobTotal,
-  settleExtractJob,
-  terminalExtractJobStatus
-} from "./chunk-YCI2PNCS.js";
-import "./chunk-AZRPG43B.js";
-import "./chunk-44HZLHDV.js";
-import "./chunk-N7KUTTCC.js";
-export {
-  abandonExtractSettlement,
-  claimFailedExtractJobForRefinalize,
-  completeExtractJob,
-  countSuccessfulPages,
-  createExtractJob,
-  createOrGetExtractJob,
-  extractJobLimitInfo,
-  failExtractJob,
-  failStaleRunningExtractJob,
-  failUnfundedExtractJob,
-  finishExtractJob,
-  getExtractJob,
-  getExtractJobByIdempotencyKey,
-  getExtractedImageLinks,
-  getExtractedPages,
-  getExtractedUrls,
-  listExtractJobs,
-  listFundedPendingExtractJobs,
-  listStaleRunningExtractJobs,
-  listUnsettledExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  saveExtractPages,
-  setExtractJobTotal,
-  settleExtractJob,
-  terminalExtractJobStatus
-};
-//# sourceMappingURL=site-extract-repository-XSPJTCIL.js.map

package/dist/site-extract-repository-XSPJTCIL.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/worker-XCPU4YSN.js

@@ -1,142 +0,0 @@
-import {
-  classifyHarvestProblem,
-  createHarvestAttemptRecorder,
-  harvestProblemResponse,
-  serializeHarvestProblem
-} from "./chunk-SFRMFGQ6.js";
-import {
-  harvest
-} from "./chunk-NGM237OO.js";
-import {
-  browserServiceApiKey,
-  runWithCostContext
-} from "./chunk-EGJKUB4Q.js";
-import "./chunk-K443GQY5.js";
-import "./chunk-CB5C3BPB.js";
-import {
-  MC_COSTS,
-  serpActualCostMc
-} from "./chunk-AZRPG43B.js";
-import "./chunk-44HZLHDV.js";
-import {
-  claimPendingJob,
-  completeJob,
-  creditMc,
-  debitMc,
-  failJob,
-  listHarvestAttempts
-} from "./chunk-N7KUTTCC.js";
-
-// src/api/webhook.ts
-async function deliverWebhook(url, payload, retries = 3) {
-  for (let attempt = 1; attempt <= retries; attempt++) {
-    try {
-      const res = await fetch(url, {
-        method: "POST",
-        headers: { "content-type": "application/json" },
-        body: JSON.stringify(payload),
-        signal: AbortSignal.timeout(1e4)
-      });
-      if (res.ok) return;
-      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
-    } catch (err) {
-      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
-    }
-    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
-  }
-  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
-}
-
-// src/api/worker.ts
-var MAX_CONCURRENT = 2;
-var running = 0;
-function countPaaQuestions(result) {
-  if (!result || typeof result !== "object") return 0;
-  const value = result;
-  if (typeof value.totalQuestions === "number") return value.totalQuestions;
-  return Array.isArray(value.flat) ? value.flat.length : 0;
-}
-function paaCostForQuestionCount(questionCount) {
-  return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa;
-}
-async function processJob(job) {
-  running++;
-  try {
-    const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
-    const headlessSentOut = { value: null };
-    const result = await runWithCostContext(
-      { op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id), headlessSentOut },
-      () => harvest({
-        ...opts,
-        kernelApiKey: browserServiceApiKey(),
-        headless: true,
-        format: "json",
-        outputDir: "/tmp/paa-output-api",
-        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id)
-      })
-    );
-    await completeJob(job.id, result);
-    const attempts = await listHarvestAttempts(job.id, job.user_id);
-    if (!opts.serpOnly && typeof opts.billingHoldMc === "number") {
-      const actualCost = paaCostForQuestionCount(countPaaQuestions(result));
-      const diff = opts.billingHoldMc - actualCost;
-      if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
-      else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
-    } else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
-      const actualCost = serpActualCostMc(headlessSentOut.value);
-      const diff = opts.billingHoldMc - actualCost;
-      if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
-      else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
-    }
-    if (job.callback_url) {
-      await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result, attempts });
-    }
-  } catch (err) {
-    const problem = classifyHarvestProblem(err);
-    await failJob(job.id, serializeHarvestProblem(problem));
-    const attempts = await listHarvestAttempts(job.id, job.user_id);
-    try {
-      const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
-      if (typeof opts.billingHoldMc === "number" && opts.billingHoldMc > 0) {
-        await creditMc(job.user_id, opts.billingHoldMc, "refund", "failed call");
-      }
-    } catch {
-    }
-    if (job.callback_url) {
-      await deliverWebhook(job.callback_url, { job_id: job.id, status: "failed", ...harvestProblemResponse(problem), attempts });
-    }
-  } finally {
-    running--;
-  }
-}
-async function tickOnce() {
-  const job = await claimPendingJob();
-  if (!job) return { claimed: false };
-  const startedAt = Date.now();
-  await processJob(job);
-  return { claimed: true, jobId: job.id, completed: true, durationMs: Date.now() - startedAt };
-}
-async function drainQueue(budget) {
-  const results = [];
-  for (let i = 0; i < budget.maxJobs; i++) {
-    if (Date.now() >= budget.deadlineMs) break;
-    const r = await tickOnce();
-    results.push(r);
-    if (!r.claimed) break;
-  }
-  return results;
-}
-function startWorker() {
-  setInterval(async () => {
-    if (running >= MAX_CONCURRENT) return;
-    const job = await claimPendingJob();
-    if (job) void processJob(job);
-  }, 2e3);
-  console.log(`[worker] started \u2014 polling every 2s, max ${MAX_CONCURRENT} concurrent`);
-}
-export {
-  drainQueue,
-  startWorker,
-  tickOnce
-};
-//# sourceMappingURL=worker-XCPU4YSN.js.map

package/dist/worker-XCPU4YSN.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS, serpActualCostMc } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\nimport { runWithCostContext } from './cost-context.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const headlessSentOut: { value: boolean | null } = { value: null }\n    const result = await runWithCostContext(\n      { op: opts.serpOnly ? 'serp' : 'paa', userId: Number(job.user_id), headlessSentOut },\n      () => harvest({\n        ...opts,\n        kernelApiKey: browserServiceApiKey(),\n        headless: true,\n        format: 'json',\n        outputDir: '/tmp/paa-output-api',\n        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n      }),\n    )\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    } else if (opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = serpActualCostMc(headlessSentOut.value)\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'serp_refund', 'headless-mode pricing settle')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'serp', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACIA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,SAAS,WAAW,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AACnE;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,kBAA6C,EAAE,OAAO,KAAK;AACjE,UAAM,SAAS,MAAM;AAAA,MACnB,EAAE,IAAI,KAAK,WAAW,SAAS,OAAO,QAAQ,OAAO,IAAI,OAAO,GAAG,gBAAgB;AAAA,MACnF,MAAM,QAAQ;AAAA,QACZ,GAAG;AAAA,QACH,cAAc,qBAAqB;AAAA,QACnC,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF,WAAW,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAClE,YAAM,aAAa,iBAAiB,gBAAgB,KAAK;AACzD,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,eAAe,8BAA8B;AAAA,eACpF,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,QAAQ,KAAK,SAAS,IAAI,KAAK;AAAA,IACtF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md

@@ -1,58 +0,0 @@
-# ADR 0001: In-page API interception for anti-bot scraping (Facebook Ad Library)
-
-- **Status:** Accepted
-- **Date:** 2026-06-03
-- **Deciders:** Andrew (operator), engineering
-- **Applies to:** `facebook_ad_search`, `facebook_page_intel`, and future scrapers of JS-heavy, anti-bot-protected sites
-
-## Context
-
-`facebook_ad_search` was unreliable — it usually returned `soft-block: no results (refunded)`, and when it did return, advertiser names came back `undefined` with `—` for library IDs. Two independent root causes:
-
-1. **DOM scraping a hostile SPA.** The Facebook Ad Library is a React app. The ads are *not* in the initial HTML — the page fires its own background GraphQL request and renders the JSON into DOM cards asynchronously. Our scraper waited for those cards to render, then re-parsed the rendered text with regexes (`See ad details … Sponsored`). That is fragile (breaks on any UI change) and slow, and it conflated "page still loading" with "no results."
-
-2. **A single static proxy.** Every Facebook scrape launched with the static `KERNEL_PROXY_ID`. Facebook flags/rate-limits a repeatedly-used IP and serves a near-empty / login-gated page to it. A real residential browser loaded the same search fine — confirming the block was IP-reputation, not the URL or a login requirement. `facebook_page_intel` worked intermittently for the same reason (same proxy, sometimes flagged).
-
-The "undefined names / — library IDs" was a third, smaller bug: a field-name mismatch between the API response (`pageName`, `sampleLibraryId`) and the MCP formatter (`name`, `libraryId`).
-
-## Decision
-
-**Stop scraping the rendered page. Intercept the JSON the page already fetches, on a browser-service proxy, with the DOM scrape kept only as a fallback.**
-
-Concretely, for the Ad Library:
-
-1. **Intercept the in-page GraphQL response.** The SPA issues `AdLibrarySearchPaginationQuery` (a `doc_id`-based POST to `/api/graphql/`) and Facebook returns the ads as JSON. We attach a Playwright `page.on('response')` listener before navigation and parse the JSON directly (`data.ad_library_main.search_results_connection.edges[].node.collated_results[]`). No request is forged — the page makes the request itself, so it carries Facebook's own session/cookies and looks like a legitimate first-party call. Implemented in `src/extractor/FacebookAdGraphql.ts`.
-2. **Configured proxy by default.** Both FB routes now launch via `kernelLaunchOptsResidential()`, which uses the configured browser-service proxy without city/ZIP targeting by default. Location-targeted residential proxying remains an explicit escalation mode rather than the default.
-3. **DOM scrape as fallback.** If no GraphQL response is captured (e.g., the query shape drifts), the existing regex DOM parse still runs on the already-loaded page.
-4. **Field aliasing for client compatibility.** The API now returns both `pageName`/`name` and `sampleLibraryId`/`libraryId`, so even already-installed MCP clients render correctly without a package update; the formatter was also fixed to read the canonical fields.
-
-This is the **same pattern already proven by the YouTube transcription InnerTube tier** (`CaptionFetcher.ts` does `page.evaluate(fetch('/youtubei/v1/player'…))` from inside the loaded page). This ADR names it as the preferred default for this class of target.
-
-### The reusable pattern
-
-> For a JS-heavy, anti-bot-protected site whose data arrives via an internal API call: load the page on a clean (residential) IP, capture or replay that internal request **from within the page's own session**, parse the JSON, and keep DOM scraping only as a fallback.
-
-Prefer **intercepting the response** (`page.on('response')`) over **replaying the request** when the page fires the call itself — it needs zero token/`doc_id`/`fb_dtsg` reconstruction and is the most robust form.
-
-## Consequences
-
-**Positive**
-- Parses structured JSON, not DOM → immune to CSS/markup changes (the usual scraper breakage).
-- Faster — no waiting for the grid to render/scroll before reading.
-- Unambiguous — an explicit ad list or an error, killing the "0 results = blocked?" guesswork.
-- Lower block rate — an in-session first-party request plus a clean residential IP looks legitimate.
-- Verified live: `Nike` returned 10 distinct, correctly-named advertisers with library IDs (Nike, Jordan, Nordstrom Rack, eBay, Whatnot…), end-to-end through the MCP, no soft-block.
-
-**Negative / risks**
-- The GraphQL `doc_id` and response shape are Facebook-internal and **drift every few months**. Mitigation: the DOM fallback, plus the capture probe technique below to re-derive the shape in minutes.
-- Still not 100% — the Ad Library is adversarial; residential IPs can occasionally be challenged.
-- Captured `doc_id` (`24922295957467452` as of 2026-06-03) is intentionally *not* hardcoded — we match on the `fb_api_req_friendly_name` (`AdLibrarySearchPaginationQuery`) so a `doc_id` rotation alone doesn't break us.
-
-**Maintenance — re-capturing the request shape when it drifts**
-Run a Kernel session on a residential proxy, attach `page.on('request'|'response')`, navigate to the Ad Library search URL, and log GraphQL `fb_api_req_friendly_name` / `doc_id` / `variables` / response keys. (A throwaway `fb-capture.ts` probe was used during this work; recreate it from this description.)
-
-## Alternatives considered
-
-- **Official Graph Ad Library API (`/ads_archive`).** Rejected: for the US it only returns political/issue ads; commercial ads (the use case) aren't exposed outside the EU.
-- **Replay the GraphQL request manually** (rebuild `lsd`/`jazoest`/`__dyn`/`doc_id`/variables). Rejected as the default: more brittle than intercepting the page's own response, though viable for pagination beyond what scrolling triggers.
-- **Just rotate proxies, keep DOM scraping.** Rejected: fixes the block but leaves the fragile, slow DOM parse and the "0 results" ambiguity.

package/docs/adr/0002-hybrid-smart-rag-vault-retrieval.md

@@ -1,62 +0,0 @@
-# ADR 0002: Hybrid Smart RAG retrieval for memory vaults
-
-- **Status:** Accepted
-- **Date:** 2026-07-16
-- **Deciders:** Andrew (operator), engineering
-- **Applies to:** MCP Memory retrieval, memory writes, vault search, and the default Skills vault
-
-## Context
-
-Semantic similarity alone can miss exact tags, dates, metadata, and the note graph that gives an Obsidian vault its useful local context. Expanding every link from every semantic result has the opposite problem: it creates a large, noisy candidate set and can make structural proximity look like evidence of relevance. The connected AI also needs a strong habit of reusing the live tag vocabulary without turning one advisory tool-call order into a persisted write precondition.
-
-Skills add a second vault-specific constraint. A stored skill is not just one note: its internal links may point to reference material, templates, and executable support files, so the vault needs a stable folder contract.
-
-## Decision
-
-**Use bounded hybrid Smart RAG for memory retrieval, and treat tag and graph context as evidence-bearing candidates rather than automatic conclusions.**
-
-The default retrieval flow is:
-
-1. Form **2–4 focused query variants**, with **3** as the default.
-2. Combine exact **tag, metadata, vault, and date** retrieval with semantic retrieval.
-3. Fuse and deduplicate those channels into a total candidate pool of **50**.
-4. Select the top **8** preliminary seeds.
-5. Expand each seed to **depth 1**, considering at most **5** outgoing links or backlinks per seed.
-6. Apply **Jina reranking** to the combined candidates and retain the top **30** for reasoning.
-
-Graph-expanded items remain candidates. They are not automatically relevant and must never become automatic note links; the AI adds an internal link only when the note contents support it.
-
-Before memory search, creation, or update, the MCP instructions strongly direct the AI to inspect the complete accessible tag inventory, reuse existing tags, and add a tag only when the inventory does not contain an adequate equivalent. This is an automatic behavioral directive, **not** a persisted call-order precondition that makes writes fail solely because a separate tag-list call was not recorded.
-
-The default **Skills** vault stores reusable skill packages. Every skill requires a `scripts/` folder and at least one of `references/` or `templates/`; a skill may have both. Skill notes use normal internal links to those supporting files.
-
-## Consequences
-
-**Positive**
-- Exact filters preserve high-precision matches while multi-query semantic retrieval recovers differently worded concepts.
-- Bounded one-hop expansion recovers directly connected context without recursively walking the vault.
-- Jina reranking makes semantic, exact, and graph-derived candidates comparable before the AI reasons over them.
-- The 50-candidate and 30-result bounds provide predictable retrieval and context costs.
-- Tag reuse becomes the default AI behavior without coupling persisted data validity to orchestration traces.
-- Skills have a portable, inspectable support-file layout.
-
-**Negative / risks**
-- Multi-query retrieval, graph expansion, and reranking add latency and Jina dependency cost.
-- A fixed pool of 50 may under-recall unusually broad vaults; 30 reranked notes may still exceed the useful reading budget for narrow questions.
-- Link quality depends on note contents and graph maintenance; a bad link can introduce a weak candidate even though reranking limits its impact.
-- The tag-inventory-first behavior remains advisory at the AI layer and can be skipped by a non-compliant client.
-- Skills that do not need executable logic must still carry a `scripts/` folder to satisfy the vault contract.
-
-## Alternatives considered
-
-- **One semantic query only.** Rejected because it misses exact structured constraints and alternate phrasings.
-- **Expand every link from every retrieved note.** Rejected because candidate growth and graph noise become unbounded; expansion is limited to the strongest eight seeds.
-- **Treat graph neighbors as automatically relevant or automatically link them.** Rejected because structural proximity is not evidence that a note answers the current request.
-- **Persist tag-list call order as a write precondition.** Rejected because it couples durable writes to client orchestration history; forceful instructions provide the intended behavior without rejecting otherwise valid writes.
-
-## Revisit when
-
-- Retrieval evaluation shows that the 50/8/5/30 bounds materially harm recall or precision.
-- Jina reranking becomes unavailable, too costly, or is outperformed by a tested replacement.
-- Vault graphs become dense enough that one-hop expansion regularly adds mostly irrelevant candidates.
-- The Skills packaging contract adopts an external standard that makes `scripts/`, `references/`, or `templates/` incompatible.

package/docs/adr/0003-waive-unrecoverable-scheduled-model-cost.md

@@ -1,22 +0,0 @@
-# ADR-0003: Waive unrecoverable scheduled model cost during repair
-
-**Status:** Accepted
-
-**Date:** 2026-07-22
-
-## Context
-
-An agent-mode scheduled run completed provider/model work on 2026-07-14, but OpenRouter did not return a model-cost value. The 75-Credit base charge was already captured. Its settlement remained `cost_pending`, which correctly prevented an unverified model delta from being guessed but also blocked every later scheduled run for the account. The original authorization token and exact vendor cost are not recoverable.
-
-## Decision
-
-For this historical repair, preserve the captured base charge, waive the unknown model-cost delta, and close the settlement and authorization as settled with an explicit `model_cost_unavailable_waived` audit reason. Do not invent or estimate a vendor cost.
-
-Future runs continue to require exact reported OpenRouter cost under the existing billing contract. A new missing-cost occurrence must retain enough durable provider evidence for reconciliation or be explicitly waived by an audited operator repair; it must never be silently estimated.
-
-## Consequences
-
-- The account is no longer permanently blocked by an unrecoverable historical delta.
-- MCP Scraper absorbs the unknown model expense for this run.
-- The repair remains visible in billing-event metadata and preserves the original base-charge receipt.
-- A productized recovery path is still needed if missing provider cost recurs.

package/docs/adr/README.md

@@ -1,13 +0,0 @@
-# Architecture Decision Records
-
-Short, durable records of *why* a non-obvious technical decision was made — the context, the choice, and the consequences — so future readers (and future us) don't re-litigate or accidentally undo it.
-
-Write one when a decision is hard to reverse, surprising, or encodes a constraint that isn't visible in the code (an anti-bot workaround, a vendor limitation, a deliberate fallback). Don't write one for routine changes.
-
-## Format
-`NNNN-kebab-title.md`, Nygard-style: **Status · Context · Decision · Consequences**. Status is `Proposed` → `Accepted` → (later) `Superseded by ADR-XXXX`. Number sequentially; never renumber.
-
-## Index
-- [0001 — In-page API interception for anti-bot scraping (Facebook Ad Library)](./0001-in-page-graphql-interception-for-anti-bot-scraping.md)
-- [0002 — Hybrid Smart RAG retrieval for memory vaults](./0002-hybrid-smart-rag-vault-retrieval.md)
-- [0003 — Waive unrecoverable scheduled model cost during repair](./0003-waive-unrecoverable-scheduled-model-cost.md)

package/docs/final-tooling-spec.md

@@ -1,206 +0,0 @@
-# Final Tooling Spec — rename, server instructions, description optimization, catalog
-
-Execution blueprint consolidating the decisions reached: (A) server namespace rename
-`mcp-scraper → scraper`, (B) `*_workflow` rename of workflow-class tools, (C) a front-loaded
-**server-instructions block** carrying the cross-tool routing map, (D) a per-tool description
-optimization pass that moves routing *out* of descriptions, and (E) catalog integration.
-
-Grounded in: the SDK's `instructions?: string` option
-(`node_modules/@modelcontextprotocol/sdk/dist/esm/server/index.d.ts:15`), verified tool registrations
-in `src/mcp/paa-mcp-server.ts` and `src/mcp/browser-agent-mcp-server.ts`, and the executor map in
-`src/mcp/http-mcp-tool-executor.ts`.
-
-Governing principle (from `docs/mcp-tool-design-guide.md`): **the model routes on names + the
-server-instructions block before it fetches any schema.** Therefore the cross-tool routing map must
-live in the instructions block, never solely in per-tool descriptions that may never be fetched.
-
----
-
-## Part A — Server namespace rename: `mcp-scraper` → `scraper`
-
-The client prepends `mcp__`, so today every tool is `mcp__mcp-scraper__…` (stutter). Renaming the
-**server name** to `scraper` yields `mcp__scraper__…`.
-
-### A.1 Exact edits — CORRECTED after verification
-
-> **The namespace `mcp__<key>__` comes from the CLIENT REGISTRATION KEY, not the McpServer `name`.**
-> Verified: `src/cli/agent-config.ts` registers the server with the key `mcp-scraper` via
-> `claude mcp add mcp-scraper …` and the Desktop config block `'mcp-scraper': {…}`. THAT key is what
-> Claude turns into `mcp__mcp-scraper__*`. Changing only `new McpServer({name})` does NOT change the
-> prefix. The full set of edits to actually rename the namespace:
-
-- `src/cli/agent-config.ts:54` — `['mcp', 'remove', 'mcp-scraper', …]` → `'scraper'`.
-- `src/cli/agent-config.ts:58` — `['mcp', 'add', 'mcp-scraper', …]` → `'scraper'`.
-- `src/cli/agent-config.ts:75` — Desktop config key `'mcp-scraper': {…}` → `'scraper': {…}`.
-- `scripts/build-mcpb.mjs:61` — `.mcpb` manifest server `name: 'mcp-scraper'` → `'scraper'` (Desktop
-  extension registration key). Line 50 (`mcp-scraper-desktop-extension`) is the extension *package* id; leave it.
-- `src/mcp/paa-mcp-server.ts:152` and `bin/mcp-scraper-combined-stdio-server.ts:61` — the McpServer
-  `name` is cosmetic self-description; update to `'scraper'` for consistency but it is NOT what drives the prefix.
-- **Tests:** `tests/unit/cli-agent-config.test.ts:32` asserts `'mcp-scraper'` — update.
-- **Migration is harder than a code change:** existing users already have the server registered under
-  `mcp-scraper` in *their* client config. Renaming the key here only affects NEW installs; existing
-  installs keep `mcp__mcp-scraper__*` until they `claude mcp remove mcp-scraper && claude mcp add scraper`
-  (or reinstall the `.mcpb`). Document this explicitly; consider shipping a migration note/command.
-
-### A.2 Leave alone (URL / artifact stability)
-- Bundle filenames/paths in `build-mcpb.mjs` (`mcp-scraper.mcpb`, `mcp-scraper-${version}.mcpb`,
-  `build/mcpb/mcp-scraper`, `dist/bin/mcp-scraper-combined-stdio-server.js`) — **keep**. These are
-  download URLs and build paths; changing them breaks existing download links for no namespace benefit.
-- Repo name, GitHub URLs — keep.
-
-### A.3 Breaking-change handling (REQUIRED — do not skip)
-Renaming the server changes **every tool identifier** clients see → invalidates:
-- User permission allowlists referencing `mcp__mcp-scraper__*`.
-- Any saved agent configs / scripted tool calls by full name.
-
-Mitigations:
-- **Do it in a single release, called out in the changelog** as a breaking namespace change with a
-  one-line migration ("re-allow tools; the namespace is now `mcp__scraper__…`").
-- Bump a **minor/major** version, not a patch.
-- Ship in the same release as Parts B–D so users re-onboard once, not repeatedly.
-
----
-
-## Part B — Workflow-class tool renames (`*_workflow` suffix)
-
-Rule: a tool whose mental model is "run a multi-step workflow" carries `_workflow` so the keyword is in
-the name (matches a user saying "run the … workflow / research / directory"). Engine plumbing
-(`workflow_run/step/status/list/suggest/artifact_read`) is **not** renamed — it's infrastructure, not a
-user-named capability.
-
-### B.1 Rename map
-| Current name | New name | File(s) |
-|---|---|---|
-| `rank_tracker_blueprint` | `rank_tracker_workflow` | `paa-mcp-server.ts:346`, schema, manifest, catalog |
-| `browser_capture_fanout` | `query_fanout_workflow` | `browser-agent-mcp-server.ts:791`, `browser-agent-tool-schemas.ts:189`, manifest, catalog |
-| `directory_workflow` | *(unchanged — already correct)* | — |
-| *(new, optional)* | `deep_research_workflow` | thin wrapper → `workflow_run` for the `deep-research` workflow; register in `paa-mcp-server.ts` |
-
-### B.2 Per-tool change checklist (apply to each renamed tool)
-1. **Registration** — `server.registerTool('<new_name>', { … })` (the string literal).
-2. **Input schema** — rename the exported `…InputSchema` only if its name encodes the old tool name;
-   the schema *shape* is unchanged.
-3. **Manifest** — `src/mcp/mcp-tool-manifest.ts` `entry({ name: '<new_name>', … })`.
-4. **Executor** — `src/mcp/http-mcp-tool-executor.ts` method name only if it mirrors the tool name; the
-   **REST route string is unchanged** (rename is MCP-surface only, not the backend).
-5. **Formatter** — `format<Tool>` function name optional; wiring in `registerTool` handler updated.
-6. **Catalog** — update `mcp_name` in `docs/tool-catalog-spec.md` §3.2/§9 and the seed.
-7. **Lint/manifest generators** — re-run `npm run mcp:manifest` and `npm run mcp:lint-tools` to
-   regenerate `docs/mcp-tool-manifest.generated.*`.
-
-### B.3 Optional backward-compat (recommended for 1–2 releases)
-Register the **old name as a hidden alias** pointing at the same handler, so in-flight callers don't
-hard-break: `server.registerTool('<old_name>', { ...sameConfig, annotations: { deprecated: true } }, handler)`.
-Remove after one release. If you'd rather hard-cut, fold it into the Part A breaking release.
-
----
-
-## Part C — Server-instructions block (the pre-fetch routing map)
-
-This is the highest-leverage change: a short string loaded **at session start** (alongside tool names),
-so the model routes correctly *before* fetching any schema.
-
-### C.1 New file — `src/mcp/server-instructions.ts`
-```ts
-export const SERVER_INSTRUCTIONS = `
-This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
-this routing map; load its schema before calling.
-
-ROUTING
-- One web page → extract_url. Whole site (crawl + SEO) → extract_site. Just the URL list → map_site_urls.
-- Web search (SERP) → search_serp. Full SERP + People-Also-Ask detail → harvest_paa.
-- Google Maps: find places → maps_search; one place deep-dive + reviews → maps_place_intel.
-- YouTube: find/list videos → youtube_harvest; transcribe one → youtube_transcribe.
-- Facebook: find ads → facebook_ad_search; transcribe an ad → facebook_ad_transcribe; transcribe a
-  video → facebook_video_transcribe; page profile → facebook_page_intel.
-- Instagram: profile inventory → instagram_profile_content; one post/reel → instagram_media_download.
-- Workflows (multi-step): local directory → directory_workflow; rank blueprint → rank_tracker_workflow;
-  AI-answer citation fan-out (AEO) → query_fanout_workflow; deep research → deep_research_workflow.
-- Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) → the browser_* agent
-  (browser_open then navigate/read), and capture profiles via browser_profile_*.
-
-NOTES
-- Bulk/full-site crawls: call extract_site with rotateProxies:true for blocked/rate-limited sites; it
-  discovers URLs and returns a saved folder/artifact, not the full content inline.
-- Tools that open a browser session or transcribe media cost credits and take longer — prefer the
-  cheapest tool that answers the question (plain search/extract before browser agents).
-- Results may save large output to disk/artifact and return a summary + path; read the path for detail.
-`.trim()
-```
-> Keep it **cross-tool only** — no per-tool parameter detail (that's the fetched schema's job). Use
-> neutral capability language; never name the underlying browser/proxy vendor (concealment policy).
-
-### C.2 Wiring (both server construction sites)
-- `src/mcp/paa-mcp-server.ts:152`:
-  `new McpServer({ name: 'scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })`
-- `bin/mcp-scraper-combined-stdio-server.ts:61`: same second argument.
-- Import `SERVER_INSTRUCTIONS` from `../src/mcp/server-instructions.js` (and `./server-instructions.js`).
-
-### C.3 The insight, enforced
-Per-tool descriptions must **NOT** be the only home for "which tool for which job." That guidance moves
-to `SERVER_INSTRUCTIONS` (always loaded). Per-tool descriptions keep only a **single** negative-space
-line for post-fetch tie-breaking (see Part D). This prevents the failure where the routing hint lives in
-a description the model never fetched.
-
----
-
-## Part D — Tool-description optimization pass
-
-For every tool in `mcp-tool-schemas.ts` / `paa-mcp-server.ts` `registerTool` descriptions, rewrite to
-this shape (apply the `docs/mcp-tool-design-guide.md` checklist):
-
-1. **Line 1 = when-to-use**, in the user's words. ("Transcribe one YouTube video to text…")
-2. **Line 2 = side-effects/cost** if any. ("Opens a browser session; costs credits per minute.")
-3. **Params**: each param's `.describe()` states intent + bounds + default, mirroring the Zod schema.
-4. **One** negative-space line max. ("For a whole site use extract_site.") — the *broad* map lives in
-   §C, so do not enumerate the full tool family here.
-5. **Delete** redundant cross-tool prose and marketing. Shorter descriptions load faster and rank cleaner.
-
-Concrete trims to make now:
-- `extract_url` / `extract_site` / `map_site_urls`: each currently restates the others; cut to one
-  negative-space line each (the trio map is in §C).
-- `search_serp` vs `harvest_paa`: both hit `/harvest/sync`; clarify in one line each that
-  search_serp = organic results only, harvest_paa = + PAA/AI-Overview/SERP features.
-- Renamed workflow tools: rewrite line 1 to lead with the keyword now in the name.
-
-Re-run `npm run mcp:lint-tools` (the craft linter) after edits; fix any flagged tools.
-
----
-
-## Part E — Catalog integration
-
-Reconcile `docs/tool-catalog-spec.md` with the renames, and ship the **static v1** (no DB) first:
-
-- Update catalog `mcp_name` values: `rank_tracker_blueprint→rank_tracker_workflow`,
-  `browser_capture_fanout→query_fanout_workflow`; add `deep_research_workflow`.
-- Categories stay **platform-first** (YouTube / Facebook / Instagram / Google Search / Google Maps /
-  Websites / Browser / Workflows), with **Reddit** served by the Browser category (no dedicated tool) —
-  or add a `reddit_*` named wrapper later if AI routing for Reddit is wanted (per the design guide).
-- **v1 implementation = static**: `src/api/catalog-seed.ts` exports `CATALOG`; `GET /catalog` in
-  `server.ts` returns it. Defer the 6-table DB schema (§2 of the catalog spec) to v2 (runtime editing).
-- Catalog params must mirror the live Zod schemas (field names/bounds) so the frontend posts valid bodies.
-
----
-
-## Part F — Build order
-1. **Part C** (server instructions) — highest leverage, lowest risk, non-breaking. Ship first.
-2. **Part D** (description optimization) — non-breaking; pairs with C (move routing out → into C).
-3. **Part B** (workflow renames) — with hidden aliases (B.3) so it's non-breaking.
-4. **Part A** (server rename) — the one breaking change; batch into a single versioned release with a
-   changelog migration note. Run after B/C/D so users re-onboard once.
-5. **Part E** static catalog — independent; any time.
-6. Regenerate `docs/mcp-tool-manifest.generated.*`; run `npm run mcp:lint-tools`; full test + live tests;
-   then the release flow (3 version files, Vercel prod first, npm publish, rebuild `.mcpb`).
-
-## Part G — Verification checklist
-- [ ] `grep -rn "mcp-scraper'" src bin scripts` → only intentional (bundle filenames), server `name` = `scraper`.
-- [ ] No tool identifier appears in two namespaces; `npm run mcp:manifest` regenerated.
-- [ ] `SERVER_INSTRUCTIONS` loads on `initialize` (verify via an MCP inspector / the live protocol test).
-- [ ] Every renamed tool: registration + manifest + catalog `mcp_name` updated; old name aliased or cut.
-- [ ] No per-tool description contains the full cross-tool map (only one negative-space line).
-- [ ] No vendor name (browser/proxy provider) in instructions or any description.
-- [ ] `npm run mcp:lint-tools` clean; `vitest run tests/unit tests/contract` green.
-
-## Non-goals (this spec)
-DB-backed catalog (v2), the `reddit_*` / `deep_research_workflow` *implementations* beyond thin
-registration, per-plan tool gating, and the frontend rendering itself.