mcp-scraper 0.21.3 → 0.21.4

package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js}

@@ -10,10 +10,10 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-XDRHBGJR.js";
-import "./chunk-OBNKOYOE.js";
+} from "./chunk-PNBYYAEK.js";
+import "./chunk-JHI373VB.js";
 import "./chunk-M2S27J6Z.js";
-import "./chunk-H7APH5UG.js";
+import "./chunk-ICT7DDHL.js";
 export {
   completeExtractJob,
   countSuccessfulPages,
@@ -27,4 +27,4 @@ export {
   setExtractJobTotal,
   settleExtractJob
 };
-//# sourceMappingURL=site-extract-repository-FDH6ZXKT.js.map
+//# sourceMappingURL=site-extract-repository-2QQSL2QC.js.map

package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js}

@@ -3,19 +3,20 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-2P5VFIJM.js";
+} from "./chunk-SDHPQTFM.js";
 import {
   harvest
-} from "./chunk-JJ5RMEGX.js";
+} from "./chunk-E5J4HJBO.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-5IU5ZFVO.js";
+} from "./chunk-MTSBI7ZH.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-XGIPATLV.js";
 import {
-  MC_COSTS
-} from "./chunk-OBNKOYOE.js";
+  MC_COSTS,
+  serpActualCostMc
+} from "./chunk-JHI373VB.js";
 import "./chunk-M2S27J6Z.js";
 import {
   claimPendingJob,
@@ -24,7 +25,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-H7APH5UG.js";
+} from "./chunk-ICT7DDHL.js";
 
 // src/api/webhook.ts
 async function deliverWebhook(url, payload, retries = 3) {
@@ -82,7 +83,7 @@ async function processJob(job) {
       if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
       else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
     } else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
-      const actualCost = headlessSentOut.value === true ? MC_COSTS.serp_headless : MC_COSTS.serp_headful;
+      const actualCost = serpActualCostMc(headlessSentOut.value);
       const diff = opts.billingHoldMc - actualCost;
       if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
       else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
@@ -138,4 +139,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-BXJQ7J4N.js.map
+//# sourceMappingURL=worker-RV2TYXE3.js.map

package/dist/worker-RV2TYXE3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS, serpActualCostMc } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\nimport { runWithCostContext } from './cost-context.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const headlessSentOut: { value: boolean | null } = { value: null }\n    const result = await runWithCostContext(\n      { op: opts.serpOnly ? 'serp' : 'paa', userId: Number(job.user_id), headlessSentOut },\n      () => harvest({\n        ...opts,\n        kernelApiKey: browserServiceApiKey(),\n        headless: true,\n        format: 'json',\n        outputDir: '/tmp/paa-output-api',\n        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n      }),\n    )\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    } else if (opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = serpActualCostMc(headlessSentOut.value)\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'serp_refund', 'headless-mode pricing settle')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'serp', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result, attempts })\n    }\n  } catch (err) {\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', ...harvestProblemResponse(problem), attempts })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACIA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,SAAS,WAAW,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AACnE;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,kBAA6C,EAAE,OAAO,KAAK;AACjE,UAAM,SAAS,MAAM;AAAA,MACnB,EAAE,IAAI,KAAK,WAAW,SAAS,OAAO,QAAQ,OAAO,IAAI,OAAO,GAAG,gBAAgB;AAAA,MACnF,MAAM,QAAQ;AAAA,QACZ,GAAG;AAAA,QACH,cAAc,qBAAqB;AAAA,QACnC,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF,WAAW,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAClE,YAAM,aAAa,iBAAiB,gBAAgB,KAAK;AACzD,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,eAAe,8BAA8B;AAAA,eACpF,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,QAAQ,KAAK,SAAS,IAAI,KAAK;AAAA,IACtF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AAAA,IAC7F;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,GAAG,uBAAuB,OAAO,GAAG,SAAS,CAAC;AAAA,IAC3H;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/docs/mcp-tool-manifest.generated.json

@@ -1,9 +1,9 @@
 {
-  "generatedAt": "2026-07-14T20:38:01.309Z",
+  "generatedAt": "2026-07-14T23:07:56.595Z",
   "generatedFrom": "dist/bin/mcp-stdio-server.js",
   "serverInfo": {
     "name": "mcp-scraper",
-    "version": "0.21.3"
+    "version": "0.21.4"
   },
   "counts": {
     "unified_stdio": 159
@@ -4264,7 +4264,7 @@
     {
       "name": "capture_serp_page_snapshots",
       "title": "SERP Intelligence Page Snapshots",
-      "description": "Capture public ranking pages as SERP Intelligence page snapshots — persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
+      "description": "Capture public ranking pages as SERP Intelligence page snapshots — persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -4276,7 +4276,7 @@
             },
             "minItems": 1,
             "maxItems": 25,
-            "description": "Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."
+            "description": "Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."
           },
           "targets": {
             "type": "array",
@@ -4313,7 +4313,7 @@
             },
             "minItems": 1,
             "maxItems": 25,
-            "description": "Structured targets. Use instead of urls when source kind or position should be preserved."
+            "description": "Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."
           },
           "maxConcurrency": {
             "type": "integer",
@@ -4335,9 +4335,6 @@
             "description": "Include sanitized browser/proxy diagnostics."
           }
         },
-        "required": [
-          "urls"
-        ],
         "additionalProperties": false,
         "$schema": "http://json-schema.org/draft-07/schema#"
       },
@@ -4355,10 +4352,18 @@
             "type": "integer",
             "minimum": 0
           },
+          "capturedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
           "failedCount": {
             "type": "integer",
             "minimum": 0
           },
+          "skippedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
           "captures": {
             "type": "array",
             "items": {
@@ -4366,6 +4371,13 @@
               "additionalProperties": {}
             }
           },
+          "attempts": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": {}
+            }
+          },
           "resolvedInputs": {
             "type": "object",
             "additionalProperties": {}
@@ -4390,8 +4402,11 @@
           "schemaVersion",
           "status",
           "count",
+          "capturedCount",
           "failedCount",
+          "skippedCount",
           "captures",
+          "attempts",
           "resolvedInputs",
           "diagnostics",
           "providerPayload"
@@ -4413,7 +4428,7 @@
     {
       "name": "capture_serp_snapshot",
       "title": "SERP Intelligence Snapshot",
-      "description": "Capture a structured SERP Intelligence snapshot of a Google query — the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
+      "description": "Capture a structured SERP Intelligence snapshot of a Google query — the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
       "inputSchema": {
         "type": "object",
         "properties": {
@@ -4477,14 +4492,14 @@
           "includePageSnapshots": {
             "type": "boolean",
             "default": false,
-            "description": "Also capture ranking-page snapshots for selected SERP URLs."
+            "description": "Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."
           },
           "pageSnapshotLimit": {
             "type": "integer",
             "minimum": 0,
             "maximum": 10,
             "default": 0,
-            "description": "Maximum ranking-page snapshots when includePageSnapshots is true."
+            "description": "Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded."
           }
         },
         "required": [
@@ -4526,6 +4541,22 @@
               }
             ]
           },
+          "organicResultCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "localPackResultCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "pageSnapshotCount": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "pageSnapshotFailedCount": {
+            "type": "integer",
+            "minimum": 0
+          },
           "snapshotId": {
             "$ref": "#/properties/query"
           },
@@ -4540,6 +4571,48 @@
               "additionalProperties": {}
             }
           },
+          "attempts": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": {}
+            }
+          },
+          "billing": {
+            "anyOf": [
+              {
+                "type": "object",
+                "properties": {
+                  "creditsUsed": {
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "null"
+                      }
+                    ]
+                  },
+                  "requestId": {
+                    "$ref": "#/properties/query"
+                  },
+                  "jobId": {
+                    "$ref": "#/properties/query"
+                  }
+                },
+                "required": [
+                  "creditsUsed",
+                  "requestId",
+                  "jobId"
+                ],
+                "additionalProperties": false
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
           "diagnostics": {
             "anyOf": [
               {
@@ -4563,9 +4636,15 @@
           "location",
           "capturedAt",
           "resultCount",
+          "organicResultCount",
+          "localPackResultCount",
+          "pageSnapshotCount",
+          "pageSnapshotFailedCount",
           "snapshotId",
           "resolvedInputs",
           "artifacts",
+          "attempts",
+          "billing",
           "diagnostics",
           "providerPayload"
         ],

package/docs/specs/oauth-mcp-spec.md

@@ -77,7 +77,7 @@ export async function resolveOAuthUser(identity: OAuthIdentity): Promise<User |
 Resolution order (each step returns on hit):
 1. `getUserByOauthSubject(identity.subject)` — fast path.
 2. If `identity.email`: `getUserByEmail(email)` → if found, `linkOauthSubject(user.id, identity.subject)` and return (links existing dashboard accounts to their Google identity by verified email — WorkOS only emits verified emails).
-3. Auto-provision: `createUser({ email, name: null })` (existing helper; generates `api_key`), then `grantSignupCredit(user.id)` (existing, idempotent), then `linkOauthSubject`. OAuth signups get the same 500-credit welcome grant as dashboard signups.
+3. Auto-provision: `createUser({ email, name: null })` (existing helper; generates `api_key`), then `linkOauthSubject`. Signup grants are retired; OAuth and dashboard accounts both choose a paid plan before metered tool use.
 4. No email claim and no subject match → return `null` (401; we never create anonymous accounts).
 
 ## 4. DB changes — `src/api/db.ts`
@@ -178,7 +178,7 @@ Self-mint tokens with `jose.generateKeyPair('RS256')` + a local JWKS served via
 
 ### `tests/unit/oauth-resolve-user.test.ts`
 
-Mock the db helpers. Cases: subject fast-path; email-link path (asserts `linkOauthSubject` called once); auto-provision path (asserts `createUser` + `grantSignupCredit` + `linkOauthSubject`); no-email no-match → null.
+Mock the db helpers. Cases: subject fast-path; email-link path (asserts `linkOauthSubject` called once); auto-provision path (asserts `createUser` + `linkOauthSubject` and no credit grant); no-email no-match → null.
 
 ### `tests/unit/oauth-metadata.test.ts`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.21.3",
+  "version": "0.21.4",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",

package/dist/chunk-5IU5ZFVO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/lib/browser-service-env.ts","../src/api/cost-context.ts","../src/api/cost-telemetry.ts","../src/api/cost-rates.ts"],"sourcesContent":["export function browserServiceApiKey(): string | undefined {\n  const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim()\n  return value || undefined\n}\n\nexport function browserServiceProxyId(): string | undefined {\n  return undefined\n}\n\nexport function browserServiceProfileName(): string | undefined {\n  const value = (\n    process.env.BROWSER_AGENT_PROFILE_NAME ??\n    process.env.BROWSER_SERVICE_PROFILE_NAME ??\n    process.env.KERNEL_BROWSER_PROFILE_NAME ??\n    process.env.KERNEL_PROFILE_NAME\n  )?.trim()\n  return value || undefined\n}\n\nexport function browserServiceProfileSaveChanges(): boolean | undefined {\n  const value = (\n    process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ??\n    process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ??\n    process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ??\n    process.env.KERNEL_PROFILE_SAVE_CHANGES\n  )?.trim().toLowerCase()\n  if (!value) return undefined\n  if (['1', 'true', 'yes', 'on'].includes(value)) return true\n  if (['0', 'false', 'no', 'off'].includes(value)) return false\n  return undefined\n}\n","import { AsyncLocalStorage } from 'node:async_hooks'\n\nexport interface CostContext {\n  op: string\n  probeRunId?: string | null\n  userId?: number | null\n  forceHeadless?: boolean\n  forceHeadful?: boolean\n  subOp?: string | null\n  headlessSentOut?: { value: boolean | null }\n}\n\nconst storage = new AsyncLocalStorage<CostContext>()\n\nexport function runWithCostContext<T>(ctx: CostContext, fn: () => T): T {\n  return storage.run(ctx, fn)\n}\n\nexport function currentCostContext(): CostContext | undefined {\n  return storage.getStore()\n}\n","import { randomUUID } from 'node:crypto'\nimport { getDb } from './db.js'\nimport { currentCostContext } from './cost-context.js'\nimport { kernelCostUsd, vendorCostUsd } from './cost-rates.js'\n\nlet _ready = false\n\nexport async function migrateCostTelemetry(): Promise<void> {\n  if (_ready) return\n  const db = getDb()\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS kernel_session_log (\n      id                    TEXT PRIMARY KEY,\n      kernel_session_id     TEXT,\n      op                    TEXT,\n      source                TEXT NOT NULL,\n      probe_run_id          TEXT,\n      user_id               INTEGER,\n      stealth               INTEGER,\n      headless_sent         INTEGER,\n      proxy_used            INTEGER,\n      fallback              INTEGER NOT NULL DEFAULT 0,\n      opened_at             TEXT,\n      closed_at             TEXT,\n      duration_ms           INTEGER NOT NULL DEFAULT 0,\n      est_cost_usd_headless REAL NOT NULL DEFAULT 0,\n      est_cost_usd_headful  REAL NOT NULL DEFAULT 0,\n      error                 TEXT,\n      created_at            TEXT NOT NULL DEFAULT (datetime('now'))\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)`)\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE kernel_session_log ADD COLUMN method TEXT`) } catch { void 0 }\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS vendor_usage_log (\n      id           TEXT PRIMARY KEY,\n      op           TEXT,\n      probe_run_id TEXT,\n      user_id      INTEGER,\n      vendor       TEXT NOT NULL,\n      model        TEXT,\n      units        REAL NOT NULL DEFAULT 0,\n      unit_type    TEXT,\n      est_cost_usd REAL NOT NULL DEFAULT 0,\n      error        TEXT,\n      created_at   TEXT NOT NULL DEFAULT (datetime('now'))\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)`)\n  await db.execute(`CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)`)\n  try { await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN method TEXT`) } catch { void 0 }\n  await db.execute(`\n    CREATE TABLE IF NOT EXISTS cost_probe_runs (\n      id                            TEXT PRIMARY KEY,\n      tool                          TEXT NOT NULL,\n      target                        TEXT,\n      started_at                    TEXT NOT NULL DEFAULT (datetime('now')),\n      wall_ms                       INTEGER NOT NULL DEFAULT 0,\n      success                       INTEGER NOT NULL DEFAULT 0,\n      error                         TEXT,\n      http_only                     INTEGER NOT NULL DEFAULT 0,\n      kernel_used                   INTEGER NOT NULL DEFAULT 0,\n      kernel_sessions               INTEGER NOT NULL DEFAULT 0,\n      kernel_seconds_total          REAL NOT NULL DEFAULT 0,\n      kernel_tier_observed          TEXT,\n      est_kernel_cost_usd_headless  REAL NOT NULL DEFAULT 0,\n      est_kernel_cost_usd_headful   REAL NOT NULL DEFAULT 0,\n      vendor_cost_usd               REAL NOT NULL DEFAULT 0,\n      charged_credits               REAL NOT NULL DEFAULT 0,\n      margin_usd_headless_starter   REAL NOT NULL DEFAULT 0,\n      margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,\n      notes                         TEXT\n    )\n  `)\n  await db.execute(`CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)`)\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN units REAL`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT`) } catch { void 0 }\n  try { await db.execute(`ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT`) } catch { void 0 }\n  _ready = true\n}\n\nexport interface KernelSessionRecord {\n  kernelSessionId?: string | null\n  source: string\n  stealth?: boolean | null\n  headlessSent?: boolean | null\n  proxyUsed?: boolean | null\n  proxySource?: string | null\n  proxyType?: string | null\n  fallback?: boolean\n  openedAtMs: number\n  closedAtMs: number\n  error?: string | null\n}\n\nexport async function recordKernelSession(r: KernelSessionRecord): Promise<void> {\n  try {\n    await migrateCostTelemetry()\n    const ctx = currentCostContext()\n    const durationMs = Math.max(0, r.closedAtMs - r.openedAtMs)\n    const db = getDb()\n    await db.execute({\n      sql: `INSERT INTO kernel_session_log\n        (id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)\n        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,\n      args: [\n        randomUUID(),\n        r.kernelSessionId ?? null,\n        ctx?.op ?? null,\n        r.source,\n        ctx?.probeRunId ?? null,\n        ctx?.userId ?? null,\n        boolToInt(r.stealth),\n        boolToInt(r.headlessSent),\n        boolToInt(r.proxyUsed),\n        r.proxySource ?? null,\n        r.proxyType ?? null,\n        r.fallback ? 1 : 0,\n        new Date(r.openedAtMs).toISOString(),\n        new Date(r.closedAtMs).toISOString(),\n        durationMs,\n        kernelCostUsd(durationMs, false),\n        kernelCostUsd(durationMs, true),\n        r.error ?? null,\n        ctx?.subOp ?? null,\n      ],\n    })\n  } catch (err) {\n    console.warn('[cost-telemetry] recordKernelSession failed:', err instanceof Error ? err.message : String(err))\n  }\n}\n\nexport interface VendorUsageRecord {\n  vendor: string\n  model?: string | null\n  units: number\n  unitType: string\n  error?: string | null\n}\n\nexport async function recordVendorUsage(r: VendorUsageRecord): Promise<void> {\n  try {\n    await migrateCostTelemetry()\n    const ctx = currentCostContext()\n    const db = getDb()\n    await db.execute({\n      sql: `INSERT INTO vendor_usage_log\n        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method)\n        VALUES (?,?,?,?,?,?,?,?,?,?,?)`,\n      args: [\n        randomUUID(),\n        ctx?.op ?? null,\n        ctx?.probeRunId ?? null,\n        ctx?.userId ?? null,\n        r.vendor,\n        r.model ?? null,\n        r.units,\n        r.unitType,\n        vendorCostUsd(r.vendor, r.units),\n        r.error ?? null,\n        ctx?.subOp ?? null,\n      ],\n    })\n  } catch (err) {\n    console.warn('[cost-telemetry] recordVendorUsage failed:', err instanceof Error ? err.message : String(err))\n  }\n}\n\nexport interface ProbeRunRecord {\n  id: string\n  tool: string\n  target?: string | null\n  wallMs: number\n  success: boolean\n  error?: string | null\n  httpOnly: boolean\n  kernelUsed: boolean\n  kernelSessions: number\n  kernelSecondsTotal: number\n  kernelTierObserved?: string | null\n  estKernelCostUsdHeadless: number\n  estKernelCostUsdHeadful: number\n  vendorCostUsd: number\n  chargedCredits: number\n  marginUsdHeadlessStarter: number\n  marginUsdHeadfulStarter: number\n  units?: number | null\n  unitType?: string | null\n  mode?: string | null\n  notes?: string | null\n}\n\nexport async function recordProbeRun(r: ProbeRunRecord): Promise<void> {\n  await migrateCostTelemetry()\n  const db = getDb()\n  await db.execute({\n    sql: `INSERT INTO cost_probe_runs\n      (id, tool, target, wall_ms, success, error, http_only, kernel_used, kernel_sessions, kernel_seconds_total, kernel_tier_observed, est_kernel_cost_usd_headless, est_kernel_cost_usd_headful, vendor_cost_usd, charged_credits, margin_usd_headless_starter, margin_usd_headful_starter, units, unit_type, mode, notes)\n      VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,\n    args: [\n      r.id, r.tool, r.target ?? null, r.wallMs, r.success ? 1 : 0, r.error ?? null,\n      r.httpOnly ? 1 : 0, r.kernelUsed ? 1 : 0, r.kernelSessions, r.kernelSecondsTotal,\n      r.kernelTierObserved ?? null, r.estKernelCostUsdHeadless, r.estKernelCostUsdHeadful,\n      r.vendorCostUsd, r.chargedCredits, r.marginUsdHeadlessStarter, r.marginUsdHeadfulStarter,\n      r.units ?? null, r.unitType ?? null, r.mode ?? null, r.notes ?? null,\n    ],\n  })\n}\n\nfunction boolToInt(v: boolean | null | undefined): number | null {\n  if (v === null || v === undefined) return null\n  return v ? 1 : 0\n}\n","export const KERNEL_HEADLESS_USD_PER_SEC = 0.0000166667\nexport const KERNEL_HEADFUL_USD_PER_SEC = 0.0001333336\n\nexport const HEADLESS_OPS = new Set<string>(['serp', 'maps_search', 'yt_channel', 'yt_transcription', 'fb_search', 'fb_ad', 'instagram'])\n\nexport const FAL_WIZPER_USD_PER_COMPUTE_SEC = 0.00111\nexport const FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4 // calibrated from 2 real Fal invoices (4.36x, 6.18x wall-clock vs actual)\nexport const DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 0.0004\n\nexport function kernelCostUsd(ms: number, headful: boolean): number {\n  const sec = Math.max(0, ms) / 1000\n  return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC)\n}\n\nexport function vendorCostUsd(vendor: string, units: number): number {\n  if (vendor === 'fal_wizper') {\n    const estimatedComputeSec = Math.max(0, units) / FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR\n    return estimatedComputeSec * FAL_WIZPER_USD_PER_COMPUTE_SEC\n  }\n  if (vendor === 'deepinfra_qwen') return (Math.max(0, units) / 1000) * DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS\n  if (vendor === 'openrouter') return Math.max(0, units) // units IS the real USD cost OpenRouter reported, not a rate-multiplied estimate\n  if (vendor === 'mcp_memory_video') return Math.max(0, units) // units IS the real costUsd mcp-memory's video pipeline reported (OpenRouter + transcription)\n  if (vendor === 'mcp_memory_ai') return Math.max(0, units) // units IS the real costUsd mcp-memory reported via /ai-debit\n  return 0\n}\n"],"mappings":";;;;;AAAO,SAAS,uBAA2C;AACzD,QAAM,SAAS,QAAQ,IAAI,2BAA2B,QAAQ,IAAI,iBAAiB,KAAK;AACxF,SAAO,SAAS;AAClB;AAEO,SAAS,wBAA4C;AAC1D,SAAO;AACT;AAEO,SAAS,4BAAgD;AAC9D,QAAM,SACJ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,gCACZ,QAAQ,IAAI,+BACZ,QAAQ,IAAI,sBACX,KAAK;AACR,SAAO,SAAS;AAClB;AAEO,SAAS,mCAAwD;AACtE,QAAM,SACJ,QAAQ,IAAI,sCACZ,QAAQ,IAAI,wCACZ,QAAQ,IAAI,uCACZ,QAAQ,IAAI,8BACX,KAAK,EAAE,YAAY;AACtB,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,CAAC,KAAK,QAAQ,OAAO,IAAI,EAAE,SAAS,KAAK,EAAG,QAAO;AACvD,MAAI,CAAC,KAAK,SAAS,MAAM,KAAK,EAAE,SAAS,KAAK,EAAG,QAAO;AACxD,SAAO;AACT;;;AC9BA,SAAS,yBAAyB;AAYlC,IAAM,UAAU,IAAI,kBAA+B;AAE5C,SAAS,mBAAsB,KAAkB,IAAgB;AACtE,SAAO,QAAQ,IAAI,KAAK,EAAE;AAC5B;AAEO,SAAS,qBAA8C;AAC5D,SAAO,QAAQ,SAAS;AAC1B;;;ACpBA,SAAS,kBAAkB;;;ACApB,IAAM,8BAA8B;AACpC,IAAM,6BAA6B;AAEnC,IAAM,eAAe,oBAAI,IAAY,CAAC,QAAQ,eAAe,cAAc,oBAAoB,aAAa,SAAS,WAAW,CAAC;AAEjI,IAAM,iCAAiC;AACvC,IAAM,6CAA6C;AACnD,IAAM,uCAAuC;AAE7C,SAAS,cAAc,IAAY,SAA0B;AAClE,QAAM,MAAM,KAAK,IAAI,GAAG,EAAE,IAAI;AAC9B,SAAO,OAAO,UAAU,6BAA6B;AACvD;AAEO,SAAS,cAAc,QAAgB,OAAuB;AACnE,MAAI,WAAW,cAAc;AAC3B,UAAM,sBAAsB,KAAK,IAAI,GAAG,KAAK,IAAI;AACjD,WAAO,sBAAsB;AAAA,EAC/B;AACA,MAAI,WAAW,iBAAkB,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,MAAQ;AACtE,MAAI,WAAW,aAAc,QAAO,KAAK,IAAI,GAAG,KAAK;AACrD,MAAI,WAAW,mBAAoB,QAAO,KAAK,IAAI,GAAG,KAAK;AAC3D,MAAI,WAAW,gBAAiB,QAAO,KAAK,IAAI,GAAG,KAAK;AACxD,SAAO;AACT;;;ADnBA,IAAI,SAAS;AAEb,eAAsB,uBAAsC;AAC1D,MAAI,OAAQ;AACZ,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAoBhB;AACD,QAAM,GAAG,QAAQ,4EAA4E;AAC7F,QAAM,GAAG,QAAQ,yFAAyF;AAC1G,QAAM,GAAG,QAAQ,yFAAyF;AAC1G,MAAI;AAAE,UAAM,GAAG,QAAQ,6DAA6D;AAAA,EAAE,QAAQ;AAAA,EAAS;AACvG,MAAI;AAAE,UAAM,GAAG,QAAQ,2DAA2D;AAAA,EAAE,QAAQ;AAAA,EAAS;AACrG,MAAI;AAAE,UAAM,GAAG,QAAQ,uDAAuD;AAAA,EAAE,QAAQ;AAAA,EAAS;AACjG,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAchB;AACD,QAAM,GAAG,QAAQ,wEAAwE;AACzF,QAAM,GAAG,QAAQ,qFAAqF;AACtG,QAAM,GAAG,QAAQ,qFAAqF;AACtG,MAAI;AAAE,UAAM,GAAG,QAAQ,qDAAqD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC/F,QAAM,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAsBhB;AACD,QAAM,GAAG,QAAQ,0EAA0E;AAC3F,MAAI;AAAE,UAAM,GAAG,QAAQ,mDAAmD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC7F,MAAI;AAAE,UAAM,GAAG,QAAQ,uDAAuD;AAAA,EAAE,QAAQ;AAAA,EAAS;AACjG,MAAI;AAAE,UAAM,GAAG,QAAQ,kDAAkD;AAAA,EAAE,QAAQ;AAAA,EAAS;AAC5F,WAAS;AACX;AAgBA,eAAsB,oBAAoB,GAAuC;AAC/E,MAAI;AACF,UAAM,qBAAqB;AAC3B,UAAM,MAAM,mBAAmB;AAC/B,UAAM,aAAa,KAAK,IAAI,GAAG,EAAE,aAAa,EAAE,UAAU;AAC1D,UAAM,KAAK,MAAM;AACjB,UAAM,GAAG,QAAQ;AAAA,MACf,KAAK;AAAA;AAAA;AAAA,MAGL,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,EAAE,mBAAmB;AAAA,QACrB,KAAK,MAAM;AAAA,QACX,EAAE;AAAA,QACF,KAAK,cAAc;AAAA,QACnB,KAAK,UAAU;AAAA,QACf,UAAU,EAAE,OAAO;AAAA,QACnB,UAAU,EAAE,YAAY;AAAA,QACxB,UAAU,EAAE,SAAS;AAAA,QACrB,EAAE,eAAe;AAAA,QACjB,EAAE,aAAa;AAAA,QACf,EAAE,WAAW,IAAI;AAAA,QACjB,IAAI,KAAK,EAAE,UAAU,EAAE,YAAY;AAAA,QACnC,IAAI,KAAK,EAAE,UAAU,EAAE,YAAY;AAAA,QACnC;AAAA,QACA,cAAc,YAAY,KAAK;AAAA,QAC/B,cAAc,YAAY,IAAI;AAAA,QAC9B,EAAE,SAAS;AAAA,QACX,KAAK,SAAS;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,YAAQ,KAAK,gDAAgD,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,EAC/G;AACF;AAUA,eAAsB,kBAAkB,GAAqC;AAC3E,MAAI;AACF,UAAM,qBAAqB;AAC3B,UAAM,MAAM,mBAAmB;AAC/B,UAAM,KAAK,MAAM;AACjB,UAAM,GAAG,QAAQ;AAAA,MACf,KAAK;AAAA;AAAA;AAAA,MAGL,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,cAAc;AAAA,QACnB,KAAK,UAAU;AAAA,QACf,EAAE;AAAA,QACF,EAAE,SAAS;AAAA,QACX,EAAE;AAAA,QACF,EAAE;AAAA,QACF,cAAc,EAAE,QAAQ,EAAE,KAAK;AAAA,QAC/B,EAAE,SAAS;AAAA,QACX,KAAK,SAAS;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,YAAQ,KAAK,8CAA8C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,EAC7G;AACF;AA2CA,SAAS,UAAU,GAA8C;AAC/D,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,SAAO,IAAI,IAAI;AACjB;","names":[]}