mcp-scraper 0.89.8 → 0.89.9

package/dist/worker-577GJULQ.js

@@ -0,0 +1 @@
+import{a as O,b as y,c as D,d as g,f as A,g as _}from"./chunk-PHBU6X5B.js";import"./chunk-XG6GCEUE.js";import{u as k}from"./chunk-F4LIB772.js";import"./chunk-M5QHXNFZ.js";import{b as f,c as H}from"./chunk-7AC66V3T.js";import{a as m,b}from"./chunk-E4ME7J6G.js";import{a as v}from"./chunk-4FROKQJN.js";import{b as M}from"./chunk-DKQZNSYM.js";import"./chunk-2TZBO52D.js";import"./chunk-2PXP7TMY.js";import{Mb as l,Qb as p,Sb as d,Ua as u,gb as h,ib as c,na as o}from"./chunk-7BEWBRMT.js";var P=2,w=0;function C(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function R(e){return m.paa_base+Math.max(1,e)*m.paa}async function N(e){w++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},i=await M({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r},()=>k({...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:v(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:A(e.id,e.user_id)}));await h(e.id,i);let s=await o(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let n=t.serpOnly?b(r.value):R(C(i));await d(Number(e.user_id),t.billingDebitKey,n,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let n=R(C(i)),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"paa_refund","overestimate refund"):a<0&&await p(e.user_id,-a,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let n=b(r.value),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"serp_refund","headless-mode pricing settle"):a<0&&await p(e.user_id,-a,"serp",t.query??e.query)}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"done",result:H(i),attempts:f(s)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=O(t),i=typeof e.options=="string"?JSON.parse(e.options):e.options,s=typeof i.billingHoldMc=="number"&&i.billingHoldMc>0;await c(e.id,y(r),g(r,{chargeStatus:s?"refund_pending":"not_charged"}));let n=await o(e.id,e.user_id);try{s&&(i.billingDebitKey?await d(Number(e.user_id),i.billingDebitKey,0,i.serpOnly?"serp_refund":"paa_refund","failed call",i.serpOnly?"serp_search":"paa_harvest"):await l(e.user_id,i.billingHoldMc,"refund","failed call"),await c(e.id,y(r),g(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"failed",...D(r),attempts:f(n)})}finally{w--}}async function x(){let e=await u();if(!e)return{claimed:!1};let t=Date.now();return await N(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function W(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let i=await x();if(t.push(i),!i.claimed)break}return t}function B(){setInterval(async()=>{if(w>=P)return;let e=await u();e&&N(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${P} concurrent`)}export{W as drainQueue,B as startWorker,x as tickOnce};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.89.8",
+  "version": "0.89.9",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -23,110 +23,11 @@
     "mcp-scraper-install": "dist/bin/mcp-scraper-install.js",
     "mcp-scraper-cli": "dist/bin/mcp-scraper-cli.js"
   },
-  "files": [
-    "dist/*.js",
-    "dist/*.cjs",
-    "dist/*.d.ts",
-    "dist/*.d.cts",
-    "dist/bin/*.js",
-    "dist/editorial-reading-room/assets",
-    "public/agent-install.md",
-    "public/.well-known/xray-install.json",
-    "README.md",
-    "CHANGELOG.md"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/VilovietaSEO/mcp-scraper.git"
-  },
   "homepage": "https://mcpscraper.dev",
-  "bugs": {
-    "url": "https://github.com/VilovietaSEO/mcp-scraper/issues"
-  },
-  "scripts": {
-    "build": "tsup",
-    "build:md": "node scripts/build-md-mirror.mjs",
-    "build:mcpb": "npm run build && node scripts/build-mcpb.mjs",
-    "build:mcpb:candidate": "npm run build && node scripts/build-mcpb.mjs --candidate",
-    "build:rates": "node --import tsx scripts/gen-rates.ts",
-    "build:geotargets": "node --import tsx scripts/gen-geotargets.ts",
-    "build:blog": "node scripts/run-ts-sandbox-safe.mjs scripts/build-blog.ts",
-    "mcp:manifest": "node scripts/run-ts-sandbox-safe.mjs scripts/generate-mcp-tool-manifest.ts",
-    "contracts:memory": "node --import tsx scripts/generate-memory-tool-contract.ts",
-    "contracts:memory-governed": "node --import tsx scripts/generate-governed-memory-mcp-schemas.ts",
-    "verify:crm-authority": "node --import tsx scripts/verify-crm-authority-boundary.ts",
-    "contracts:openapi": "node --import tsx scripts/generate-personal-assistant-openapi.ts",
-    "contracts:assistant": "npm run mcp:manifest && npm run contracts:memory && npm run contracts:openapi",
-    "contracts:assistant:verify": "node --import tsx scripts/verify-personal-assistant-contract.ts",
-    "contracts:assistant:verify:release": "node --import tsx scripts/verify-personal-assistant-contract.ts --release",
-    "mcp:memory-image-schemas": "node scripts/generate-memory-image-mcp-schemas.mjs",
-    "mcp:measure-catalog": "node scripts/run-ts-sandbox-safe.mjs scripts/measure-mcp-catalog.ts",
-    "mcp:lint-tools": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts",
-    "mcp:lint-tools:write": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts --write",
-    "xray:install-contract": "node --import tsx scripts/generate-xray-install-contract.ts",
-    "xray:install-contract:check": "node --import tsx scripts/generate-xray-install-contract.ts --check",
-    "api": "node scripts/run-ts-sandbox-safe.mjs bin/api-server.ts",
-    "dev": "node scripts/run-ts-sandbox-safe.mjs src/cli.ts",
-    "test": "vitest run tests/unit tests/contract",
-    "test:live": "vitest run --config vitest.live.config.ts",
-    "test:site-similarity": "node --import tsx scripts/run-site-similarity-live.ts",
-    "reanalyze:site-similarity": "node --import tsx scripts/reanalyze-site-similarity-zip.ts",
-    "test:headless-harvest": "node scripts/run-ts-sandbox-safe.mjs tests/live/harness/headless-harvest-canary.ts",
-    "test:integration": "node scripts/run-integration-tests.mjs",
-    "test:smoke": "vitest run tests/smoke",
-    "test:stripe": "node scripts/stripe-setup-smoke.mjs",
-    "typecheck": "tsc --noEmit",
-    "test:kernel": "vitest run tests/integration/kernel.smoke.test.ts",
-    "sync:env": "bash scripts/push-env.sh",
-    "sync:env:apply": "bash scripts/push-env.sh --apply",
-    "locations:publish-geonames": "node --import tsx scripts/publish-geonames-location-data.ts",
-    "locations:sync": "node --import tsx scripts/sync-hosted-location-data.ts",
-    "integrations:migrate-scheduler": "node --import tsx scripts/migrate-scheduler-integrations.ts",
-    "test:webhook": "node scripts/test-webhook.mjs",
-    "test:e2e": "playwright test",
-    "email:preview": "node scripts/run-ts-sandbox-safe.mjs scripts/email/render-previews.ts",
-    "email:preflight": "node --import tsx scripts/email/preflight.ts",
-    "email:avatar": "node --import tsx scripts/email/generate-sender-avatar.ts",
-    "email:inbound:setup": "node --import tsx scripts/email/setup-resend-inbound.ts",
-    "email:audience": "node scripts/run-ts-sandbox-safe.mjs scripts/email/sync-resend-audience.ts",
-    "email:drafts": "node scripts/run-ts-sandbox-safe.mjs scripts/email/create-resend-drafts.ts",
-    "email:workshop-reminders": "node --import tsx scripts/email/refresh-workshop-reminders.ts",
-    "email:workshop-reminders:apply": "node --import tsx scripts/email/refresh-workshop-reminders.ts --apply",
-    "billing:preview-credit-lot-repair": "node --import tsx scripts/billing/preview-credit-lot-repair.ts",
-    "billing:apply-credit-lot-repair": "node --import tsx scripts/billing/apply-credit-lot-repair.ts",
-    "brightdata:usage": "node scripts/brightdata-usage-report.mjs",
-    "brightdata:test:bandwidth": "node --import tsx scripts/brightdata-paa-bandwidth-test.ts",
-    "db:migrate": "node --import tsx scripts/run-core-migration.ts",
-    "db:verify": "node --import tsx scripts/verify-core-schema.ts",
-    "release:server:predeploy": "npm run db:migrate && npm run db:verify && npm run version:check:core",
-    "oauth-store:migrate": "node --import tsx scripts/migrate-oauth-store.ts --apply-schema",
-    "oauth-store:verify": "node --import tsx scripts/migrate-oauth-store.ts --verify",
-    "oauth-store:test": "vitest run tests/unit/oauth-store.test.ts",
-    "oauth-store:test-remote": "OAUTH_STORE_REMOTE_REQUIRED=1 vitest run --config vitest.live.config.ts tests/live/oauth-store-remote.live.test.ts",
-    "oauth:prepare-production-canary": "node --import tsx scripts/prepare-mcp-oauth-production-canary.ts",
-    "oauth:verify-live": "node scripts/verify-mcp-oauth-live.mjs",
-    "analytics:migrate": "node --import tsx scripts/run-analytics-migration.ts",
-    "analytics:contract": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts",
-    "analytics:contract:check": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts --check",
-    "assistant:migrate": "node --import tsx scripts/run-personal-assistant-migration.ts",
-    "assistant:schema:verify": "vitest run tests/unit/personal-assistant-db.test.ts tests/unit/personal-assistant-production-stores.test.ts tests/integration/personal-assistant-db.test.ts",
-    "assistant:deployment:preflight": "node --import tsx scripts/verify-personal-assistant-deployment-config.ts",
-    "analytics:rollups": "node --import tsx scripts/run-analytics-rollups.ts",
-    "test:mcp-output-contract": "vitest run --config vitest.mcp-output-contract.config.ts tests/unit/golden-payloads.test.ts tests/unit/mcp-formatter-structured.test.ts tests/unit/mcp-2026-stdio.test.ts",
-    "version:check": "node scripts/verify-release-version-sync.mjs",
-    "version:check:core": "node scripts/verify-release-version-sync.mjs --core",
-    "prebuild": "npm run version:check:core",
-    "mcp:verify": "npm run xray:install-contract:check && npm run build && npm run mcp:manifest && npm run version:check && npm run build:rates && npm run mcp:measure-catalog && npm run mcp:lint-tools && npm run test:mcp-output-contract && npm run typecheck && npm test && npm run package:verify-entrypoints",
-    "prepack": "npm run mcp:verify",
-    "package:verify-entrypoints": "node scripts/verify-package-entrypoints.mjs",
-    "product:install": "node scripts/run-product-command.mjs install",
-    "product:build": "node scripts/run-product-command.mjs build",
-    "product:typecheck": "node scripts/run-product-command.mjs typecheck",
-    "product:test": "node scripts/run-product-command.mjs test",
-    "product:verify-layout": "node scripts/verify-product-repository-layout.mjs",
-    "package:public": "npm run mcp:verify && node scripts/build-public-package.mjs",
-    "prepublishOnly": "node -e \"throw new Error('Publish the verified archive produced by npm run package:public')\""
+  "engines": {
+    "node": ">=20.18.1"
   },
+  "license": "ISC",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
     "@fal-ai/client": "^1.10.1",
@@ -165,38 +66,17 @@
     "youtube-transcript": "^1.3.1",
     "zod": "4.4.3"
   },
-  "devDependencies": {
-    "@onkernel/cli": "^0.19.2",
-    "@playwright/test": "^1.60.0",
-    "@types/better-sqlite3": "^7.6.13",
-    "@types/jsdom": "^28.0.3",
-    "@types/node": "^20.0.0",
-    "@types/papaparse": "^5.3.0",
-    "@types/pg": "^8.23.1",
-    "@types/turndown": "^5.0.6",
-    "@types/yauzl": "^2.10.3",
-    "axe-core": "^4.13.0",
-    "better-sqlite3": "^12.9.0",
-    "bgutils-js": "^3.2.0",
-    "dotenv": "^17.4.2",
-    "esbuild": "^0.27.7",
-    "fflate": "^0.7.4",
-    "jsdom": "^29.1.1",
-    "tsup": "^8.0.0",
-    "tsx": "^4.0.0",
-    "typescript": "^5.4.0",
-    "vitest": "^1.6.0",
-    "youtubei.js": "^17.0.1",
-    "@onkernel/sdk": "^0.93.0"
-  },
-  "engines": {
-    "node": ">=20.18.1"
-  },
-  "directories": {
-    "doc": "docs",
-    "test": "tests"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC"
+  "files": [
+    "dist/*.js",
+    "dist/*.cjs",
+    "dist/*.d.ts",
+    "dist/*.d.cts",
+    "dist/bin/*.js",
+    "dist/editorial-reading-room/assets",
+    "public/agent-install.md",
+    "public/.well-known/xray-install.json",
+    "README.md",
+    "CHANGELOG.md",
+    "THIRD_PARTY_NOTICES.html"
+  ]
 }

package/dist/chunk-MM6QBDPC.js

@@ -1 +0,0 @@
-function M(E,t=!1){let _=t?3e4:4e5,e=Math.min(_+15e3,795e3);return{serverMs:_,clientMs:e}}export{M as a};

package/dist/chunk-Z3WXA4ZU.js

@@ -1 +0,0 @@
-var o="0.89.8";export{o as a};

package/dist/worker-GTIKQJIN.js

@@ -1 +0,0 @@
-import{a as O,b as y,c as D,d as g,f as A,g as _}from"./chunk-PHBU6X5B.js";import"./chunk-XG6GCEUE.js";import{u as k}from"./chunk-KX72IORC.js";import"./chunk-M5QHXNFZ.js";import{b as f,c as H}from"./chunk-7AC66V3T.js";import{a as m,b}from"./chunk-E4ME7J6G.js";import{a as v}from"./chunk-4FROKQJN.js";import{b as M}from"./chunk-DKQZNSYM.js";import"./chunk-2TZBO52D.js";import"./chunk-MM6QBDPC.js";import{Mb as l,Qb as p,Sb as d,Ua as u,gb as h,ib as c,na as o}from"./chunk-7BEWBRMT.js";var P=2,w=0;function C(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function R(e){return m.paa_base+Math.max(1,e)*m.paa}async function N(e){w++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},i=await M({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r},()=>k({...t,kernelApiKey:v(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:A(e.id,e.user_id)}));await h(e.id,i);let s=await o(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let n=t.serpOnly?b(r.value):R(C(i));await d(Number(e.user_id),t.billingDebitKey,n,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let n=R(C(i)),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"paa_refund","overestimate refund"):a<0&&await p(e.user_id,-a,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let n=b(r.value),a=t.billingHoldMc-n;a>0?await l(e.user_id,a,"serp_refund","headless-mode pricing settle"):a<0&&await p(e.user_id,-a,"serp",t.query??e.query)}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"done",result:H(i),attempts:f(s)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=O(t),i=typeof e.options=="string"?JSON.parse(e.options):e.options,s=typeof i.billingHoldMc=="number"&&i.billingHoldMc>0;await c(e.id,y(r),g(r,{chargeStatus:s?"refund_pending":"not_charged"}));let n=await o(e.id,e.user_id);try{s&&(i.billingDebitKey?await d(Number(e.user_id),i.billingDebitKey,0,i.serpOnly?"serp_refund":"paa_refund","failed call",i.serpOnly?"serp_search":"paa_harvest"):await l(e.user_id,i.billingHoldMc,"refund","failed call"),await c(e.id,y(r),g(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await _(e.callback_url,{job_id:e.id,status:"failed",...D(r),attempts:f(n)})}finally{w--}}async function x(){let e=await u();if(!e)return{claimed:!1};let t=Date.now();return await N(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function W(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let i=await x();if(t.push(i),!i.claimed)break}return t}function B(){setInterval(async()=>{if(w>=P)return;let e=await u();e&&N(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${P} concurrent`)}export{W as drainQueue,B as startWorker,x as tickOnce};