mcp-scraper 0.88.2 → 0.89.0

package/dist/site-extract-repository-SE776XDC.js

@@ -0,0 +1 @@
+import{A as s,B as t,C as u,D as v,E as w,F as x,G as y,H as z,I as A,J as B,K as C,L as D,M as E,i as a,j as b,k as c,l as d,m as e,n as f,o as g,p as h,q as i,r as j,s as k,t as l,u as m,v as n,w as o,x as p,y as q,z as r}from"./chunk-CFI6CXIV.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-GMWKPIYX.js";export{a as XRAY_ENTITLED_EXTRACT_BILLING_CLASS,q as abandonExtractSettlement,p as claimFailedExtractJobForRefinalize,z as completeExtractJob,w as countSuccessfulPages,d as createExtractJob,f as createOrGetExtractJob,b as extractJobLimitInfo,A as failExtractJob,o as failStaleRunningExtractJob,B as failUnfundedExtractJob,E as finalizeXRayEntitledExtractJobCharge,y as finishExtractJob,g as getExtractJob,e as getExtractJobByIdempotencyKey,v as getExtractedImageLinks,u as getExtractedPages,x as getExtractedUrls,h as listExtractJobs,t as listExtractPages,j as listFundedPendingExtractJobs,n as listStaleRunningExtractJobs,i as listUnsettledExtractJobs,k as markExtractJobDispatchAttempt,l as recordExtractJobDispatchFailure,m as recordExtractSettlementFailure,s as saveExtractPages,C as setExtractJobPublicError,r as setExtractJobTotal,D as settleExtractJob,c as terminalExtractJobStatus};

package/dist/worker-XUDSM3AL.js

@@ -0,0 +1 @@
+import{a as h,b as f,c as O,d as y,f as A,g as b}from"./chunk-E2WRWV3A.js";import"./chunk-YQZGZBB4.js";import{u as k}from"./chunk-7YGVI5J4.js";import"./chunk-QPWPR5XG.js";import{b as p,c as M}from"./chunk-X54CQLK2.js";import{a as m,b as H}from"./chunk-MZN4U5BL.js";import{a as w}from"./chunk-4FROKQJN.js";import{b as _}from"./chunk-MASR22K4.js";import"./chunk-3GP5CYZX.js";import{$a as c,Cb as o,Gb as d,La as u,Za as v,ha as l}from"./chunk-GMWKPIYX.js";var C=2,g=0;function R(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function x(e){return m.paa_base+Math.max(1,e)*m.paa}async function P(e){g++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},a=await _({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r},()=>k({...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:w(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:A(e.id,e.user_id)}));await v(e.id,a);let s=await l(e.id,e.user_id);if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let n=x(R(a)),i=t.billingHoldMc-n;i>0?await o(e.user_id,i,"paa_refund","overestimate refund"):i<0&&await d(e.user_id,-i,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let n=H(r.value),i=t.billingHoldMc-n;i>0?await o(e.user_id,i,"serp_refund","headless-mode pricing settle"):i<0&&await d(e.user_id,-i,"serp",t.query??e.query)}e.callback_url&&await b(e.callback_url,{job_id:e.id,status:"done",result:M(a),attempts:p(s)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=h(t),a=typeof e.options=="string"?JSON.parse(e.options):e.options,s=typeof a.billingHoldMc=="number"&&a.billingHoldMc>0;await c(e.id,f(r),y(r,{chargeStatus:s?"refund_pending":"not_charged"}));let n=await l(e.id,e.user_id);try{s&&(await o(e.user_id,a.billingHoldMc,"refund","failed call"),await c(e.id,f(r),y(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await b(e.callback_url,{job_id:e.id,status:"failed",...O(r),attempts:p(n)})}finally{g--}}async function J(){let e=await u();if(!e)return{claimed:!1};let t=Date.now();return await P(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function W(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let a=await J();if(t.push(a),!a.claimed)break}return t}function B(){setInterval(async()=>{if(g>=C)return;let e=await u();e&&P(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${C} concurrent`)}export{W as drainQueue,B as startWorker,J as tickOnce};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.88.2",
+  "version": "0.89.0",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -23,97 +23,11 @@
     "mcp-scraper-install": "dist/bin/mcp-scraper-install.js",
     "mcp-scraper-cli": "dist/bin/mcp-scraper-cli.js"
   },
-  "files": [
-    "dist/*.js",
-    "dist/*.cjs",
-    "dist/*.d.ts",
-    "dist/*.d.cts",
-    "dist/bin/*.js",
-    "dist/editorial-reading-room/assets",
-    "public/agent-install.md",
-    "public/.well-known/xray-install.json",
-    "README.md",
-    "CHANGELOG.md"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/VilovietaSEO/mcp-scraper.git"
-  },
   "homepage": "https://mcpscraper.dev",
-  "bugs": {
-    "url": "https://github.com/VilovietaSEO/mcp-scraper/issues"
-  },
-  "scripts": {
-    "build": "tsup",
-    "build:md": "node scripts/build-md-mirror.mjs",
-    "build:mcpb": "npm run build && node scripts/build-mcpb.mjs",
-    "build:mcpb:candidate": "npm run build && node scripts/build-mcpb.mjs --candidate",
-    "build:rates": "node --import tsx scripts/gen-rates.ts",
-    "build:geotargets": "node --import tsx scripts/gen-geotargets.ts",
-    "build:blog": "node scripts/run-ts-sandbox-safe.mjs scripts/build-blog.ts",
-    "mcp:manifest": "node scripts/run-ts-sandbox-safe.mjs scripts/generate-mcp-tool-manifest.ts",
-    "contracts:memory": "node --import tsx scripts/generate-memory-tool-contract.ts",
-    "contracts:memory-governed": "node --import tsx scripts/generate-governed-memory-mcp-schemas.ts",
-    "verify:crm-authority": "node --import tsx scripts/verify-crm-authority-boundary.ts",
-    "contracts:openapi": "node --import tsx scripts/generate-personal-assistant-openapi.ts",
-    "contracts:assistant": "npm run mcp:manifest && npm run contracts:memory && npm run contracts:openapi",
-    "contracts:assistant:verify": "node --import tsx scripts/verify-personal-assistant-contract.ts",
-    "contracts:assistant:verify:release": "node --import tsx scripts/verify-personal-assistant-contract.ts --release",
-    "mcp:memory-image-schemas": "node scripts/generate-memory-image-mcp-schemas.mjs",
-    "mcp:measure-catalog": "node scripts/run-ts-sandbox-safe.mjs scripts/measure-mcp-catalog.ts",
-    "mcp:lint-tools": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts",
-    "mcp:lint-tools:write": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts --write",
-    "xray:install-contract": "node --import tsx scripts/generate-xray-install-contract.ts",
-    "xray:install-contract:check": "node --import tsx scripts/generate-xray-install-contract.ts --check",
-    "api": "node scripts/run-ts-sandbox-safe.mjs bin/api-server.ts",
-    "dev": "node scripts/run-ts-sandbox-safe.mjs src/cli.ts",
-    "test": "vitest run tests/unit tests/contract",
-    "test:live": "vitest run --config vitest.live.config.ts",
-    "test:site-similarity": "node --import tsx scripts/run-site-similarity-live.ts",
-    "reanalyze:site-similarity": "node --import tsx scripts/reanalyze-site-similarity-zip.ts",
-    "test:headless-harvest": "node scripts/run-ts-sandbox-safe.mjs tests/live/harness/headless-harvest-canary.ts",
-    "test:integration": "node scripts/run-integration-tests.mjs",
-    "test:smoke": "vitest run tests/smoke",
-    "test:stripe": "node scripts/stripe-setup-smoke.mjs",
-    "typecheck": "tsc --noEmit",
-    "test:kernel": "vitest run tests/integration/kernel.smoke.test.ts",
-    "sync:env": "bash scripts/push-env.sh",
-    "sync:env:apply": "bash scripts/push-env.sh --apply",
-    "locations:publish-geonames": "node --import tsx scripts/publish-geonames-location-data.ts",
-    "locations:sync": "node --import tsx scripts/sync-hosted-location-data.ts",
-    "integrations:migrate-scheduler": "node --import tsx scripts/migrate-scheduler-integrations.ts",
-    "test:webhook": "node scripts/test-webhook.mjs",
-    "test:e2e": "playwright test",
-    "email:preview": "node scripts/run-ts-sandbox-safe.mjs scripts/email/render-previews.ts",
-    "email:preflight": "node --import tsx scripts/email/preflight.ts",
-    "email:avatar": "node --import tsx scripts/email/generate-sender-avatar.ts",
-    "email:inbound:setup": "node --import tsx scripts/email/setup-resend-inbound.ts",
-    "email:audience": "node scripts/run-ts-sandbox-safe.mjs scripts/email/sync-resend-audience.ts",
-    "email:drafts": "node scripts/run-ts-sandbox-safe.mjs scripts/email/create-resend-drafts.ts",
-    "email:workshop-reminders": "node --import tsx scripts/email/refresh-workshop-reminders.ts",
-    "email:workshop-reminders:apply": "node --import tsx scripts/email/refresh-workshop-reminders.ts --apply",
-    "billing:preview-credit-lot-repair": "node --import tsx scripts/billing/preview-credit-lot-repair.ts",
-    "billing:apply-credit-lot-repair": "node --import tsx scripts/billing/apply-credit-lot-repair.ts",
-    "brightdata:usage": "node scripts/brightdata-usage-report.mjs",
-    "brightdata:test:bandwidth": "node --import tsx scripts/brightdata-paa-bandwidth-test.ts",
-    "db:migrate": "node --import tsx scripts/run-core-migration.ts",
-    "analytics:migrate": "node --import tsx scripts/run-analytics-migration.ts",
-    "analytics:contract": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts",
-    "analytics:contract:check": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts --check",
-    "assistant:migrate": "node --import tsx scripts/run-personal-assistant-migration.ts",
-    "assistant:schema:verify": "vitest run tests/unit/personal-assistant-db.test.ts tests/unit/personal-assistant-production-stores.test.ts tests/integration/personal-assistant-db.test.ts",
-    "assistant:deployment:preflight": "node --import tsx scripts/verify-personal-assistant-deployment-config.ts",
-    "analytics:rollups": "node --import tsx scripts/run-analytics-rollups.ts",
-    "test:mcp-output-contract": "vitest run --config vitest.mcp-output-contract.config.ts tests/unit/golden-payloads.test.ts tests/unit/mcp-formatter-structured.test.ts tests/unit/mcp-2026-stdio.test.ts",
-    "mcp:verify": "npm run xray:install-contract:check && npm run build && npm run mcp:manifest && npm run build:rates && npm run mcp:measure-catalog && npm run mcp:lint-tools && npm run test:mcp-output-contract && npm run typecheck && npm test && npm run package:verify-entrypoints",
-    "prepack": "npm run mcp:verify",
-    "package:verify-entrypoints": "node scripts/verify-package-entrypoints.mjs",
-    "product:install": "node scripts/run-product-command.mjs install",
-    "product:build": "node scripts/run-product-command.mjs build",
-    "product:typecheck": "node scripts/run-product-command.mjs typecheck",
-    "product:test": "node scripts/run-product-command.mjs test",
-    "product:verify-layout": "node scripts/verify-product-repository-layout.mjs"
+  "engines": {
+    "node": ">=20.18.1"
   },
+  "license": "ISC",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
     "@fal-ai/client": "^1.10.1",
@@ -123,7 +37,6 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@neondatabase/serverless": "^1.1.0",
-    "@onkernel/sdk": "^0.93.0",
     "@resvg/resvg-js": "^2.6.2",
     "@vercel/blob": "^2.4.1",
     "@vercel/functions": "^3.9.5",
@@ -153,37 +66,17 @@
     "youtube-transcript": "^1.3.1",
     "zod": "4.4.3"
   },
-  "devDependencies": {
-    "@onkernel/cli": "^0.19.2",
-    "@playwright/test": "^1.60.0",
-    "@types/better-sqlite3": "^7.6.13",
-    "@types/jsdom": "^28.0.3",
-    "@types/node": "^20.0.0",
-    "@types/papaparse": "^5.3.0",
-    "@types/pg": "^8.23.1",
-    "@types/turndown": "^5.0.6",
-    "@types/yauzl": "^2.10.3",
-    "axe-core": "^4.13.0",
-    "better-sqlite3": "^12.9.0",
-    "bgutils-js": "^3.2.0",
-    "dotenv": "^17.4.2",
-    "esbuild": "^0.27.7",
-    "fflate": "^0.7.4",
-    "jsdom": "^29.1.1",
-    "tsup": "^8.0.0",
-    "tsx": "^4.0.0",
-    "typescript": "^5.4.0",
-    "vitest": "^1.6.0",
-    "youtubei.js": "^17.0.1"
-  },
-  "engines": {
-    "node": ">=20.18.1"
-  },
-  "directories": {
-    "doc": "docs",
-    "test": "tests"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC"
+  "files": [
+    "dist/*.js",
+    "dist/*.cjs",
+    "dist/*.d.ts",
+    "dist/*.d.cts",
+    "dist/bin/*.js",
+    "dist/editorial-reading-room/assets",
+    "public/agent-install.md",
+    "public/.well-known/xray-install.json",
+    "README.md",
+    "CHANGELOG.md",
+    "THIRD_PARTY_NOTICES.html"
+  ]
 }

package/dist/analytics-repository-GGJJCVVP.js

@@ -1,194 +0,0 @@
-import {
-  ANALYTICS_CONTENT_SORTS,
-  AnalyticsRepositoryError,
-  ENGAGED_SESSION_MS,
-  MAX_ENGAGED_MS,
-  analyticsAcquisition,
-  analyticsBusinessMetrics,
-  analyticsChannelBreakdown,
-  analyticsContent,
-  analyticsConversions,
-  analyticsCsvCell,
-  analyticsDimensions,
-  analyticsEventCounts,
-  analyticsForecast,
-  analyticsHealth,
-  analyticsIdentityPromotionAllowed,
-  analyticsIdentityResolutionAllowed,
-  analyticsOverview,
-  analyticsPaths,
-  analyticsTimeseries,
-  appendAnalyticsAuthoritativeOutcomeVersion,
-  archiveAnalyticsActivationDestination,
-  archiveAnalyticsCampaignLink,
-  assignAnalyticsIdentityNode,
-  backfillAnalyticsConfirmedHistory,
-  claimAnalyticsCrmImportRows,
-  claimAnalyticsFormDeliveryJobs,
-  closeAnalyticsPool,
-  completeAnalyticsCrmImportRow,
-  completeAnalyticsFormBridgeDelivery,
-  completeAnalyticsFormDelivery,
-  consumeAnalyticsSurveyInvite,
-  createAnalyticsActivationDestination,
-  createAnalyticsCampaignLink,
-  createAnalyticsConversion,
-  createAnalyticsCrmImport,
-  createAnalyticsExport,
-  createAnalyticsForm,
-  createAnalyticsPixel,
-  createAnalyticsSite,
-  deferAnalyticsCrmImportRow,
-  deferAnalyticsFormDelivery,
-  deleteAnalyticsSite,
-  deterministicAnalyticsCrmEntityId,
-  enrichAnalyticsExistingCrmIdentity,
-  getAnalyticsActivationDestinationConnectionRef,
-  getAnalyticsPersonJourney,
-  getAnalyticsPool,
-  getPublicAnalyticsForm,
-  identityHmac,
-  ingestAnalyticsEvents,
-  insertAnalyticsRevenueSetupRevision,
-  isAnalyticsFormPlacementApproved,
-  linkAnalyticsFormIdentity,
-  linkAnalyticsIdentityInTransaction,
-  listAnalyticsActivationDestinations,
-  listAnalyticsActivationReceipts,
-  listAnalyticsCampaignLinks,
-  listAnalyticsCrmImports,
-  listAnalyticsForms,
-  listAnalyticsHostGroups,
-  listAnalyticsPeople,
-  listAnalyticsPixels,
-  listAnalyticsSites,
-  migrateAnalytics,
-  normalizeAnalyticsPath,
-  normalizeAnalyticsUrl,
-  normalizeContentOptions,
-  normalizeObservedHostname,
-  pollAnalyticsActivationDiagnostics,
-  prepareAnalyticsLinkerIssue,
-  projectAnalyticsAuthoritativeConversion,
-  projectAnalyticsPixelEventConsent,
-  queueAnalyticsActivation,
-  queueAnalyticsFormBridgeTransaction,
-  queueAnalyticsFormDelivery,
-  recordAnalyticsFormSubmission,
-  redeemAnalyticsLinkerRecord,
-  refreshAnalyticsDailyRollups,
-  refreshAnalyticsDailyRollupsIfDue,
-  requireAnalyticsAccess,
-  requireAnalyticsEditor,
-  resolveAnalyticsCampaignLink,
-  resolveAnalyticsConfirmedActivationIdentity,
-  retryAnalyticsActivationJob,
-  sanitizeAnalyticsProperties,
-  sanitizeClickIds,
-  setAnalyticsActivationReadiness,
-  setAnalyticsPixelDomainState,
-  sweepAnalyticsRestrictedRetention,
-  testAnalyticsActivationDestination,
-  updateAnalyticsBusinessModel,
-  updateAnalyticsPixel,
-  upsertAnalyticsAdSpend,
-  upsertAnalyticsHostGroup
-} from "./chunk-X623GTBV.js";
-import "./chunk-C5Z4OFKW.js";
-import "./chunk-YXNDOQXN.js";
-export {
-  ANALYTICS_CONTENT_SORTS,
-  AnalyticsRepositoryError,
-  ENGAGED_SESSION_MS,
-  MAX_ENGAGED_MS,
-  analyticsAcquisition,
-  analyticsBusinessMetrics,
-  analyticsChannelBreakdown,
-  analyticsContent,
-  analyticsConversions,
-  analyticsCsvCell,
-  analyticsDimensions,
-  analyticsEventCounts,
-  analyticsForecast,
-  analyticsHealth,
-  analyticsIdentityPromotionAllowed,
-  analyticsIdentityResolutionAllowed,
-  analyticsOverview,
-  analyticsPaths,
-  analyticsTimeseries,
-  appendAnalyticsAuthoritativeOutcomeVersion,
-  archiveAnalyticsActivationDestination,
-  archiveAnalyticsCampaignLink,
-  assignAnalyticsIdentityNode,
-  backfillAnalyticsConfirmedHistory,
-  claimAnalyticsCrmImportRows,
-  claimAnalyticsFormDeliveryJobs,
-  closeAnalyticsPool,
-  completeAnalyticsCrmImportRow,
-  completeAnalyticsFormBridgeDelivery,
-  completeAnalyticsFormDelivery,
-  consumeAnalyticsSurveyInvite,
-  createAnalyticsActivationDestination,
-  createAnalyticsCampaignLink,
-  createAnalyticsConversion,
-  createAnalyticsCrmImport,
-  createAnalyticsExport,
-  createAnalyticsForm,
-  createAnalyticsPixel,
-  createAnalyticsSite,
-  deferAnalyticsCrmImportRow,
-  deferAnalyticsFormDelivery,
-  deleteAnalyticsSite,
-  deterministicAnalyticsCrmEntityId,
-  enrichAnalyticsExistingCrmIdentity,
-  getAnalyticsActivationDestinationConnectionRef,
-  getAnalyticsPersonJourney,
-  getAnalyticsPool,
-  getPublicAnalyticsForm,
-  identityHmac,
-  ingestAnalyticsEvents,
-  insertAnalyticsRevenueSetupRevision,
-  isAnalyticsFormPlacementApproved,
-  linkAnalyticsFormIdentity,
-  linkAnalyticsIdentityInTransaction,
-  listAnalyticsActivationDestinations,
-  listAnalyticsActivationReceipts,
-  listAnalyticsCampaignLinks,
-  listAnalyticsCrmImports,
-  listAnalyticsForms,
-  listAnalyticsHostGroups,
-  listAnalyticsPeople,
-  listAnalyticsPixels,
-  listAnalyticsSites,
-  migrateAnalytics,
-  normalizeAnalyticsPath,
-  normalizeAnalyticsUrl,
-  normalizeContentOptions,
-  normalizeObservedHostname,
-  pollAnalyticsActivationDiagnostics,
-  prepareAnalyticsLinkerIssue,
-  projectAnalyticsAuthoritativeConversion,
-  projectAnalyticsPixelEventConsent,
-  queueAnalyticsActivation,
-  queueAnalyticsFormBridgeTransaction,
-  queueAnalyticsFormDelivery,
-  recordAnalyticsFormSubmission,
-  redeemAnalyticsLinkerRecord,
-  refreshAnalyticsDailyRollups,
-  refreshAnalyticsDailyRollupsIfDue,
-  requireAnalyticsAccess,
-  requireAnalyticsEditor,
-  resolveAnalyticsCampaignLink,
-  resolveAnalyticsConfirmedActivationIdentity,
-  retryAnalyticsActivationJob,
-  sanitizeAnalyticsProperties,
-  sanitizeClickIds,
-  setAnalyticsActivationReadiness,
-  setAnalyticsPixelDomainState,
-  sweepAnalyticsRestrictedRetention,
-  testAnalyticsActivationDestination,
-  updateAnalyticsBusinessModel,
-  updateAnalyticsPixel,
-  upsertAnalyticsAdSpend,
-  upsertAnalyticsHostGroup
-};