mcp-scraper 0.88.2 → 0.89.1

package/dist/site-extract-repository-7W5ZUNPR.js

@@ -0,0 +1 @@
+import{A as s,B as t,C as u,D as v,E as w,F as x,G as y,H as z,I as A,J as B,K as C,L as D,M as E,i as a,j as b,k as c,l as d,m as e,n as f,o as g,p as h,q as i,r as j,s as k,t as l,u as m,v as n,w as o,x as p,y as q,z as r}from"./chunk-DGBLGFNJ.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-63T4W2LX.js";export{a as XRAY_ENTITLED_EXTRACT_BILLING_CLASS,q as abandonExtractSettlement,p as claimFailedExtractJobForRefinalize,z as completeExtractJob,w as countSuccessfulPages,d as createExtractJob,f as createOrGetExtractJob,b as extractJobLimitInfo,A as failExtractJob,o as failStaleRunningExtractJob,B as failUnfundedExtractJob,E as finalizeXRayEntitledExtractJobCharge,y as finishExtractJob,g as getExtractJob,e as getExtractJobByIdempotencyKey,v as getExtractedImageLinks,u as getExtractedPages,x as getExtractedUrls,h as listExtractJobs,t as listExtractPages,j as listFundedPendingExtractJobs,n as listStaleRunningExtractJobs,i as listUnsettledExtractJobs,k as markExtractJobDispatchAttempt,l as recordExtractJobDispatchFailure,m as recordExtractSettlementFailure,s as saveExtractPages,C as setExtractJobPublicError,r as setExtractJobTotal,D as settleExtractJob,c as terminalExtractJobStatus};

package/dist/worker-ZOTOKG56.js

@@ -0,0 +1 @@
+import{a as h,b as f,c as O,d as y,f as A,g as b}from"./chunk-PRWZNHQB.js";import"./chunk-YQZGZBB4.js";import{u as k}from"./chunk-6QO4F2KC.js";import"./chunk-QPWPR5XG.js";import{b as p,c as M}from"./chunk-FXEWHK54.js";import{a as m,b as H}from"./chunk-MZN4U5BL.js";import{a as w}from"./chunk-4FROKQJN.js";import{b as _}from"./chunk-RYNHNEF3.js";import"./chunk-3GP5CYZX.js";import{$a as v,Eb as o,Ib as d,Na as u,bb as c,ja as l}from"./chunk-63T4W2LX.js";var C=2,g=0;function R(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function x(e){return m.paa_base+Math.max(1,e)*m.paa}async function P(e){g++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,r={value:null},a=await _({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:r},()=>k({...t,kernelApiKey:w(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:A(e.id,e.user_id)}));await v(e.id,a);let s=await l(e.id,e.user_id);if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let n=x(R(a)),i=t.billingHoldMc-n;i>0?await o(e.user_id,i,"paa_refund","overestimate refund"):i<0&&await d(e.user_id,-i,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let n=H(r.value),i=t.billingHoldMc-n;i>0?await o(e.user_id,i,"serp_refund","headless-mode pricing settle"):i<0&&await d(e.user_id,-i,"serp",t.query??e.query)}e.callback_url&&await b(e.callback_url,{job_id:e.id,status:"done",result:M(a),attempts:p(s)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let r=h(t),a=typeof e.options=="string"?JSON.parse(e.options):e.options,s=typeof a.billingHoldMc=="number"&&a.billingHoldMc>0;await c(e.id,f(r),y(r,{chargeStatus:s?"refund_pending":"not_charged"}));let n=await l(e.id,e.user_id);try{s&&(await o(e.user_id,a.billingHoldMc,"refund","failed call"),await c(e.id,f(r),y(r,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await b(e.callback_url,{job_id:e.id,status:"failed",...O(r),attempts:p(n)})}finally{g--}}async function J(){let e=await u();if(!e)return{claimed:!1};let t=Date.now();return await P(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function W(e){let t=[];for(let r=0;r<e.maxJobs&&!(Date.now()>=e.deadlineMs);r++){let a=await J();if(t.push(a),!a.claimed)break}return t}function B(){setInterval(async()=>{if(g>=C)return;let e=await u();e&&P(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${C} concurrent`)}export{W as drainQueue,B as startWorker,J as tickOnce};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.88.2",
+  "version": "0.89.1",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -97,6 +97,11 @@
     "brightdata:usage": "node scripts/brightdata-usage-report.mjs",
     "brightdata:test:bandwidth": "node --import tsx scripts/brightdata-paa-bandwidth-test.ts",
     "db:migrate": "node --import tsx scripts/run-core-migration.ts",
+    "oauth-store:migrate": "node --import tsx scripts/migrate-oauth-store.ts --apply-schema",
+    "oauth-store:verify": "node --import tsx scripts/migrate-oauth-store.ts --verify",
+    "oauth-store:test": "vitest run tests/unit/oauth-store.test.ts",
+    "oauth-store:test-remote": "OAUTH_STORE_REMOTE_REQUIRED=1 vitest run --config vitest.live.config.ts tests/live/oauth-store-remote.live.test.ts",
+    "oauth:verify-live": "node scripts/verify-mcp-oauth-live.mjs",
     "analytics:migrate": "node --import tsx scripts/run-analytics-migration.ts",
     "analytics:contract": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts",
     "analytics:contract:check": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts --check",
@@ -112,7 +117,9 @@
     "product:build": "node scripts/run-product-command.mjs build",
     "product:typecheck": "node scripts/run-product-command.mjs typecheck",
     "product:test": "node scripts/run-product-command.mjs test",
-    "product:verify-layout": "node scripts/verify-product-repository-layout.mjs"
+    "product:verify-layout": "node scripts/verify-product-repository-layout.mjs",
+    "package:public": "npm run mcp:verify && node scripts/build-public-package.mjs",
+    "prepublishOnly": "node -e \"throw new Error('Publish the verified archive produced by npm run package:public')\""
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
@@ -123,7 +130,6 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/server": "^2.0.0",
     "@neondatabase/serverless": "^1.1.0",
-    "@onkernel/sdk": "^0.93.0",
     "@resvg/resvg-js": "^2.6.2",
     "@vercel/blob": "^2.4.1",
     "@vercel/functions": "^3.9.5",
@@ -174,7 +180,8 @@
     "tsx": "^4.0.0",
     "typescript": "^5.4.0",
     "vitest": "^1.6.0",
-    "youtubei.js": "^17.0.1"
+    "youtubei.js": "^17.0.1",
+    "@onkernel/sdk": "^0.93.0"
   },
   "engines": {
     "node": ">=20.18.1"

package/dist/analytics-repository-GGJJCVVP.js

@@ -1,194 +0,0 @@
-import {
-  ANALYTICS_CONTENT_SORTS,
-  AnalyticsRepositoryError,
-  ENGAGED_SESSION_MS,
-  MAX_ENGAGED_MS,
-  analyticsAcquisition,
-  analyticsBusinessMetrics,
-  analyticsChannelBreakdown,
-  analyticsContent,
-  analyticsConversions,
-  analyticsCsvCell,
-  analyticsDimensions,
-  analyticsEventCounts,
-  analyticsForecast,
-  analyticsHealth,
-  analyticsIdentityPromotionAllowed,
-  analyticsIdentityResolutionAllowed,
-  analyticsOverview,
-  analyticsPaths,
-  analyticsTimeseries,
-  appendAnalyticsAuthoritativeOutcomeVersion,
-  archiveAnalyticsActivationDestination,
-  archiveAnalyticsCampaignLink,
-  assignAnalyticsIdentityNode,
-  backfillAnalyticsConfirmedHistory,
-  claimAnalyticsCrmImportRows,
-  claimAnalyticsFormDeliveryJobs,
-  closeAnalyticsPool,
-  completeAnalyticsCrmImportRow,
-  completeAnalyticsFormBridgeDelivery,
-  completeAnalyticsFormDelivery,
-  consumeAnalyticsSurveyInvite,
-  createAnalyticsActivationDestination,
-  createAnalyticsCampaignLink,
-  createAnalyticsConversion,
-  createAnalyticsCrmImport,
-  createAnalyticsExport,
-  createAnalyticsForm,
-  createAnalyticsPixel,
-  createAnalyticsSite,
-  deferAnalyticsCrmImportRow,
-  deferAnalyticsFormDelivery,
-  deleteAnalyticsSite,
-  deterministicAnalyticsCrmEntityId,
-  enrichAnalyticsExistingCrmIdentity,
-  getAnalyticsActivationDestinationConnectionRef,
-  getAnalyticsPersonJourney,
-  getAnalyticsPool,
-  getPublicAnalyticsForm,
-  identityHmac,
-  ingestAnalyticsEvents,
-  insertAnalyticsRevenueSetupRevision,
-  isAnalyticsFormPlacementApproved,
-  linkAnalyticsFormIdentity,
-  linkAnalyticsIdentityInTransaction,
-  listAnalyticsActivationDestinations,
-  listAnalyticsActivationReceipts,
-  listAnalyticsCampaignLinks,
-  listAnalyticsCrmImports,
-  listAnalyticsForms,
-  listAnalyticsHostGroups,
-  listAnalyticsPeople,
-  listAnalyticsPixels,
-  listAnalyticsSites,
-  migrateAnalytics,
-  normalizeAnalyticsPath,
-  normalizeAnalyticsUrl,
-  normalizeContentOptions,
-  normalizeObservedHostname,
-  pollAnalyticsActivationDiagnostics,
-  prepareAnalyticsLinkerIssue,
-  projectAnalyticsAuthoritativeConversion,
-  projectAnalyticsPixelEventConsent,
-  queueAnalyticsActivation,
-  queueAnalyticsFormBridgeTransaction,
-  queueAnalyticsFormDelivery,
-  recordAnalyticsFormSubmission,
-  redeemAnalyticsLinkerRecord,
-  refreshAnalyticsDailyRollups,
-  refreshAnalyticsDailyRollupsIfDue,
-  requireAnalyticsAccess,
-  requireAnalyticsEditor,
-  resolveAnalyticsCampaignLink,
-  resolveAnalyticsConfirmedActivationIdentity,
-  retryAnalyticsActivationJob,
-  sanitizeAnalyticsProperties,
-  sanitizeClickIds,
-  setAnalyticsActivationReadiness,
-  setAnalyticsPixelDomainState,
-  sweepAnalyticsRestrictedRetention,
-  testAnalyticsActivationDestination,
-  updateAnalyticsBusinessModel,
-  updateAnalyticsPixel,
-  upsertAnalyticsAdSpend,
-  upsertAnalyticsHostGroup
-} from "./chunk-X623GTBV.js";
-import "./chunk-C5Z4OFKW.js";
-import "./chunk-YXNDOQXN.js";
-export {
-  ANALYTICS_CONTENT_SORTS,
-  AnalyticsRepositoryError,
-  ENGAGED_SESSION_MS,
-  MAX_ENGAGED_MS,
-  analyticsAcquisition,
-  analyticsBusinessMetrics,
-  analyticsChannelBreakdown,
-  analyticsContent,
-  analyticsConversions,
-  analyticsCsvCell,
-  analyticsDimensions,
-  analyticsEventCounts,
-  analyticsForecast,
-  analyticsHealth,
-  analyticsIdentityPromotionAllowed,
-  analyticsIdentityResolutionAllowed,
-  analyticsOverview,
-  analyticsPaths,
-  analyticsTimeseries,
-  appendAnalyticsAuthoritativeOutcomeVersion,
-  archiveAnalyticsActivationDestination,
-  archiveAnalyticsCampaignLink,
-  assignAnalyticsIdentityNode,
-  backfillAnalyticsConfirmedHistory,
-  claimAnalyticsCrmImportRows,
-  claimAnalyticsFormDeliveryJobs,
-  closeAnalyticsPool,
-  completeAnalyticsCrmImportRow,
-  completeAnalyticsFormBridgeDelivery,
-  completeAnalyticsFormDelivery,
-  consumeAnalyticsSurveyInvite,
-  createAnalyticsActivationDestination,
-  createAnalyticsCampaignLink,
-  createAnalyticsConversion,
-  createAnalyticsCrmImport,
-  createAnalyticsExport,
-  createAnalyticsForm,
-  createAnalyticsPixel,
-  createAnalyticsSite,
-  deferAnalyticsCrmImportRow,
-  deferAnalyticsFormDelivery,
-  deleteAnalyticsSite,
-  deterministicAnalyticsCrmEntityId,
-  enrichAnalyticsExistingCrmIdentity,
-  getAnalyticsActivationDestinationConnectionRef,
-  getAnalyticsPersonJourney,
-  getAnalyticsPool,
-  getPublicAnalyticsForm,
-  identityHmac,
-  ingestAnalyticsEvents,
-  insertAnalyticsRevenueSetupRevision,
-  isAnalyticsFormPlacementApproved,
-  linkAnalyticsFormIdentity,
-  linkAnalyticsIdentityInTransaction,
-  listAnalyticsActivationDestinations,
-  listAnalyticsActivationReceipts,
-  listAnalyticsCampaignLinks,
-  listAnalyticsCrmImports,
-  listAnalyticsForms,
-  listAnalyticsHostGroups,
-  listAnalyticsPeople,
-  listAnalyticsPixels,
-  listAnalyticsSites,
-  migrateAnalytics,
-  normalizeAnalyticsPath,
-  normalizeAnalyticsUrl,
-  normalizeContentOptions,
-  normalizeObservedHostname,
-  pollAnalyticsActivationDiagnostics,
-  prepareAnalyticsLinkerIssue,
-  projectAnalyticsAuthoritativeConversion,
-  projectAnalyticsPixelEventConsent,
-  queueAnalyticsActivation,
-  queueAnalyticsFormBridgeTransaction,
-  queueAnalyticsFormDelivery,
-  recordAnalyticsFormSubmission,
-  redeemAnalyticsLinkerRecord,
-  refreshAnalyticsDailyRollups,
-  refreshAnalyticsDailyRollupsIfDue,
-  requireAnalyticsAccess,
-  requireAnalyticsEditor,
-  resolveAnalyticsCampaignLink,
-  resolveAnalyticsConfirmedActivationIdentity,
-  retryAnalyticsActivationJob,
-  sanitizeAnalyticsProperties,
-  sanitizeClickIds,
-  setAnalyticsActivationReadiness,
-  setAnalyticsPixelDomainState,
-  sweepAnalyticsRestrictedRetention,
-  testAnalyticsActivationDestination,
-  updateAnalyticsBusinessModel,
-  updateAnalyticsPixel,
-  upsertAnalyticsAdSpend,
-  upsertAnalyticsHostGroup
-};