mcp-scraper 0.66.0 → 0.66.2

package/dist/chunk-AAGVOPP5.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/api/webhook.ts","../src/api/harvest-problems.ts","../src/api/harvest-attempt-events.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import {\n  buildPublicErrorEnvelope,\n  CaptchaError,\n  LocationMismatchError,\n  PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n  RequestAbortedError,\n  type PublicChargeStatus,\n  type PublicErrorDetails,\n  type PublicErrorEnvelope,\n  publicErrorMessage,\n} from '../errors.js'\nimport { isVendorUnavailableError } from './vendor-errors.js'\n\nexport type HarvestProblemCode =\n  | 'request_aborted'\n  | 'captcha_exhausted'\n  | 'location_mismatch'\n  | 'proxy_tunnel_failed'\n  | 'proxy_unavailable'\n  | 'harvest_timeout'\n  | 'service_unavailable'\n  | 'vendor_unavailable'\n  | 'browser_session_interrupted'\n  | 'extraction_failed'\n\nexport interface HarvestProblem {\n  error_code: HarvestProblemCode\n  error_type: string\n  message: string\n  retryable: boolean\n  httpStatus: number\n  terminalStatus: 'cancelled' | 'failed'\n}\n\nexport interface PublicErrorContext {\n  retryAfterSeconds?: number\n  chargeStatus?: PublicChargeStatus\n  details?: PublicErrorDetails\n}\n\nexport type PublicErrorResponse = PublicErrorEnvelope & { error: string }\n\nfunction errorMessage(err: unknown): string {\n  return err instanceof Error ? err.message : String(err)\n}\n\nfunction looksLikeTimeout(err: unknown, message: string): boolean {\n  if (err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError')) return true\n  return /timeout|timed out|Timeout \\d+ms exceeded|deadline/i.test(message)\n}\n\nfunction looksLikeCaptcha(message: string): boolean {\n  return /captcha|recaptcha|unusual traffic|google\\.com\\/sorry|blocked/i.test(message)\n}\n\nfunction looksLikeProxyTunnelFailure(message: string): boolean {\n  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message)\n}\n\nfunction looksLikeProxyUnavailable(message: string): boolean {\n  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message)\n}\n\nfunction looksLikeInternalServiceUnavailable(message: string): boolean {\n  return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*dashboard/i.test(message)\n}\n\nfunction looksLikeBrowserSessionInterrupted(message: string): boolean {\n  return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message)\n}\n\nexport function classifyHarvestProblem(err: unknown): HarvestProblem {\n  const message = errorMessage(err)\n\n  if (err instanceof RequestAbortedError) {\n    return {\n      error_code: 'request_aborted',\n      error_type: 'request_aborted',\n      message: publicErrorMessage('request_aborted'),\n      retryable: true,\n      httpStatus: 408,\n      terminalStatus: 'cancelled',\n    }\n  }\n\n  if (err instanceof CaptchaError || looksLikeCaptcha(message)) {\n    return {\n      error_code: 'captcha_exhausted',\n      error_type: 'captcha',\n      message: publicErrorMessage('captcha_exhausted'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (isVendorUnavailableError(err, message)) {\n    return {\n      error_code: 'vendor_unavailable',\n      error_type: 'service_unavailable',\n      message: publicErrorMessage('vendor_unavailable'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (err instanceof LocationMismatchError) {\n    return {\n      error_code: 'location_mismatch',\n      error_type: 'location_mismatch',\n      message: publicErrorMessage('location_mismatch'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyTunnelFailure(message)) {\n    return {\n      error_code: 'proxy_tunnel_failed',\n      error_type: 'connection',\n      message: publicErrorMessage('proxy_tunnel_failed'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyUnavailable(message)) {\n    return {\n      error_code: 'proxy_unavailable',\n      error_type: 'connection',\n      message: publicErrorMessage('proxy_unavailable'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeBrowserSessionInterrupted(message)) {\n    return {\n      error_code: 'browser_session_interrupted',\n      error_type: 'extraction',\n      message: publicErrorMessage('browser_session_interrupted'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeTimeout(err, message)) {\n    return {\n      error_code: 'harvest_timeout',\n      error_type: 'timeout',\n      message: publicErrorMessage('harvest_timeout'),\n      retryable: true,\n      httpStatus: 504,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeInternalServiceUnavailable(message)) {\n    return {\n      error_code: 'service_unavailable',\n      error_type: 'service_unavailable',\n      message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  return {\n    error_code: 'extraction_failed',\n    error_type: 'extraction',\n    message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n    retryable: false,\n    httpStatus: 500,\n    terminalStatus: 'failed',\n  }\n}\n\nexport function serializeHarvestProblem(problem: HarvestProblem): string {\n  return JSON.stringify({\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    message: problem.message,\n    retryable: problem.retryable,\n  })\n}\n\nexport function harvestProblemResponse(problem: HarvestProblem, context: PublicErrorContext = {}): PublicErrorResponse {\n  const envelope = buildPublicErrorEnvelope({\n    errorCode: problem.error_code,\n    errorType: problem.error_type,\n    retryable: problem.retryable,\n    retryAfterSeconds: context.retryAfterSeconds,\n    chargeStatus: context.chargeStatus,\n    details: context.details,\n  })\n  return {\n    error: envelope.message,\n    ...envelope,\n  }\n}\n\nexport function harvestProblemEnvelope(problem: HarvestProblem, context: PublicErrorContext = {}): PublicErrorEnvelope {\n  const { error: _legacyAlias, ...envelope } = harvestProblemResponse(problem, context)\n  return envelope\n}\n\n/**\n * Shared public boundary for private execution failures. Callers may attach\n * settlement state only when their own billing branch has proved it.\n */\nexport function publicErrorResponse(err: unknown, context: PublicErrorContext = {}): PublicErrorResponse {\n  return harvestProblemResponse(classifyHarvestProblem(err), context)\n}\n\nexport interface PublicErrorBoundaryResult {\n  body: PublicErrorResponse\n  status: 408 | 500 | 503 | 504\n}\n\n/**\n * Classify an execution failure once so HTTP routes cannot drift from the\n * shared public envelope or accidentally expose the private exception text.\n */\nexport function publicErrorBoundary(err: unknown, context: PublicErrorContext = {}): PublicErrorBoundaryResult {\n  const problem = classifyHarvestProblem(err)\n  return {\n    body: harvestProblemResponse(problem, context),\n    status: problem.httpStatus as PublicErrorBoundaryResult['status'],\n  }\n}\n","import type { HarvestAttemptLogEvent } from '../harvest.js'\nimport { finishHarvestAttempt, startHarvestAttempt } from './db.js'\n\nexport function createHarvestAttemptRecorder(jobId: string, userId: number | bigint) {\n  return async (event: HarvestAttemptLogEvent): Promise<void> => {\n    if (event.type === 'started') {\n      await startHarvestAttempt({\n        jobId,\n        userId,\n        attemptNumber: event.attemptNumber,\n        maxAttempts: event.maxAttempts,\n        query: event.query,\n        location: event.location,\n        maxQuestions: event.maxQuestions,\n        startedAt: event.startedAt,\n      })\n      return\n    }\n\n    await finishHarvestAttempt({\n      jobId,\n      attemptNumber: event.attemptNumber,\n      outcome: event.outcome,\n      kernelSessionId: event.kernelSessionId,\n      questionCount: event.questionCount,\n      durationMs: event.durationMs,\n      error: event.error,\n      willRetry: event.willRetry,\n      kernelDeleteStarted: event.cleanup.kernelDeleteStarted,\n      kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,\n      kernelDeleteError: event.cleanup.kernelDeleteError,\n      browserCloseSucceeded: event.cleanup.browserCloseSucceeded,\n      browserCloseError: event.cleanup.browserCloseError,\n      debug: event.debug,\n      completedAt: event.completedAt,\n    })\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACyBA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,iBAAiB,KAAc,SAA0B;AAChE,MAAI,eAAe,iBAAiB,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAe,QAAO;AACtG,SAAO,qDAAqD,KAAK,OAAO;AAC1E;AAEA,SAAS,iBAAiB,SAA0B;AAClD,SAAO,gEAAgE,KAAK,OAAO;AACrF;AAEA,SAAS,4BAA4B,SAA0B;AAC7D,SAAO,gKAAgK,KAAK,OAAO;AACrL;AAEA,SAAS,0BAA0B,SAA0B;AAC3D,SAAO,4GAA4G,KAAK,OAAO;AACjI;AAEA,SAAS,oCAAoC,SAA0B;AACrE,SAAO,wJAAwJ,KAAK,OAAO;AAC7K;AAEA,SAAS,mCAAmC,SAA0B;AACpE,SAAO,iJAAiJ,KAAK,OAAO;AACtK;AAEO,SAAS,uBAAuB,KAA8B;AACnE,QAAM,UAAU,aAAa,GAAG;AAEhC,MAAI,eAAe,qBAAqB;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,iBAAiB;AAAA,MAC7C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,gBAAgB,iBAAiB,OAAO,GAAG;AAC5D,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,yBAAyB,KAAK,OAAO,GAAG;AAC1C,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,oBAAoB;AAAA,MAChD,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,uBAAuB;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,4BAA4B,OAAO,GAAG;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,qBAAqB;AAAA,MACjD,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,0BAA0B,OAAO,GAAG;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,mCAAmC,OAAO,GAAG;AAC/C,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,6BAA6B;AAAA,MACzD,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,iBAAiB,KAAK,OAAO,GAAG;AAClC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,iBAAiB;AAAA,MAC7C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,oCAAoC,OAAO,GAAG;AAChD,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,gBAAgB;AAAA,EAClB;AACF;AAEO,SAAS,wBAAwB,SAAiC;AACvE,SAAO,KAAK,UAAU;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,SAAS,QAAQ;AAAA,IACjB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACH;AAEO,SAAS,uBAAuB,SAAyB,UAA8B,CAAC,GAAwB;AACrH,QAAM,WAAW,yBAAyB;AAAA,IACxC,WAAW,QAAQ;AAAA,IACnB,WAAW,QAAQ;AAAA,IACnB,WAAW,QAAQ;AAAA,IACnB,mBAAmB,QAAQ;AAAA,IAC3B,cAAc,QAAQ;AAAA,IACtB,SAAS,QAAQ;AAAA,EACnB,CAAC;AACD,SAAO;AAAA,IACL,OAAO,SAAS;AAAA,IAChB,GAAG;AAAA,EACL;AACF;AAEO,SAAS,uBAAuB,SAAyB,UAA8B,CAAC,GAAwB;AACrH,QAAM,EAAE,OAAO,cAAc,GAAG,SAAS,IAAI,uBAAuB,SAAS,OAAO;AACpF,SAAO;AACT;AAmBO,SAAS,oBAAoB,KAAc,UAA8B,CAAC,GAA8B;AAC7G,QAAM,UAAU,uBAAuB,GAAG;AAC1C,SAAO;AAAA,IACL,MAAM,uBAAuB,SAAS,OAAO;AAAA,IAC7C,QAAQ,QAAQ;AAAA,EAClB;AACF;;;ACxOO,SAAS,6BAA6B,OAAe,QAAyB;AACnF,SAAO,OAAO,UAAiD;AAC7D,QAAI,MAAM,SAAS,WAAW;AAC5B,YAAM,oBAAoB;AAAA,QACxB;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,aAAa,MAAM;AAAA,QACnB,OAAO,MAAM;AAAA,QACb,UAAU,MAAM;AAAA,QAChB,cAAc,MAAM;AAAA,QACpB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA,eAAe,MAAM;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,iBAAiB,MAAM;AAAA,MACvB,eAAe,MAAM;AAAA,MACrB,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,WAAW,MAAM;AAAA,MACjB,qBAAqB,MAAM,QAAQ;AAAA,MACnC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,OAAO,MAAM;AAAA,MACb,aAAa,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACF;","names":[]}