mcp-scraper 0.38.2 → 0.40.1

package/dist/chunk-SFRMFGQ6.js

@@ -1,158 +0,0 @@
-import {
-  CaptchaError,
-  LocationMismatchError,
-  RequestAbortedError
-} from "./chunk-44HZLHDV.js";
-import {
-  finishHarvestAttempt,
-  startHarvestAttempt
-} from "./chunk-N7KUTTCC.js";
-
-// src/api/harvest-problems.ts
-function errorMessage(err) {
-  return err instanceof Error ? err.message : String(err);
-}
-function looksLikeTimeout(err, message) {
-  if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
-  return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
-}
-function looksLikeCaptcha(message) {
-  return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
-}
-function looksLikeProxyTunnelFailure(message) {
-  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
-}
-function looksLikeProxyUnavailable(message) {
-  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
-}
-function classifyHarvestProblem(err) {
-  const message = errorMessage(err);
-  if (err instanceof RequestAbortedError) {
-    return {
-      error_code: "request_aborted",
-      error_type: "request_aborted",
-      message,
-      retryable: true,
-      httpStatus: 408,
-      terminalStatus: "cancelled"
-    };
-  }
-  if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
-    return {
-      error_code: "captcha_exhausted",
-      error_type: "captcha",
-      message,
-      retryable: true,
-      httpStatus: 503,
-      terminalStatus: "failed"
-    };
-  }
-  if (err instanceof LocationMismatchError) {
-    return {
-      error_code: "location_mismatch",
-      error_type: "location_mismatch",
-      message,
-      retryable: true,
-      httpStatus: 503,
-      terminalStatus: "failed"
-    };
-  }
-  if (looksLikeProxyTunnelFailure(message)) {
-    return {
-      error_code: "proxy_tunnel_failed",
-      error_type: "proxy_tunnel_failed",
-      message,
-      retryable: true,
-      httpStatus: 503,
-      terminalStatus: "failed"
-    };
-  }
-  if (looksLikeProxyUnavailable(message)) {
-    return {
-      error_code: "proxy_unavailable",
-      error_type: "proxy_unavailable",
-      message,
-      retryable: true,
-      httpStatus: 503,
-      terminalStatus: "failed"
-    };
-  }
-  if (looksLikeTimeout(err, message)) {
-    return {
-      error_code: "harvest_timeout",
-      error_type: "timeout",
-      message,
-      retryable: true,
-      httpStatus: 504,
-      terminalStatus: "failed"
-    };
-  }
-  return {
-    error_code: "extraction_failed",
-    error_type: "extraction",
-    message,
-    retryable: false,
-    httpStatus: 500,
-    terminalStatus: "failed"
-  };
-}
-function serializeHarvestProblem(problem) {
-  return JSON.stringify({
-    error_code: problem.error_code,
-    error_type: problem.error_type,
-    message: problem.message,
-    retryable: problem.retryable
-  });
-}
-function harvestProblemResponse(problem) {
-  return {
-    error: problem.message,
-    error_code: problem.error_code,
-    error_type: problem.error_type,
-    retryable: problem.retryable
-  };
-}
-
-// src/api/harvest-attempt-events.ts
-function createHarvestAttemptRecorder(jobId, userId) {
-  return async (event) => {
-    if (event.type === "started") {
-      await startHarvestAttempt({
-        jobId,
-        userId,
-        attemptNumber: event.attemptNumber,
-        maxAttempts: event.maxAttempts,
-        query: event.query,
-        location: event.location,
-        maxQuestions: event.maxQuestions,
-        startedAt: event.startedAt
-      });
-      return;
-    }
-    await finishHarvestAttempt({
-      jobId,
-      attemptNumber: event.attemptNumber,
-      outcome: event.outcome,
-      kernelSessionId: event.kernelSessionId,
-      questionCount: event.questionCount,
-      durationMs: event.durationMs,
-      error: event.error,
-      willRetry: event.willRetry,
-      kernelDeleteStarted: event.cleanup.kernelDeleteStarted,
-      kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,
-      kernelDeleteError: event.cleanup.kernelDeleteError,
-      browserCloseSucceeded: event.cleanup.browserCloseSucceeded,
-      browserCloseError: event.cleanup.browserCloseError,
-      debug: event.debug,
-      completedAt: event.completedAt
-    });
-  };
-}
-
-export {
-  classifyHarvestProblem,
-  serializeHarvestProblem,
-  harvestProblemResponse,
-  createHarvestAttemptRecorder
-};
-//# sourceMappingURL=chunk-SFRMFGQ6.js.map

package/dist/chunk-SFRMFGQ6.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/api/harvest-problems.ts","../src/api/harvest-attempt-events.ts"],"sourcesContent":["import { CaptchaError, LocationMismatchError, RequestAbortedError } from '../errors.js'\n\nexport type HarvestProblemCode =\n  | 'request_aborted'\n  | 'captcha_exhausted'\n  | 'location_mismatch'\n  | 'proxy_tunnel_failed'\n  | 'proxy_unavailable'\n  | 'harvest_timeout'\n  | 'extraction_failed'\n\nexport interface HarvestProblem {\n  error_code: HarvestProblemCode\n  error_type: string\n  message: string\n  retryable: boolean\n  httpStatus: number\n  terminalStatus: 'cancelled' | 'failed'\n}\n\nfunction errorMessage(err: unknown): string {\n  return err instanceof Error ? err.message : String(err)\n}\n\nfunction looksLikeTimeout(err: unknown, message: string): boolean {\n  if (err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError')) return true\n  return /timeout|timed out|Timeout \\d+ms exceeded|deadline/i.test(message)\n}\n\nfunction looksLikeCaptcha(message: string): boolean {\n  return /captcha|recaptcha|unusual traffic|google\\.com\\/sorry|blocked/i.test(message)\n}\n\nfunction looksLikeProxyTunnelFailure(message: string): boolean {\n  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message)\n}\n\nfunction looksLikeProxyUnavailable(message: string): boolean {\n  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message)\n}\n\nexport function classifyHarvestProblem(err: unknown): HarvestProblem {\n  const message = errorMessage(err)\n\n  if (err instanceof RequestAbortedError) {\n    return {\n      error_code: 'request_aborted',\n      error_type: 'request_aborted',\n      message,\n      retryable: true,\n      httpStatus: 408,\n      terminalStatus: 'cancelled',\n    }\n  }\n\n  if (err instanceof CaptchaError || looksLikeCaptcha(message)) {\n    return {\n      error_code: 'captcha_exhausted',\n      error_type: 'captcha',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (err instanceof LocationMismatchError) {\n    return {\n      error_code: 'location_mismatch',\n      error_type: 'location_mismatch',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyTunnelFailure(message)) {\n    return {\n      error_code: 'proxy_tunnel_failed',\n      error_type: 'proxy_tunnel_failed',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyUnavailable(message)) {\n    return {\n      error_code: 'proxy_unavailable',\n      error_type: 'proxy_unavailable',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeTimeout(err, message)) {\n    return {\n      error_code: 'harvest_timeout',\n      error_type: 'timeout',\n      message,\n      retryable: true,\n      httpStatus: 504,\n      terminalStatus: 'failed',\n    }\n  }\n\n  return {\n    error_code: 'extraction_failed',\n    error_type: 'extraction',\n    message,\n    retryable: false,\n    httpStatus: 500,\n    terminalStatus: 'failed',\n  }\n}\n\nexport function serializeHarvestProblem(problem: HarvestProblem): string {\n  return JSON.stringify({\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    message: problem.message,\n    retryable: problem.retryable,\n  })\n}\n\nexport function harvestProblemResponse(problem: HarvestProblem): {\n  error: string\n  error_code: HarvestProblemCode\n  error_type: string\n  retryable: boolean\n} {\n  return {\n    error: problem.message,\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    retryable: problem.retryable,\n  }\n}\n","import type { HarvestAttemptLogEvent } from '../harvest.js'\nimport { finishHarvestAttempt, startHarvestAttempt } from './db.js'\n\nexport function createHarvestAttemptRecorder(jobId: string, userId: number | bigint) {\n  return async (event: HarvestAttemptLogEvent): Promise<void> => {\n    if (event.type === 'started') {\n      await startHarvestAttempt({\n        jobId,\n        userId,\n        attemptNumber: event.attemptNumber,\n        maxAttempts: event.maxAttempts,\n        query: event.query,\n        location: event.location,\n        maxQuestions: event.maxQuestions,\n        startedAt: event.startedAt,\n      })\n      return\n    }\n\n    await finishHarvestAttempt({\n      jobId,\n      attemptNumber: event.attemptNumber,\n      outcome: event.outcome,\n      kernelSessionId: event.kernelSessionId,\n      questionCount: event.questionCount,\n      durationMs: event.durationMs,\n      error: event.error,\n      willRetry: event.willRetry,\n      kernelDeleteStarted: event.cleanup.kernelDeleteStarted,\n      kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,\n      kernelDeleteError: event.cleanup.kernelDeleteError,\n      browserCloseSucceeded: event.cleanup.browserCloseSucceeded,\n      browserCloseError: event.cleanup.browserCloseError,\n      debug: event.debug,\n      completedAt: event.completedAt,\n    })\n  }\n}\n"],"mappings":";;;;;;;;;;;AAoBA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,iBAAiB,KAAc,SAA0B;AAChE,MAAI,eAAe,iBAAiB,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAe,QAAO;AACtG,SAAO,qDAAqD,KAAK,OAAO;AAC1E;AAEA,SAAS,iBAAiB,SAA0B;AAClD,SAAO,gEAAgE,KAAK,OAAO;AACrF;AAEA,SAAS,4BAA4B,SAA0B;AAC7D,SAAO,gKAAgK,KAAK,OAAO;AACrL;AAEA,SAAS,0BAA0B,SAA0B;AAC3D,SAAO,4GAA4G,KAAK,OAAO;AACjI;AAEO,SAAS,uBAAuB,KAA8B;AACnE,QAAM,UAAU,aAAa,GAAG;AAEhC,MAAI,eAAe,qBAAqB;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,gBAAgB,iBAAiB,OAAO,GAAG;AAC5D,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,uBAAuB;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,4BAA4B,OAAO,GAAG;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,0BAA0B,OAAO,GAAG;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,iBAAiB,KAAK,OAAO,GAAG;AAClC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,gBAAgB;AAAA,EAClB;AACF;AAEO,SAAS,wBAAwB,SAAiC;AACvE,SAAO,KAAK,UAAU;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,SAAS,QAAQ;AAAA,IACjB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACH;AAEO,SAAS,uBAAuB,SAKrC;AACA,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB;AACF;;;AC1IO,SAAS,6BAA6B,OAAe,QAAyB;AACnF,SAAO,OAAO,UAAiD;AAC7D,QAAI,MAAM,SAAS,WAAW;AAC5B,YAAM,oBAAoB;AAAA,QACxB;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,aAAa,MAAM;AAAA,QACnB,OAAO,MAAM;AAAA,QACb,UAAU,MAAM;AAAA,QAChB,cAAc,MAAM;AAAA,QACpB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA,eAAe,MAAM;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,iBAAiB,MAAM;AAAA,MACvB,eAAe,MAAM;AAAA,MACrB,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,WAAW,MAAM;AAAA,MACjB,qBAAqB,MAAM,QAAQ;AAAA,MACnC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,OAAO,MAAM;AAAA,MACb,aAAa,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/chunk-YCI2PNCS.js

@@ -1,499 +0,0 @@
-import {
-  LedgerOperation
-} from "./chunk-AZRPG43B.js";
-import {
-  sanitizeVendorName
-} from "./chunk-44HZLHDV.js";
-import {
-  CREDIT_LOT_TTL,
-  getDb,
-  settleDebitMcIdempotent
-} from "./chunk-N7KUTTCC.js";
-
-// src/api/site-extract-repository.ts
-function extractJobLimitInfo(job) {
-  const effectiveMaxPages = Math.max(1, Number(job.options.effectiveMaxPages ?? job.options.maxPages ?? 1));
-  const requestedMaxPages = Math.max(effectiveMaxPages, Number(job.options.requestedMaxPages ?? effectiveMaxPages));
-  const creditLimited = requestedMaxPages > effectiveMaxPages;
-  return {
-    requestedMaxPages,
-    effectiveMaxPages,
-    creditLimited,
-    // If discovery exhausted below the funded cap, the smaller hold did not
-    // actually truncate this crawl. Hitting the cap is conservatively partial.
-    creditTruncated: creditLimited && job.totalUrls >= effectiveMaxPages
-  };
-}
-function terminalExtractJobStatus(progress, creditTruncated = false) {
-  if (progress.successfulUrls === 0) return "failed";
-  if (progress.failedUrls > 0 || progress.remainingUrls > 0 || creditTruncated) return "partial";
-  return "complete";
-}
-function rowToJob(r) {
-  const totalUrls = Number(r.total_urls ?? 0);
-  const legacyProgress = r.attempted_urls == null;
-  const attemptedUrls = Number(legacyProgress ? r.done_urls ?? 0 : r.attempted_urls);
-  const successfulUrls = Number(legacyProgress ? r.done_urls ?? 0 : r.successful_urls ?? 0);
-  const failedUrls = Number(r.failed_urls ?? Math.max(0, attemptedUrls - successfulUrls));
-  return {
-    id: String(r.id),
-    userId: r.user_id != null ? Number(r.user_id) : null,
-    idempotencyKey: r.idempotency_key != null ? String(r.idempotency_key) : null,
-    requestFingerprint: r.request_fingerprint != null ? String(r.request_fingerprint) : null,
-    status: r.status != null ? String(r.status) : "pending",
-    startUrl: String(r.start_url ?? ""),
-    options: r.options ? JSON.parse(String(r.options)) : {},
-    totalUrls,
-    doneUrls: attemptedUrls,
-    attemptedUrls,
-    successfulUrls,
-    failedUrls,
-    remainingUrls: Math.max(0, totalUrls - attemptedUrls),
-    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
-    error: r.error != null ? String(r.error) : null,
-    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
-    createdAt: String(r.created_at ?? ""),
-    updatedAt: String(r.updated_at ?? "")
-  };
-}
-async function createExtractJob(jobId, userId, startUrl, options) {
-  const db = getDb();
-  await db.execute({
-    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
-    args: [jobId, userId, startUrl, JSON.stringify(options)]
-  });
-}
-async function getExtractJobByIdempotencyKey(userId, idempotencyKey) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1`,
-    args: [userId, idempotencyKey]
-  });
-  return res.rows[0] ? rowToJob(res.rows[0]) : null;
-}
-async function createOrGetExtractJob(input) {
-  const db = getDb();
-  const inserted = await db.execute({
-    sql: `INSERT OR IGNORE INTO site_extract_jobs
-            (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,
-    args: [
-      input.jobId,
-      input.userId,
-      input.startUrl,
-      JSON.stringify(input.options),
-      input.idempotencyKey,
-      input.requestFingerprint
-    ]
-  });
-  const job = await getExtractJobByIdempotencyKey(input.userId, input.idempotencyKey);
-  if (!job) throw new Error("idempotent extract job was not persisted");
-  return {
-    job,
-    created: inserted.rowsAffected === 1,
-    conflict: job.requestFingerprint !== input.requestFingerprint
-  };
-}
-async function getExtractJob(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
-  return res.rows[0] ? rowToJob(res.rows[0]) : null;
-}
-async function listExtractJobs(userId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50`, args: [userId] });
-  return res.rows.map((r) => rowToJob(r));
-}
-async function listUnsettledExtractJobs(limit = 25) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT * FROM site_extract_jobs
-          WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
-            AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
-          ORDER BY updated_at ASC LIMIT ?`,
-    args: [Math.max(1, Math.min(100, Math.round(limit)))]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function listFundedPendingExtractJobs(limit = 25, staleMinutes = 2) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(1, Math.min(60, Math.round(staleMinutes)));
-  const res = await db.execute({
-    sql: `SELECT j.* FROM site_extract_jobs j
-          JOIN billing_debits d
-            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
-           AND d.user_id = j.user_id
-           AND d.status = 'applied'
-          WHERE j.status = 'pending'
-            AND j.billed_mc IS NULL
-            AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
-            AND j.updated_at <= datetime('now', ?)
-          ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function markExtractJobDispatchAttempt(jobId) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET dispatch_attempts = dispatch_attempts + 1,
-              updated_at = datetime('now'),
-              next_dispatch_at = datetime('now', '+5 minutes'),
-              dispatch_error = NULL,
-              status = CASE
-                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
-                ELSE status
-              END,
-              error = CASE
-                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
-                  THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'
-                ELSE error
-              END
-          WHERE id = ? AND status = 'pending'`,
-    args: [jobId]
-  });
-  return getExtractJob(jobId);
-}
-async function recordExtractJobDispatchFailure(jobId, error) {
-  const db = getDb();
-  const safeError = sanitizeVendorName(error).slice(0, 1e3);
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET
-            dispatch_attempts = dispatch_attempts + 1,
-            dispatch_error = ?,
-            next_dispatch_at = datetime('now', '+5 minutes'),
-            updated_at = datetime('now'),
-            status = CASE
-              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
-              ELSE status
-            END,
-            error = CASE
-              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
-                THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'
-              ELSE error
-            END
-          WHERE id = ? AND status = 'pending'`,
-    args: [safeError, jobId]
-  });
-  return getExtractJob(jobId);
-}
-async function recordExtractSettlementFailure(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET
-            settlement_attempts = settlement_attempts + 1,
-            settlement_error = ?,
-            next_settlement_at = datetime('now', '+15 minutes'),
-            updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 1e3), jobId]
-  });
-}
-async function listStaleRunningExtractJobs(limit = 25, staleMinutes = 60) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)));
-  const res = await db.execute({
-    sql: `SELECT j.* FROM site_extract_jobs j
-          LEFT JOIN billing_debits d
-            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
-           AND d.user_id = j.user_id
-           AND d.status = 'applied'
-          WHERE j.status = 'running'
-            AND j.billed_mc IS NULL
-            AND (
-              json_extract(j.options, '$.debitKey') IS NULL
-              OR d.idempotency_key IS NOT NULL
-            )
-            AND j.updated_at <= datetime('now', ?)
-          ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function failStaleRunningExtractJob(jobId, staleMinutes = 60) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)));
-  const result = await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET status = 'failed',
-              error = 'Background crawl stopped without a heartbeat; completed pages were preserved and the unused hold will be refunded.',
-              updated_at = datetime('now')
-          WHERE id = ?
-            AND status = 'running'
-            AND billed_mc IS NULL
-            AND updated_at <= datetime('now', ?)`,
-    args: [jobId, `-${safeStaleMinutes} minutes`]
-  });
-  return result.rowsAffected === 1;
-}
-async function claimFailedExtractJobForRefinalize(jobId) {
-  const db = getDb();
-  const claimed = await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET status = 'running', updated_at = datetime('now')
-          WHERE id = ? AND status = 'failed'`,
-    args: [jobId]
-  });
-  if (claimed.rowsAffected !== 1) return null;
-  return getExtractJob(jobId);
-}
-async function abandonExtractSettlement(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 1e3), jobId]
-  });
-}
-async function setExtractJobTotal(jobId, totalUrls) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-    args: [totalUrls, jobId]
-  });
-}
-async function saveExtractPages(jobId, pages) {
-  if (pages.length === 0) return;
-  const db = getDb();
-  await db.batch([
-    ...pages.map((p) => {
-      const { discoveryLinks: _ephemeralDiscoveryLinks, ...storedPage } = p;
-      return {
-        sql: `INSERT INTO site_extract_pages (job_id, url, page)
-            SELECT ?, ?, ?
-            WHERE EXISTS (
-              SELECT 1 FROM site_extract_jobs
-              WHERE id = ? AND status IN ('pending', 'running')
-            )
-            ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
-            WHERE NOT CASE
-                    WHEN json_valid(site_extract_pages.page) = 0 THEN 0
-                    WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(site_extract_pages.page, '$.status') >= 200
-                      AND json_extract(site_extract_pages.page, '$.status') < 300
-                      AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END
-               OR CASE
-                    WHEN json_valid(excluded.page) = 0 THEN 0
-                    WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(excluded.page, '$.status') >= 200
-                      AND json_extract(excluded.page, '$.status') < 300
-                      AND json_extract(excluded.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END`,
-        args: [jobId, p.archivedUrl ?? p.url, JSON.stringify(storedPage), jobId]
-      };
-    }),
-    {
-      sql: `UPDATE site_extract_jobs SET
-            done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            successful_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0
-              END
-            ),
-            failed_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND NOT CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE COALESCE(
-                  json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0,
-                  0
-                )
-              END
-            ),
-            updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-      args: [jobId, jobId, jobId, jobId, jobId]
-    }
-  ], "write");
-}
-async function getExtractedPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
-  return res.rows.map((r) => JSON.parse(String(r.page)));
-}
-async function getExtractedImageLinks(jobId, limit = 181) {
-  const db = getDb();
-  const safeLimit = Math.max(1, Math.min(5001, Math.round(limit)));
-  const res = await db.execute({
-    sql: `SELECT DISTINCT CAST(images.value AS TEXT) AS url
-          FROM site_extract_pages p,
-               json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
-          WHERE p.job_id = ?
-            AND images.type = 'text'
-            AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
-          LIMIT ?`,
-    args: [jobId, safeLimit]
-  });
-  return res.rows.map((row) => String(row.url));
-}
-async function countSuccessfulPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT COUNT(*) AS n FROM site_extract_pages
-            WHERE job_id = ? AND CASE
-              WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                THEN json_extract(page, '$.extractionStatus') = 'successful'
-              ELSE COALESCE(
-                json_extract(page, '$.status') >= 200
-                AND json_extract(page, '$.status') < 300
-                AND json_extract(page, '$.wordCount') > 0,
-                0
-              )
-            END`,
-    args: [jobId]
-  });
-  return Number(res.rows[0]?.n ?? 0);
-}
-async function getExtractedUrls(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT url FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
-  return new Set(res.rows.map((r) => String(r.url)));
-}
-async function finishExtractJob(jobId, artifacts, status, error = null, allowFailedRecovery = false) {
-  const db = getDb();
-  const result = await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = ?, artifacts = ?, error = ?, updated_at = datetime('now')
-          WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,
-    args: [status, JSON.stringify(artifacts ?? []), error ? sanitizeVendorName(error).slice(0, 2e3) : null, jobId, allowFailedRecovery ? 1 : 0]
-  });
-  return result.rowsAffected === 1;
-}
-async function completeExtractJob(jobId, artifacts) {
-  await finishExtractJob(jobId, artifacts, "complete");
-}
-async function failExtractJob(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'failed', error = ?, updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-    args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
-  });
-}
-async function failUnfundedExtractJob(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'failed', billed_mc = 0, error = ?, updated_at = datetime('now')
-          WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
-  });
-}
-async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
-  const db = getDb();
-  const job = await getExtractJob(jobId);
-  if (!job || job.billedMc != null) return;
-  const debitKey = typeof job.options.debitKey === "string" ? job.options.debitKey : null;
-  if (debitKey) {
-    const settlement = await settleDebitMcIdempotent(
-      userId,
-      debitKey,
-      Math.max(0, netChargeMc),
-      LedgerOperation.EXTRACT_SITE_REFUND,
-      reference,
-      "extract_site"
-    );
-    await db.execute({
-      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`,
-      args: [settlement.final_amount_mc, jobId]
-    });
-    return;
-  }
-  const safeRefundMc = Math.max(0, refundMc);
-  const safeNetChargeMc = Math.max(0, netChargeMc);
-  await db.batch([
-    {
-      sql: `UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL`,
-      args: [jobId]
-    },
-    {
-      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
-            SELECT ?, ?, ?, ?, datetime(j.created_at, ?)
-            FROM site_extract_jobs j
-            WHERE j.id = ?
-              AND j.billed_mc = -1
-              AND ? > 0
-              AND changes() = 1
-              AND datetime(j.created_at, ?) > datetime('now')`,
-      args: [
-        userId,
-        safeRefundMc,
-        safeRefundMc,
-        LedgerOperation.EXTRACT_SITE_REFUND,
-        CREDIT_LOT_TTL,
-        jobId,
-        safeRefundMc,
-        CREDIT_LOT_TTL
-      ]
-    },
-    {
-      sql: `UPDATE users SET balance_mc = balance_mc + ?
-            WHERE id = ? AND ? > 0 AND changes() = 1`,
-      args: [safeRefundMc, userId, safeRefundMc]
-    },
-    {
-      sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
-            SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,
-      args: [userId, safeRefundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference, safeRefundMc]
-    },
-    {
-      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')
-            WHERE id = ? AND billed_mc = -1`,
-      args: [safeNetChargeMc, jobId]
-    }
-  ], "write");
-}
-
-export {
-  extractJobLimitInfo,
-  terminalExtractJobStatus,
-  createExtractJob,
-  getExtractJobByIdempotencyKey,
-  createOrGetExtractJob,
-  getExtractJob,
-  listExtractJobs,
-  listUnsettledExtractJobs,
-  listFundedPendingExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  listStaleRunningExtractJobs,
-  failStaleRunningExtractJob,
-  claimFailedExtractJobForRefinalize,
-  abandonExtractSettlement,
-  setExtractJobTotal,
-  saveExtractPages,
-  getExtractedPages,
-  getExtractedImageLinks,
-  countSuccessfulPages,
-  getExtractedUrls,
-  finishExtractJob,
-  completeExtractJob,
-  failExtractJob,
-  failUnfundedExtractJob,
-  settleExtractJob
-};
-//# sourceMappingURL=chunk-YCI2PNCS.js.map