mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-IHXAXYIS.js

@@ -1,843 +0,0 @@
-import {
-  createPrivateArtifact,
-  createPrivateArtifactFromStream,
-  privateArtifactOwnerId,
-  readPrivateArtifactBuffer,
-  renewPrivateArtifactDownload
-} from "./chunk-DNM65UCK.js";
-import {
-  LedgerOperation
-} from "./chunk-4QMUF6XM.js";
-import {
-  CREDIT_LOT_TTL,
-  getDb,
-  parsePublicErrorEnvelope,
-  sanitizeVendorName,
-  serializePublicErrorEnvelope,
-  settleDebitMcIdempotent
-} from "./chunk-YXNDOQXN.js";
-
-// src/api/site-extract-content-store.ts
-import { createHash } from "crypto";
-import { gunzipSync, gzipSync } from "zlib";
-
-// src/api/site-extract-artifacts.ts
-var SITE_EXTRACT_ARTIFACT_PREFIX = "site-extracts/";
-var SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
-var SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
-function siteExtractArtifactToken() {
-  return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
-}
-function hostedByEnvironment() {
-  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
-}
-function policy() {
-  return {
-    prefix: SITE_EXTRACT_ARTIFACT_PREFIX,
-    artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,
-    downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,
-    token: siteExtractArtifactToken()
-  };
-}
-function siteExtractArtifactOwnerId(artifactId) {
-  return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX);
-}
-async function createSiteExtractBundleArtifactStream(args) {
-  const pointer = await createPrivateArtifactFromStream({
-    policy: policy(),
-    ownerId: args.ownerId,
-    artifactKey: `${args.jobId}.zip`,
-    createdAt: args.createdAt,
-    filename: `${args.jobId}-site-export.zip`,
-    contentType: "application/zip",
-    content: args.content
-  });
-  return {
-    key: pointer.artifactId,
-    url: pointer.downloadUrl ?? "",
-    bytes: pointer.bytes,
-    contentType: pointer.contentType,
-    filename: pointer.filename,
-    sha256: pointer.sha256,
-    expiresAt: pointer.expiresAt,
-    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,
-    kind: "bundle"
-  };
-}
-async function createSiteExtractImageArtifact(args) {
-  const pointer = await createPrivateArtifact({
-    policy: policy(),
-    ownerId: args.ownerId,
-    scopeSegments: [args.jobId, "images"],
-    artifactKey: `${args.imageId}.bin`,
-    createdAt: /* @__PURE__ */ new Date(),
-    filename: args.filename,
-    contentType: args.contentType,
-    content: args.content
-  });
-  return {
-    key: pointer.artifactId,
-    url: pointer.downloadUrl ?? "",
-    bytes: pointer.bytes,
-    contentType: pointer.contentType,
-    filename: pointer.filename,
-    sha256: pointer.sha256,
-    expiresAt: pointer.expiresAt,
-    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,
-    kind: "image",
-    imageId: args.imageId,
-    sourceUrl: args.sourceUrl,
-    sourcePage: args.sourcePage
-  };
-}
-async function createSiteExtractContentChunkArtifact(args) {
-  return createPrivateArtifact({
-    policy: policy(),
-    ownerId: args.ownerId,
-    scopeSegments: [args.jobId, "content"],
-    artifactKey: `${args.chunkKey}.json.gz`,
-    createdAt: args.createdAt,
-    filename: `${args.chunkKey}.json.gz`,
-    contentType: "application/gzip",
-    content: args.content
-  });
-}
-async function renewSiteExtractArtifactDownload(args) {
-  return renewPrivateArtifactDownload({
-    policy: policy(),
-    artifactId: args.artifactId,
-    ownerId: args.ownerId
-  });
-}
-async function readSiteExtractArtifactBuffer(artifactId) {
-  return readPrivateArtifactBuffer({
-    policy: policy(),
-    artifactId,
-    maxBytes: 50 * 1024 * 1024
-  });
-}
-async function readOwnedSiteExtractArtifactBuffer(args) {
-  if (siteExtractArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
-  return readSiteExtractArtifactBuffer(args.artifactId);
-}
-async function readOwnedSiteExtractContentChunk(args) {
-  if (siteExtractArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
-  return readPrivateArtifactBuffer({
-    policy: policy(),
-    artifactId: args.artifactId,
-    maxBytes: 25 * 1024 * 1024
-  });
-}
-async function readOwnedSiteExtractImageArtifact(args) {
-  if (siteExtractArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
-  return readPrivateArtifactBuffer({
-    policy: policy(),
-    artifactId: args.artifactId,
-    maxBytes: 10 * 1024 * 1024
-  });
-}
-async function cleanupExpiredSiteExtractArtifacts(args = {}) {
-  const now = args.now ?? /* @__PURE__ */ new Date();
-  const token = siteExtractArtifactToken();
-  if (!token) return { deleted: 0, store: hostedByEnvironment() ? "none" : "local" };
-  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {
-    return { deleted: 0, store: "private-vercel-blob", skipped: true };
-  }
-  const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS;
-  const { list, del } = await import("@vercel/blob");
-  let cursor;
-  let deleted = 0;
-  for (let page = 0; page < 20; page += 1) {
-    const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
-    const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
-    if (expired.length > 0) {
-      await del(expired.map((blob) => blob.pathname), { token });
-      deleted += expired.length;
-    }
-    if (!result.hasMore || !result.cursor) break;
-    cursor = result.cursor;
-  }
-  return { deleted, store: "private-vercel-blob" };
-}
-
-// src/api/site-extract-content-store.ts
-var MAX_PAGES_PER_CHUNK = 10;
-var MAX_UNCOMPRESSED_CHUNK_BYTES = 20 * 1024 * 1024;
-function sha256(value) {
-  return createHash("sha256").update(value).digest("hex");
-}
-function pageContent(page) {
-  if (!page.acquiredHtml || page.extractionStatus === "failed") return null;
-  const pageId = page.pageId ?? sha256(page.archivedUrl ?? page.url);
-  return {
-    pageId,
-    url: page.archivedUrl ?? page.url,
-    html: page.acquiredHtml,
-    mainHtml: page.mainHtml ?? "",
-    markdown: page.bodyMarkdown,
-    htmlSha256: page.htmlSha256 ?? sha256(page.acquiredHtml),
-    markdownSha256: page.markdownSha256 ?? sha256(page.bodyMarkdown)
-  };
-}
-function serializedChunk(pages) {
-  return Buffer.from(JSON.stringify({ version: "site-extract-content.v1", pages }));
-}
-async function persistSiteExtractPageContent(input) {
-  const contentPages = input.pages.flatMap((page) => {
-    const content = pageContent(page);
-    return content ? [content] : [];
-  });
-  const refs = /* @__PURE__ */ new Map();
-  let chunk = [];
-  let chunkNumber = 0;
-  const flush = async () => {
-    if (!chunk.length) return;
-    const uncompressed = serializedChunk(chunk);
-    if (uncompressed.length > MAX_UNCOMPRESSED_CHUNK_BYTES) {
-      throw new Error(`site export content chunk exceeds ${MAX_UNCOMPRESSED_CHUNK_BYTES} bytes`);
-    }
-    const identity = sha256(chunk.map((page) => page.pageId).join("\0")).slice(0, 16);
-    const pointer = await createSiteExtractContentChunkArtifact({
-      ownerId: input.ownerId,
-      jobId: input.jobId,
-      createdAt: input.createdAt,
-      chunkKey: `content-${String(chunkNumber).padStart(4, "0")}-${identity}`,
-      content: gzipSync(uncompressed, { level: 6 })
-    });
-    for (const page of chunk) {
-      refs.set(page.url, {
-        artifactId: pointer.artifactId,
-        artifactSha256: pointer.sha256,
-        pageId: page.pageId,
-        uncompressedBytes: uncompressed.length
-      });
-    }
-    chunkNumber++;
-    chunk = [];
-  };
-  for (const page of contentPages) {
-    const candidate = [...chunk, page];
-    if (chunk.length > 0 && (candidate.length > MAX_PAGES_PER_CHUNK || serializedChunk(candidate).length > MAX_UNCOMPRESSED_CHUNK_BYTES)) {
-      await flush();
-    }
-    chunk.push(page);
-    if (serializedChunk(chunk).length > MAX_UNCOMPRESSED_CHUNK_BYTES) {
-      throw new Error(`page ${page.pageId} exceeds the durable content chunk limit`);
-    }
-  }
-  await flush();
-  return refs;
-}
-function createSiteExtractContentReader(ownerId) {
-  const cache = /* @__PURE__ */ new Map();
-  return async (ref) => {
-    let chunk = cache.get(ref.artifactId);
-    if (!chunk) {
-      const compressed = await readOwnedSiteExtractContentChunk({ artifactId: ref.artifactId, ownerId });
-      if (!compressed) throw new Error("site export content chunk is missing or unauthorized");
-      if (sha256(compressed) !== ref.artifactSha256) throw new Error("site export content chunk checksum mismatch");
-      const decoded = gunzipSync(compressed);
-      if (decoded.length > MAX_UNCOMPRESSED_CHUNK_BYTES) throw new Error("site export content chunk exceeds its read limit");
-      chunk = JSON.parse(decoded.toString("utf8"));
-      if (chunk.version !== "site-extract-content.v1" || !Array.isArray(chunk.pages)) {
-        throw new Error("site export content chunk has an unsupported contract");
-      }
-      cache.set(ref.artifactId, chunk);
-      while (cache.size > 2) cache.delete(cache.keys().next().value);
-    }
-    const page = chunk.pages.find((candidate) => candidate.pageId === ref.pageId);
-    if (!page) throw new Error("site export page is missing from its content chunk");
-    if (sha256(page.html) !== page.htmlSha256 || sha256(page.markdown) !== page.markdownSha256) {
-      throw new Error("site export page checksum mismatch");
-    }
-    return page;
-  };
-}
-
-// src/api/site-extract-repository.ts
-var XRAY_ENTITLED_EXTRACT_BILLING_CLASS = "xray_entitlement";
-function extractJobLimitInfo(job) {
-  const effectiveMaxPages = Math.max(1, Number(job.options.effectiveMaxPages ?? job.options.maxPages ?? 1));
-  const requestedMaxPages = Math.max(effectiveMaxPages, Number(job.options.requestedMaxPages ?? effectiveMaxPages));
-  const creditLimited = requestedMaxPages > effectiveMaxPages;
-  return {
-    requestedMaxPages,
-    effectiveMaxPages,
-    creditLimited,
-    // If discovery exhausted below the funded cap, the smaller hold did not
-    // actually truncate this crawl. Hitting the cap is conservatively partial.
-    creditTruncated: creditLimited && job.totalUrls >= effectiveMaxPages
-  };
-}
-function terminalExtractJobStatus(progress, creditTruncated = false) {
-  if (progress.successfulUrls === 0) return "failed";
-  if (progress.failedUrls > 0 || progress.remainingUrls > 0 || creditTruncated) return "partial";
-  return "complete";
-}
-function rowToJob(r) {
-  const totalUrls = Number(r.total_urls ?? 0);
-  const legacyProgress = r.attempted_urls == null;
-  const attemptedUrls = Number(legacyProgress ? r.done_urls ?? 0 : r.attempted_urls);
-  const successfulUrls = Number(legacyProgress ? r.done_urls ?? 0 : r.successful_urls ?? 0);
-  const failedUrls = Number(r.failed_urls ?? Math.max(0, attemptedUrls - successfulUrls));
-  return {
-    id: String(r.id),
-    userId: r.user_id != null ? Number(r.user_id) : null,
-    idempotencyKey: r.idempotency_key != null ? String(r.idempotency_key) : null,
-    requestFingerprint: r.request_fingerprint != null ? String(r.request_fingerprint) : null,
-    status: r.status != null ? String(r.status) : "pending",
-    startUrl: String(r.start_url ?? ""),
-    options: r.options ? JSON.parse(String(r.options)) : {},
-    totalUrls,
-    doneUrls: attemptedUrls,
-    attemptedUrls,
-    successfulUrls,
-    failedUrls,
-    remainingUrls: Math.max(0, totalUrls - attemptedUrls),
-    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
-    error: r.error != null ? String(r.error) : null,
-    publicError: parsePublicErrorEnvelope(r.public_error_json),
-    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
-    createdAt: String(r.created_at ?? ""),
-    updatedAt: String(r.updated_at ?? "")
-  };
-}
-async function createExtractJob(jobId, userId, startUrl, options) {
-  const db = getDb();
-  await db.execute({
-    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
-    args: [jobId, userId, startUrl, JSON.stringify(options)]
-  });
-}
-async function getExtractJobByIdempotencyKey(userId, idempotencyKey) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1`,
-    args: [userId, idempotencyKey]
-  });
-  return res.rows[0] ? rowToJob(res.rows[0]) : null;
-}
-async function createOrGetExtractJob(input) {
-  const db = getDb();
-  const inserted = await db.execute({
-    sql: `INSERT OR IGNORE INTO site_extract_jobs
-            (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,
-    args: [
-      input.jobId,
-      input.userId,
-      input.startUrl,
-      JSON.stringify(input.options),
-      input.idempotencyKey,
-      input.requestFingerprint
-    ]
-  });
-  const job = await getExtractJobByIdempotencyKey(input.userId, input.idempotencyKey);
-  if (!job) throw new Error("idempotent extract job was not persisted");
-  return {
-    job,
-    created: inserted.rowsAffected === 1,
-    conflict: job.requestFingerprint !== input.requestFingerprint
-  };
-}
-async function getExtractJob(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
-  return res.rows[0] ? rowToJob(res.rows[0]) : null;
-}
-async function listExtractJobs(userId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50`, args: [userId] });
-  return res.rows.map((r) => rowToJob(r));
-}
-async function listUnsettledExtractJobs(limit = 25) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT * FROM site_extract_jobs
-          WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
-            AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
-          ORDER BY updated_at ASC LIMIT ?`,
-    args: [Math.max(1, Math.min(100, Math.round(limit)))]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function listFundedPendingExtractJobs(limit = 25, staleMinutes = 2) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(1, Math.min(60, Math.round(staleMinutes)));
-  const res = await db.execute({
-    sql: `SELECT j.* FROM site_extract_jobs j
-          LEFT JOIN billing_debits d
-            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
-           AND d.user_id = j.user_id
-           AND d.status = 'applied'
-          WHERE j.status = 'pending'
-            AND j.billed_mc IS NULL
-            AND (
-              d.idempotency_key IS NOT NULL
-              OR (
-                json_extract(j.options, '$.billingClass') = ?
-                AND json_extract(j.options, '$.heldMc') = 0
-                AND json_extract(j.options, '$.debitKey') IS NULL
-                AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
-                AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
-              )
-            )
-            AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
-            AND j.updated_at <= datetime('now', ?)
-          ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [
-      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
-      XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
-      `-${safeStaleMinutes} minutes`,
-      Math.max(1, Math.min(100, Math.round(limit)))
-    ]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function markExtractJobDispatchAttempt(jobId) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET dispatch_attempts = dispatch_attempts + 1,
-              updated_at = datetime('now'),
-              next_dispatch_at = datetime('now', '+5 minutes'),
-              dispatch_error = NULL,
-              status = CASE
-                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
-                ELSE status
-              END,
-              error = CASE
-                WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
-                  THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'
-                ELSE error
-              END
-          WHERE id = ? AND status = 'pending'`,
-    args: [jobId]
-  });
-  return getExtractJob(jobId);
-}
-async function recordExtractJobDispatchFailure(jobId, error) {
-  const db = getDb();
-  const safeError = sanitizeVendorName(error).slice(0, 1e3);
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET
-            dispatch_attempts = dispatch_attempts + 1,
-            dispatch_error = ?,
-            next_dispatch_at = datetime('now', '+5 minutes'),
-            updated_at = datetime('now'),
-            status = CASE
-              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
-              ELSE status
-            END,
-            error = CASE
-              WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
-                THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'
-              ELSE error
-            END
-          WHERE id = ? AND status = 'pending'`,
-    args: [safeError, jobId]
-  });
-  return getExtractJob(jobId);
-}
-async function recordExtractSettlementFailure(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET
-            settlement_attempts = settlement_attempts + 1,
-            settlement_error = ?,
-            next_settlement_at = datetime('now', '+15 minutes'),
-            updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 1e3), jobId]
-  });
-}
-async function listStaleRunningExtractJobs(limit = 25, staleMinutes = 60) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)));
-  const res = await db.execute({
-    sql: `SELECT j.* FROM site_extract_jobs j
-          LEFT JOIN billing_debits d
-            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
-           AND d.user_id = j.user_id
-           AND d.status = 'applied'
-          WHERE j.status = 'running'
-            AND j.billed_mc IS NULL
-            AND (
-              json_extract(j.options, '$.debitKey') IS NULL
-              OR d.idempotency_key IS NOT NULL
-            )
-            AND j.updated_at <= datetime('now', ?)
-          ORDER BY j.updated_at ASC LIMIT ?`,
-    args: [`-${safeStaleMinutes} minutes`, Math.max(1, Math.min(100, Math.round(limit)))]
-  });
-  return res.rows.map((row) => rowToJob(row));
-}
-async function failStaleRunningExtractJob(jobId, staleMinutes = 60) {
-  const db = getDb();
-  const safeStaleMinutes = Math.max(15, Math.min(24 * 60, Math.round(staleMinutes)));
-  const result = await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET status = 'failed',
-              error = 'Background crawl stopped without a heartbeat; completed pages were preserved and settlement is pending.',
-              public_error_json = ?,
-              updated_at = datetime('now')
-          WHERE id = ?
-            AND status = 'running'
-            AND billed_mc IS NULL
-            AND updated_at <= datetime('now', ?)`,
-    args: [
-      serializePublicErrorEnvelope({
-        error_code: "extraction_failed",
-        error_type: "extraction",
-        message: "The background crawl stopped without a heartbeat. Completed pages were preserved; retry after settlement completes.",
-        retryable: true,
-        charge_status: "refund_pending"
-      }),
-      jobId,
-      `-${safeStaleMinutes} minutes`
-    ]
-  });
-  return result.rowsAffected === 1;
-}
-async function claimFailedExtractJobForRefinalize(jobId) {
-  const db = getDb();
-  const claimed = await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET status = 'running', updated_at = datetime('now')
-          WHERE id = ? AND status = 'failed'`,
-    args: [jobId]
-  });
-  if (claimed.rowsAffected !== 1) return null;
-  return getExtractJob(jobId);
-}
-async function abandonExtractSettlement(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 1e3), jobId]
-  });
-}
-async function setExtractJobTotal(jobId, totalUrls) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-    args: [totalUrls, jobId]
-  });
-}
-async function saveExtractPages(jobId, pages) {
-  if (pages.length === 0) return;
-  const db = getDb();
-  const job = await getExtractJob(jobId);
-  const contentRefs = job?.userId != null ? await persistSiteExtractPageContent({
-    ownerId: String(job.userId),
-    jobId,
-    createdAt: job.createdAt,
-    pages
-  }) : /* @__PURE__ */ new Map();
-  await db.batch([
-    ...pages.map((p) => {
-      const {
-        discoveryLinks: _ephemeralDiscoveryLinks,
-        acquiredHtml: _durableHtml,
-        mainHtml: _durableMainHtml,
-        ...storedPage
-      } = p;
-      storedPage.contentRef = contentRefs.get(p.archivedUrl ?? p.url) ?? p.contentRef;
-      return {
-        sql: `INSERT INTO site_extract_pages (job_id, url, page)
-            SELECT ?, ?, ?
-            WHERE EXISTS (
-              SELECT 1 FROM site_extract_jobs
-              WHERE id = ? AND status IN ('pending', 'running')
-            )
-            ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
-            WHERE NOT CASE
-                    WHEN json_valid(site_extract_pages.page) = 0 THEN 0
-                    WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(site_extract_pages.page, '$.status') >= 200
-                      AND json_extract(site_extract_pages.page, '$.status') < 300
-                      AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END
-               OR CASE
-                    WHEN json_valid(excluded.page) = 0 THEN 0
-                    WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(excluded.page, '$.status') >= 200
-                      AND json_extract(excluded.page, '$.status') < 300
-                      AND json_extract(excluded.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END`,
-        args: [jobId, p.archivedUrl ?? p.url, JSON.stringify(storedPage), jobId]
-      };
-    }),
-    {
-      sql: `UPDATE site_extract_jobs SET
-            done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            successful_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0
-              END
-            ),
-            failed_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND NOT CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE COALESCE(
-                  json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0,
-                  0
-                )
-              END
-            ),
-            updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-      args: [jobId, jobId, jobId, jobId, jobId]
-    }
-  ], "write");
-}
-async function listExtractPages(jobId) {
-  const db = getDb();
-  const result = await db.execute({
-    sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid`,
-    args: [jobId]
-  });
-  return result.rows.map((row) => JSON.parse(String(row.page)));
-}
-async function getExtractedPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
-  return res.rows.map((r) => JSON.parse(String(r.page)));
-}
-async function getExtractedImageLinks(jobId, limit = 181) {
-  const db = getDb();
-  const safeLimit = Math.max(1, Math.min(5001, Math.round(limit)));
-  const res = await db.execute({
-    sql: `SELECT DISTINCT CAST(images.value AS TEXT) AS url
-          FROM site_extract_pages p,
-               json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
-          WHERE p.job_id = ?
-            AND images.type = 'text'
-            AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
-          LIMIT ?`,
-    args: [jobId, safeLimit]
-  });
-  return res.rows.map((row) => String(row.url));
-}
-async function countSuccessfulPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT COUNT(*) AS n FROM site_extract_pages
-            WHERE job_id = ? AND CASE
-              WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                THEN json_extract(page, '$.extractionStatus') = 'successful'
-              ELSE COALESCE(
-                json_extract(page, '$.status') >= 200
-                AND json_extract(page, '$.status') < 300
-                AND json_extract(page, '$.wordCount') > 0,
-                0
-              )
-            END`,
-    args: [jobId]
-  });
-  return Number(res.rows[0]?.n ?? 0);
-}
-async function getExtractedUrls(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT url FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
-  return new Set(res.rows.map((r) => String(r.url)));
-}
-async function finishExtractJob(jobId, artifacts, status, error = null, allowFailedRecovery = false, publicError = null) {
-  const db = getDb();
-  const result = await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = ?, artifacts = ?, error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,
-    args: [
-      status,
-      JSON.stringify(artifacts ?? []),
-      error ? sanitizeVendorName(error).slice(0, 2e3) : null,
-      serializePublicErrorEnvelope(publicError),
-      jobId,
-      allowFailedRecovery ? 1 : 0
-    ]
-  });
-  return result.rowsAffected === 1;
-}
-async function completeExtractJob(jobId, artifacts) {
-  await finishExtractJob(jobId, artifacts, "complete");
-}
-async function failExtractJob(jobId, error, publicError) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'failed', error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running')`,
-    args: [sanitizeVendorName(error).slice(0, 2e3), serializePublicErrorEnvelope(publicError), jobId]
-  });
-}
-async function failUnfundedExtractJob(jobId, error, publicError) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET status = 'failed', billed_mc = 0, error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,
-    args: [sanitizeVendorName(error).slice(0, 2e3), serializePublicErrorEnvelope(publicError), jobId]
-  });
-}
-async function setExtractJobPublicError(jobId, publicError) {
-  const job = await getExtractJob(jobId);
-  const normalized = job?.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS && publicError ? { ...publicError, charge_status: "not_charged" } : publicError;
-  await getDb().execute({
-    sql: `UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,
-    args: [serializePublicErrorEnvelope(normalized), jobId]
-  });
-}
-async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
-  const db = getDb();
-  const job = await getExtractJob(jobId);
-  if (!job || job.billedMc != null) return;
-  if (job.options.billingClass === XRAY_ENTITLED_EXTRACT_BILLING_CLASS) {
-    if (!await finalizeXRayEntitledExtractJobCharge(jobId)) {
-      throw new Error("xray_extract_zero_charge_finalization_rejected");
-    }
-    return;
-  }
-  const debitKey = typeof job.options.debitKey === "string" ? job.options.debitKey : null;
-  if (debitKey) {
-    const settlement = await settleDebitMcIdempotent(
-      userId,
-      debitKey,
-      Math.max(0, netChargeMc),
-      LedgerOperation.EXTRACT_SITE_REFUND,
-      reference,
-      "extract_site"
-    );
-    await db.execute({
-      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`,
-      args: [settlement.final_amount_mc, jobId]
-    });
-    return;
-  }
-  const safeRefundMc = Math.max(0, refundMc);
-  const safeNetChargeMc = Math.max(0, netChargeMc);
-  await db.batch([
-    {
-      sql: `UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL`,
-      args: [jobId]
-    },
-    {
-      sql: `INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
-            SELECT ?, ?, ?, ?, datetime(j.created_at, ?)
-            FROM site_extract_jobs j
-            WHERE j.id = ?
-              AND j.billed_mc = -1
-              AND ? > 0
-              AND changes() = 1
-              AND datetime(j.created_at, ?) > datetime('now')`,
-      args: [
-        userId,
-        safeRefundMc,
-        safeRefundMc,
-        LedgerOperation.EXTRACT_SITE_REFUND,
-        CREDIT_LOT_TTL,
-        jobId,
-        safeRefundMc,
-        CREDIT_LOT_TTL
-      ]
-    },
-    {
-      sql: `UPDATE users SET balance_mc = balance_mc + ?
-            WHERE id = ? AND ? > 0 AND changes() = 1`,
-      args: [safeRefundMc, userId, safeRefundMc]
-    },
-    {
-      sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
-            SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,
-      args: [userId, safeRefundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference, safeRefundMc]
-    },
-    {
-      sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')
-            WHERE id = ? AND billed_mc = -1`,
-      args: [safeNetChargeMc, jobId]
-    }
-  ], "write");
-}
-async function finalizeXRayEntitledExtractJobCharge(jobId) {
-  const db = getDb();
-  const result = await db.execute({
-    sql: `UPDATE site_extract_jobs
-          SET billed_mc = 0, updated_at = datetime('now')
-          WHERE id = ?
-            AND billed_mc IS NULL
-            AND json_extract(options, '$.billingClass') = ?
-            AND json_extract(options, '$.heldMc') = 0
-            AND json_extract(options, '$.debitKey') IS NULL
-            AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
-            AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
-            AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
-            AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,
-    args: [jobId, XRAY_ENTITLED_EXTRACT_BILLING_CLASS, XRAY_ENTITLED_EXTRACT_BILLING_CLASS]
-  });
-  return result.rowsAffected === 1;
-}
-
-export {
-  SITE_EXTRACT_ARTIFACT_PREFIX,
-  createSiteExtractBundleArtifactStream,
-  createSiteExtractImageArtifact,
-  renewSiteExtractArtifactDownload,
-  readOwnedSiteExtractArtifactBuffer,
-  readOwnedSiteExtractImageArtifact,
-  cleanupExpiredSiteExtractArtifacts,
-  createSiteExtractContentReader,
-  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
-  extractJobLimitInfo,
-  terminalExtractJobStatus,
-  createExtractJob,
-  getExtractJobByIdempotencyKey,
-  createOrGetExtractJob,
-  getExtractJob,
-  listExtractJobs,
-  listUnsettledExtractJobs,
-  listFundedPendingExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  listStaleRunningExtractJobs,
-  failStaleRunningExtractJob,
-  claimFailedExtractJobForRefinalize,
-  abandonExtractSettlement,
-  setExtractJobTotal,
-  saveExtractPages,
-  listExtractPages,
-  getExtractedPages,
-  getExtractedImageLinks,
-  countSuccessfulPages,
-  getExtractedUrls,
-  finishExtractJob,
-  completeExtractJob,
-  failExtractJob,
-  failUnfundedExtractJob,
-  setExtractJobPublicError,
-  settleExtractJob,
-  finalizeXRayEntitledExtractJobCharge
-};