mcp-scraper 0.88.2 → 0.89.0

package/dist/site-extract-repository-VYFZASPU.js

@@ -1,69 +0,0 @@
-import {
-  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
-  abandonExtractSettlement,
-  claimFailedExtractJobForRefinalize,
-  completeExtractJob,
-  countSuccessfulPages,
-  createExtractJob,
-  createOrGetExtractJob,
-  extractJobLimitInfo,
-  failExtractJob,
-  failStaleRunningExtractJob,
-  failUnfundedExtractJob,
-  finalizeXRayEntitledExtractJobCharge,
-  finishExtractJob,
-  getExtractJob,
-  getExtractJobByIdempotencyKey,
-  getExtractedImageLinks,
-  getExtractedPages,
-  getExtractedUrls,
-  listExtractJobs,
-  listExtractPages,
-  listFundedPendingExtractJobs,
-  listStaleRunningExtractJobs,
-  listUnsettledExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  saveExtractPages,
-  setExtractJobPublicError,
-  setExtractJobTotal,
-  settleExtractJob,
-  terminalExtractJobStatus
-} from "./chunk-IHXAXYIS.js";
-import "./chunk-DNM65UCK.js";
-import "./chunk-4QMUF6XM.js";
-import "./chunk-YXNDOQXN.js";
-export {
-  XRAY_ENTITLED_EXTRACT_BILLING_CLASS,
-  abandonExtractSettlement,
-  claimFailedExtractJobForRefinalize,
-  completeExtractJob,
-  countSuccessfulPages,
-  createExtractJob,
-  createOrGetExtractJob,
-  extractJobLimitInfo,
-  failExtractJob,
-  failStaleRunningExtractJob,
-  failUnfundedExtractJob,
-  finalizeXRayEntitledExtractJobCharge,
-  finishExtractJob,
-  getExtractJob,
-  getExtractJobByIdempotencyKey,
-  getExtractedImageLinks,
-  getExtractedPages,
-  getExtractedUrls,
-  listExtractJobs,
-  listExtractPages,
-  listFundedPendingExtractJobs,
-  listStaleRunningExtractJobs,
-  listUnsettledExtractJobs,
-  markExtractJobDispatchAttempt,
-  recordExtractJobDispatchFailure,
-  recordExtractSettlementFailure,
-  saveExtractPages,
-  setExtractJobPublicError,
-  setExtractJobTotal,
-  settleExtractJob,
-  terminalExtractJobStatus
-};

package/dist/worker-LDCAULWL.js

@@ -1,146 +0,0 @@
-import {
-  classifyHarvestProblem,
-  createHarvestAttemptRecorder,
-  deliverWebhook,
-  harvestProblemEnvelope,
-  harvestProblemResponse,
-  serializeHarvestProblem
-} from "./chunk-NVUKO5NN.js";
-import "./chunk-OZJMVCDK.js";
-import {
-  harvest
-} from "./chunk-S24LFPL7.js";
-import {
-  sanitizeAttempts,
-  sanitizeHarvestResult
-} from "./chunk-6HAV7LCE.js";
-import {
-  MC_COSTS,
-  serpActualCostMc
-} from "./chunk-4QMUF6XM.js";
-import {
-  browserServiceApiKey
-} from "./chunk-OM7HVEJ3.js";
-import {
-  runWithCostContext
-} from "./chunk-OPQIGAFB.js";
-import "./chunk-GXBZXWXB.js";
-import {
-  claimPendingJob,
-  completeJob,
-  creditMc,
-  debitMc,
-  failJob,
-  listHarvestAttempts
-} from "./chunk-YXNDOQXN.js";
-
-// src/api/worker.ts
-var MAX_CONCURRENT = 2;
-var running = 0;
-function countPaaQuestions(result) {
-  if (!result || typeof result !== "object") return 0;
-  const value = result;
-  if (typeof value.totalQuestions === "number") return value.totalQuestions;
-  return Array.isArray(value.flat) ? value.flat.length : 0;
-}
-function paaCostForQuestionCount(questionCount) {
-  return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa;
-}
-async function processJob(job) {
-  running++;
-  try {
-    const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
-    const headlessSentOut = { value: null };
-    const result = await runWithCostContext(
-      { op: opts.serpOnly ? "serp" : "paa", userId: Number(job.user_id), headlessSentOut },
-      () => harvest({
-        ...opts,
-        kernelApiKey: browserServiceApiKey(),
-        headless: true,
-        format: "json",
-        outputDir: "/tmp/paa-output-api",
-        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id)
-      })
-    );
-    await completeJob(job.id, result);
-    const attempts = await listHarvestAttempts(job.id, job.user_id);
-    if (!opts.serpOnly && typeof opts.billingHoldMc === "number") {
-      const actualCost = paaCostForQuestionCount(countPaaQuestions(result));
-      const diff = opts.billingHoldMc - actualCost;
-      if (diff > 0) await creditMc(job.user_id, diff, "paa_refund", "overestimate refund");
-      else if (diff < 0) await debitMc(job.user_id, -diff, "paa", opts.query ?? job.query);
-    } else if (opts.serpOnly && typeof opts.billingHoldMc === "number") {
-      const actualCost = serpActualCostMc(headlessSentOut.value);
-      const diff = opts.billingHoldMc - actualCost;
-      if (diff > 0) await creditMc(job.user_id, diff, "serp_refund", "headless-mode pricing settle");
-      else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
-    }
-    if (job.callback_url) {
-      await deliverWebhook(job.callback_url, {
-        job_id: job.id,
-        status: "done",
-        result: sanitizeHarvestResult(result),
-        attempts: sanitizeAttempts(attempts)
-      });
-    }
-  } catch (err) {
-    console.error("[harvest/worker] failed", {
-      jobId: job.id,
-      error: err instanceof Error ? err.message : String(err)
-    });
-    const problem = classifyHarvestProblem(err);
-    const failedOptions = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
-    const hasHold = typeof failedOptions.billingHoldMc === "number" && failedOptions.billingHoldMc > 0;
-    await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, {
-      chargeStatus: hasHold ? "refund_pending" : "not_charged"
-    }));
-    const attempts = await listHarvestAttempts(job.id, job.user_id);
-    try {
-      if (hasHold) {
-        await creditMc(job.user_id, failedOptions.billingHoldMc, "refund", "failed call");
-        await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refunded" }));
-      }
-    } catch {
-    }
-    if (job.callback_url) {
-      await deliverWebhook(job.callback_url, {
-        job_id: job.id,
-        status: "failed",
-        ...harvestProblemResponse(problem),
-        attempts: sanitizeAttempts(attempts)
-      });
-    }
-  } finally {
-    running--;
-  }
-}
-async function tickOnce() {
-  const job = await claimPendingJob();
-  if (!job) return { claimed: false };
-  const startedAt = Date.now();
-  await processJob(job);
-  return { claimed: true, jobId: job.id, completed: true, durationMs: Date.now() - startedAt };
-}
-async function drainQueue(budget) {
-  const results = [];
-  for (let i = 0; i < budget.maxJobs; i++) {
-    if (Date.now() >= budget.deadlineMs) break;
-    const r = await tickOnce();
-    results.push(r);
-    if (!r.claimed) break;
-  }
-  return results;
-}
-function startWorker() {
-  setInterval(async () => {
-    if (running >= MAX_CONCURRENT) return;
-    const job = await claimPendingJob();
-    if (job) void processJob(job);
-  }, 2e3);
-  console.log(`[worker] started \u2014 polling every 2s, max ${MAX_CONCURRENT} concurrent`);
-}
-export {
-  drainQueue,
-  startWorker,
-  tickOnce
-};