mcp-scraper 0.79.2 → 0.79.4

package/dist/{chunk-JECPQBLZ.js → chunk-RUBI3OXR.js}

@@ -19,7 +19,7 @@ import {
 } from "./chunk-T3MZISOF.js";
 import {
   sanitizeOutboundDiagnostics
-} from "./chunk-B6L73AO4.js";
+} from "./chunk-6HAV7LCE.js";
 import {
   MC_PER_CREDIT,
   PAA_BASE_CREDITS,
@@ -34,7 +34,7 @@ import {
 import {
   recordVendorUsage,
   vendorCostUsd
-} from "./chunk-NGLZ6UJR.js";
+} from "./chunk-5UAFBKJR.js";
 import {
   buildSerpEmailQuery,
   classifyEmailScope,
@@ -45,14 +45,14 @@ import {
 } from "./chunk-VXLU74YZ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2RTELL6M.js";
+} from "./chunk-N63ZFFU2.js";
 import {
   PUBLIC_ERROR_CODES,
   buildPublicErrorEnvelope,
   normalizePublicErrorCode,
   publicErrorMessage,
   sanitizeVendorName
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/harvest-timeout.ts
 var SCRAPE_FUNCTION_MAX_MS = 8e5;
@@ -1139,9 +1139,64 @@ function publicPaaLifecycle(value) {
     knownQuestions: finiteNonNegative(row.knownQuestions) ?? questionsFound,
     processedQuestions: finiteNonNegative(row.processedQuestions) ?? 0,
     failedInteractions: finiteNonNegative(row.failedInteractions) ?? 0,
-    noGrowthConfirmations: finiteNonNegative(row.noGrowthConfirmations) ?? 0
+    noGrowthConfirmations: finiteNonNegative(row.noGrowthConfirmations) ?? 0,
+    visibleControls: finiteNonNegative(row.visibleControls) ?? 0,
+    eligibleControls: finiteNonNegative(row.eligibleControls) ?? 0,
+    dispatchedInteractions: finiteNonNegative(row.dispatchedInteractions) ?? 0,
+    confirmedInteractions: finiteNonNegative(row.confirmedInteractions) ?? 0,
+    unconfirmedInteractions: finiteNonNegative(row.unconfirmedInteractions) ?? 0
   };
 }
+function publicPaaInteraction(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const row = value;
+  const id = nullableBoundedString(row.id, 200);
+  const question = nullableBoundedString(row.question, 2e3);
+  if (!id || !question) return null;
+  const outcome = row.outcome === "confirmed" || row.outcome === "unconfirmed" || row.outcome === "dispatch_failed" ? row.outcome : "unconfirmed";
+  const allowedSignals = /* @__PURE__ */ new Set(["expanded", "answer", "source", "question_growth"]);
+  return {
+    id,
+    questionId: nullableBoundedString(row.questionId, 100) ?? "",
+    question,
+    batchNumber: finiteNonNegative(row.batchNumber) ?? 0,
+    controlAttempt: finiteNonNegative(row.controlAttempt) ?? 0,
+    observedAt: nullableBoundedString(row.observedAt, 100),
+    dispatched: row.dispatched === true,
+    outcome,
+    confirmationSignals: Array.isArray(row.confirmationSignals) ? row.confirmationSignals.filter((signal) => typeof signal === "string" && allowedSignals.has(signal)) : [],
+    confirmationCheckpointMs: finiteNonNegative(row.confirmationCheckpointMs),
+    confirmedAtMs: finiteNonNegative(row.confirmedAtMs),
+    dispatchDurationMs: finiteNonNegative(row.dispatchDurationMs) ?? 0,
+    totalDurationMs: finiteNonNegative(row.totalDurationMs) ?? 0,
+    beforeQuestionCount: finiteNonNegative(row.beforeQuestionCount) ?? 0,
+    afterQuestionCount: finiteNonNegative(row.afterQuestionCount) ?? 0,
+    newQuestionCount: finiteNonNegative(row.newQuestionCount) ?? 0,
+    expandedBefore: row.expandedBefore === true,
+    expandedAfter: row.expandedAfter === true,
+    answerObserved: row.answerObserved === true,
+    sourceCount: finiteNonNegative(row.sourceCount) ?? 0,
+    errorCode: nullableBoundedString(row.errorCode, 100)
+  };
+}
+function publicPaaInteractionSummary(value) {
+  const row = structuredRecord(value);
+  return Object.fromEntries([
+    "observations",
+    "dispatched",
+    "confirmed",
+    "unconfirmed",
+    "dispatchFailures",
+    "confirmedWithoutDispatchAck",
+    "expandedSignals",
+    "answerSignals",
+    "sourceSignals",
+    "questionGrowthBatches",
+    "checkpoint700ms",
+    "checkpoint1000ms",
+    "checkpoint1400ms"
+  ].map((key) => [key, finiteNonNegative(row[key]) ?? 0]));
+}
 function publicPaaSource(value) {
   if (!value || typeof value !== "object") return null;
   const source = value;
@@ -1318,6 +1373,11 @@ function durablePaaProgress(result, options) {
     row.source_cite ?? row.sourceCite ?? row.source_site ?? row.sourceSite ?? row.source_url ?? row.sourceUrl
   ) !== null).length;
   const lifecycle = publicPaaLifecycle(diagnostics.paaLifecycle ?? structuredRecord(result.progress).lifecycle);
+  const rawProgress = structuredRecord(result.progress);
+  const interactions = Array.isArray(rawProgress.interactions) ? rawProgress.interactions.flatMap((value) => {
+    const interaction = publicPaaInteraction(value);
+    return interaction ? [interaction] : [];
+  }) : [];
   if (lifecycle) {
     lifecycle.automaticRetries = Math.max(
       finiteNonNegative(lifecycle.automaticRetries) ?? 0,
@@ -1331,7 +1391,9 @@ function durablePaaProgress(result, options) {
     sourcedQuestions: sourced,
     missingAnswers: finiteNonNegative(completeness.paaWithoutAnswer ?? completeness.missingAnswers) ?? Math.max(0, captured - answered),
     missingSources: finiteNonNegative(completeness.paaWithoutSource ?? completeness.missingSources) ?? Math.max(0, captured - sourced),
-    lifecycle
+    lifecycle,
+    interactionSummary: publicPaaInteractionSummary(rawProgress.interactionSummary),
+    recentInteractions: interactions.slice(-20)
   };
 }
 function durablePaaResult(result) {
@@ -1356,11 +1418,14 @@ function durablePaaAttempts(value) {
     if (!entry || typeof entry !== "object" || Array.isArray(entry)) return [];
     const row = entry;
     return [{
+      attemptId: nullableBoundedString(row.id ?? row.attemptId, 200) ?? "",
       attemptNumber: finiteNonNegative(row.attempt_number ?? row.attemptNumber) ?? index + 1,
+      status: nullableBoundedString(row.status, 100),
       outcome: nullableBoundedString(row.outcome),
       questionCount: finiteNonNegative(row.question_count ?? row.questionCount),
       provider: nullableBoundedString(row.browser_provider ?? row.provider, 100),
       providerSessionId: nullableBoundedString(row.provider_session_id ?? row.providerSessionId, 200),
+      providerSessionObservedAt: nullableBoundedString(row.provider_session_observed_at ?? row.providerSessionObservedAt, 100),
       providerLookupStatus: nullableBoundedString(row.provider_lookup_status ?? row.providerLookupStatus, 100),
       providerStatus: nullableBoundedString(row.provider_status ?? row.providerStatus, 100),
       providerErrorCode: nullableBoundedString(row.provider_error_code ?? row.providerErrorCode, 100),
@@ -1371,7 +1436,10 @@ function durablePaaAttempts(value) {
       providerBandwidthBytes: finiteNonNegative(row.provider_bandwidth_bytes ?? row.providerBandwidthBytes),
       providerCaptcha: typeof (row.provider_captcha ?? row.providerCaptcha) === "boolean" ? row.provider_captcha ?? row.providerCaptcha : null,
       providerCheckedAt: nullableBoundedString(row.provider_checked_at ?? row.providerCheckedAt, 100),
-      willRetry: typeof (row.will_retry ?? row.willRetry) === "boolean" ? row.will_retry ?? row.willRetry : null
+      willRetry: typeof (row.will_retry ?? row.willRetry) === "boolean" ? row.will_retry ?? row.willRetry : null,
+      durationMs: finiteNonNegative(row.duration_ms ?? row.durationMs),
+      startedAt: nullableBoundedString(row.created_at ?? row.startedAt, 100),
+      completedAt: nullableBoundedString(row.completed_at ?? row.completedAt, 100)
     }];
   });
 }
@@ -1401,6 +1469,8 @@ function formatHarvestPaaStart(raw) {
   const state = durablePaaState(parsed.data.status ?? parsed.data.state);
   const structuredContent = {
     jobId,
+    operationId: jobId,
+    taskId: jobId,
     status: durablePaaStatus(state),
     replayed: parsed.data.replayed === true,
     statusTool: "harvest_paa_status"
@@ -1429,6 +1499,8 @@ function formatHarvestPaaStatus(raw) {
   const billingState = billingSettled ? billedMc === 0 ? "refunded" : "settled" : state === "failed" || state === "cancelled" ? "refund_pending" : holdMc !== null ? "held" : "unknown";
   const structuredContent = {
     jobId,
+    operationId: jobId,
+    taskId: jobId,
     status: durablePaaStatus(state),
     replayed: parsed.data.replayed === true,
     progress,
@@ -8546,8 +8618,51 @@ var PaaLifecycleOutput = z6.object({
   knownQuestions: z6.number().int().min(0),
   processedQuestions: z6.number().int().min(0),
   failedInteractions: z6.number().int().min(0),
-  noGrowthConfirmations: z6.number().int().min(0)
+  noGrowthConfirmations: z6.number().int().min(0),
+  visibleControls: z6.number().int().min(0),
+  eligibleControls: z6.number().int().min(0),
+  dispatchedInteractions: z6.number().int().min(0),
+  confirmedInteractions: z6.number().int().min(0),
+  unconfirmedInteractions: z6.number().int().min(0)
 }).nullable();
+var PaaInteractionSummaryOutput = z6.object({
+  observations: z6.number().int().min(0),
+  dispatched: z6.number().int().min(0),
+  confirmed: z6.number().int().min(0),
+  unconfirmed: z6.number().int().min(0),
+  dispatchFailures: z6.number().int().min(0),
+  confirmedWithoutDispatchAck: z6.number().int().min(0),
+  expandedSignals: z6.number().int().min(0),
+  answerSignals: z6.number().int().min(0),
+  sourceSignals: z6.number().int().min(0),
+  questionGrowthBatches: z6.number().int().min(0),
+  checkpoint700ms: z6.number().int().min(0),
+  checkpoint1000ms: z6.number().int().min(0),
+  checkpoint1400ms: z6.number().int().min(0)
+});
+var PaaInteractionOutput = z6.object({
+  id: z6.string(),
+  questionId: z6.string(),
+  question: z6.string(),
+  batchNumber: z6.number().int().min(0),
+  controlAttempt: z6.number().int().min(0),
+  observedAt: NullableString,
+  dispatched: z6.boolean(),
+  outcome: z6.enum(["confirmed", "unconfirmed", "dispatch_failed"]),
+  confirmationSignals: z6.array(z6.enum(["expanded", "answer", "source", "question_growth"])),
+  confirmationCheckpointMs: z6.union([z6.literal(700), z6.literal(1e3), z6.literal(1400)]).nullable(),
+  confirmedAtMs: z6.number().int().min(0).nullable(),
+  dispatchDurationMs: z6.number().int().min(0),
+  totalDurationMs: z6.number().int().min(0),
+  beforeQuestionCount: z6.number().int().min(0),
+  afterQuestionCount: z6.number().int().min(0),
+  newQuestionCount: z6.number().int().min(0),
+  expandedBefore: z6.boolean(),
+  expandedAfter: z6.boolean(),
+  answerObserved: z6.boolean(),
+  sourceCount: z6.number().int().min(0),
+  errorCode: z6.enum(["click_failed", "click_ack_timeout", "control_missing", "confirmation_timeout"]).nullable()
+});
 var HarvestPaaOutputSchema = {
   query: z6.string(),
   location: NullableString,
@@ -8571,14 +8686,19 @@ var HarvestPaaDurableProgressOutput = z6.object({
   sourcedQuestions: z6.number().int().min(0),
   missingAnswers: z6.number().int().min(0),
   missingSources: z6.number().int().min(0),
-  lifecycle: PaaLifecycleOutput
+  lifecycle: PaaLifecycleOutput,
+  interactionSummary: PaaInteractionSummaryOutput,
+  recentInteractions: z6.array(PaaInteractionOutput).max(20)
 });
 var HarvestPaaDurableAttemptOutput = z6.object({
+  attemptId: z6.string(),
   attemptNumber: z6.number().int().min(1),
+  status: NullableString,
   outcome: NullableString,
   questionCount: z6.number().int().min(0).nullable(),
   provider: NullableString,
   providerSessionId: NullableString,
+  providerSessionObservedAt: NullableString,
   providerLookupStatus: NullableString,
   providerStatus: NullableString,
   providerErrorCode: NullableString,
@@ -8589,7 +8709,10 @@ var HarvestPaaDurableAttemptOutput = z6.object({
   providerBandwidthBytes: z6.number().int().min(0).nullable(),
   providerCaptcha: z6.boolean().nullable(),
   providerCheckedAt: NullableString,
-  willRetry: z6.boolean().nullable()
+  willRetry: z6.boolean().nullable(),
+  durationMs: z6.number().int().min(0).nullable(),
+  startedAt: NullableString,
+  completedAt: NullableString
 });
 var HarvestPaaDurableBillingOutput = z6.object({
   state: z6.enum(["held", "settled", "refund_pending", "refunded", "unknown"]),
@@ -8647,6 +8770,8 @@ var HarvestPaaDurableStateOutput = z6.discriminatedUnion("state", [
 ]);
 var HarvestPaaStartOutputSchema = {
   jobId: z6.string(),
+  operationId: z6.string(),
+  taskId: z6.string(),
   status: z6.object({
     state: z6.enum(["pending", "running"]),
     terminal: z6.literal(false),
@@ -8658,6 +8783,8 @@ var HarvestPaaStartOutputSchema = {
 };
 var HarvestPaaStatusOutputSchema = {
   jobId: z6.string(),
+  operationId: z6.string(),
+  taskId: z6.string(),
   status: HarvestPaaDurableStateOutput,
   replayed: z6.boolean(),
   progress: HarvestPaaDurableProgressOutput,
@@ -14436,7 +14563,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   });
   server.registerTool("harvest_paa_status", {
     title: "Check Durable Google PAA Harvest",
-    description: "Poll an owner-scoped harvest_paa_start job. Returns state, saved progress, automatic-recovery count, target status, discovery status, material completeness, attempts, terminal rows, and billing. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
+    description: "Poll an owner-scoped harvest_paa_start job. Returns the job/operation/task correlation ID, saved progress, automatic-recovery count, target and discovery status, material completeness, recent per-control interaction outcomes with 0.7/1.0/1.4-second confirmation telemetry, provider-session-correlated attempts, terminal rows, and billing. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
     inputSchema: HarvestPaaStatusInputSchema,
     outputSchema: recordOutputSchema("harvest_paa_status", HarvestPaaStatusOutputSchema),
     annotations: { ...liveWebToolAnnotations("Check Durable Google PAA Harvest"), idempotentHint: true, openWorldHint: false }

package/dist/{chunk-SL7YVA4S.js → chunk-YBR32LOD.js}

@@ -15,7 +15,7 @@ import {
   sanitizeVendorName,
   serializePublicErrorEnvelope,
   settleDebitMcIdempotent
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/site-extract-content-store.ts
 import { createHash } from "crypto";

package/dist/{chunk-OU4FV2RP.js → chunk-YXNDOQXN.js}

@@ -488,6 +488,7 @@ var CORE_SCHEMA_COLUMNS = {
     "debug_json",
     "browser_provider",
     "provider_session_id",
+    "provider_session_observed_at",
     "provider_lookup_status",
     "provider_status",
     "provider_error_code",
@@ -711,6 +712,7 @@ async function migrate(options = {}) {
       debug_json               TEXT,
       browser_provider         TEXT,
       provider_session_id      TEXT,
+      provider_session_observed_at TEXT,
       provider_lookup_status   TEXT,
       provider_status          TEXT,
       provider_error_code      TEXT,
@@ -737,6 +739,7 @@ async function migrate(options = {}) {
   for (const column of [
     "browser_provider TEXT",
     "provider_session_id TEXT",
+    "provider_session_observed_at TEXT",
     "provider_lookup_status TEXT",
     "provider_status TEXT",
     "provider_error_code TEXT",
@@ -1912,6 +1915,7 @@ function rowToHarvestAttemptLog(row) {
     browser_close_error: row.browser_close_error != null ? String(row.browser_close_error) : null,
     browser_provider: row.browser_provider != null ? String(row.browser_provider) : null,
     provider_session_id: row.provider_session_id != null ? String(row.provider_session_id) : null,
+    provider_session_observed_at: row.provider_session_observed_at != null ? String(row.provider_session_observed_at) : null,
     provider_lookup_status: row.provider_lookup_status != null ? String(row.provider_lookup_status) : null,
     provider_status: row.provider_status != null ? String(row.provider_status) : null,
     provider_error_code: row.provider_error_code != null ? String(row.provider_error_code) : null,
@@ -1991,6 +1995,49 @@ async function startHarvestAttempt(input) {
     ]
   });
 }
+async function attachHarvestAttemptProviderSession(input) {
+  const result = await getDb().execute({
+    sql: `
+      UPDATE harvest_attempts SET
+        browser_provider = ?,
+        provider_session_id = ?,
+        provider_session_observed_at = COALESCE(provider_session_observed_at, ?),
+        provider_lookup_status = CASE
+          WHEN provider_lookup_status = 'complete' THEN provider_lookup_status
+          ELSE 'pending'
+        END
+      WHERE job_id = ? AND attempt_number = ? AND status = 'running'
+    `,
+    args: [
+      input.browserProvider,
+      input.providerSessionId,
+      input.observedAt,
+      input.jobId,
+      input.attemptNumber
+    ]
+  });
+  return result.rowsAffected === 1;
+}
+async function finalizeExpiredHarvestAttempts(jobId, willRetry, completedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const result = await getDb().execute({
+    sql: `
+      UPDATE harvest_attempts SET
+        status = 'finished',
+        outcome = COALESCE(outcome, 'timeout'),
+        question_count = COALESCE(question_count, 0),
+        duration_ms = COALESCE(duration_ms, MAX(0, CAST((julianday(?) - julianday(created_at)) * 86400000 AS INTEGER))),
+        error = COALESCE(error, 'Durable worker lease expired before attempt completion'),
+        will_retry = ?,
+        provider_error_code = COALESCE(provider_error_code, 'timeout'),
+        provider_error_message = COALESCE(provider_error_message, 'Durable worker lease expired before attempt completion'),
+        provider_reason_source = COALESCE(provider_reason_source, 'client_runtime'),
+        completed_at = COALESCE(completed_at, ?)
+      WHERE job_id = ? AND status = 'running'
+    `,
+    args: [completedAt, willRetry ? 1 : 0, completedAt, jobId]
+  });
+  return result.rowsAffected;
+}
 async function finishHarvestAttempt(input) {
   await getDb().execute({
     sql: `
@@ -3869,6 +3916,8 @@ export {
   createRunningJob,
   getOrCreateRunningJob,
   startHarvestAttempt,
+  attachHarvestAttemptProviderSession,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   updateHarvestAttemptProviderTelemetry,
   listHarvestAttemptsPendingProviderTelemetry,

package/dist/{db-TA567JLA.js → db-Z34LPZNR.js}

@@ -7,6 +7,7 @@ import {
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
   appendWorkflowArtifacts,
+  attachHarvestAttemptProviderSession,
   cancelJob,
   checkRateLimit,
   checkpointDurablePaaJob,
@@ -49,6 +50,7 @@ import {
   failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   generateApiKey,
   getBillingLedgerSummary,
@@ -137,7 +139,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,
@@ -147,6 +149,7 @@ export {
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
   appendWorkflowArtifacts,
+  attachHarvestAttemptProviderSession,
   cancelJob,
   checkRateLimit,
   checkpointDurablePaaJob,
@@ -189,6 +192,7 @@ export {
   failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   generateApiKey,
   getBillingLedgerSummary,

package/dist/{extract-bundle-I2GNC25L.js → extract-bundle-S3ZGNFML.js}

@@ -4,13 +4,13 @@ import {
   downloadAsset,
   embedCommonsTexts,
   publicizeExtractionFailure
-} from "./chunk-BTQ23KCT.js";
+} from "./chunk-F5GQJWZU.js";
 import {
   createSiteExtractBundleArtifactStream,
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-SL7YVA4S.js";
+} from "./chunk-YBR32LOD.js";
 import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
@@ -23,7 +23,7 @@ import "./chunk-VFCUGTCF.js";
 import "./chunk-GGZEC22A.js";
 import {
   getDb
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";

package/dist/{gmail-service-GGQVJUC3.js → gmail-service-YFS2OV4S.js}

@@ -7,12 +7,12 @@ import {
   GmailServiceError,
   normalizeGmailMessage,
   parseGmailAddresses
-} from "./chunk-LB2GNACG.js";
-import "./chunk-Q5L2OCOQ.js";
+} from "./chunk-3423R23T.js";
+import "./chunk-C5Z4OFKW.js";
 import "./chunk-T3MZISOF.js";
 import "./chunk-VFCUGTCF.js";
-import "./chunk-NGLZ6UJR.js";
-import "./chunk-OU4FV2RP.js";
+import "./chunk-5UAFBKJR.js";
+import "./chunk-YXNDOQXN.js";
 export {
   GMAIL_ATTACHMENT_REF_TTL_MS,
   GMAIL_INLINE_BODY_BYTES,