mcp-scraper 0.69.1 → 0.72.1

package/dist/chunk-T3MZISOF.js

@@ -0,0 +1,240 @@
+// src/api/connected-data-artifacts.ts
+import { createHash, randomUUID } from "crypto";
+import { mkdir, readFile, stat, writeFile } from "fs/promises";
+import { homedir } from "os";
+import { dirname, join } from "path";
+var CONNECTED_DATA_ARTIFACT_PREFIX = "connected-data-exports/";
+var CONNECTED_DATA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
+var CONNECTED_DATA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
+function privateBlobToken() {
+  return process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
+}
+function localBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
+}
+function isHosted() {
+  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
+}
+function safeFilename(value) {
+  const safe = value.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+  return (safe || "connected-data-export").slice(0, 120);
+}
+function artifactTimestamp(artifactId) {
+  if (!artifactId.startsWith(CONNECTED_DATA_ARTIFACT_PREFIX)) return null;
+  const filename = artifactId.split("/").at(-1) ?? "";
+  const match = filename.match(/^(\d{13})-/);
+  if (!match) return null;
+  const timestamp = Number(match[1]);
+  return Number.isFinite(timestamp) ? timestamp : null;
+}
+function connectedDataArtifactOwnerId(artifactId) {
+  if (!artifactId.startsWith(CONNECTED_DATA_ARTIFACT_PREFIX)) return null;
+  const rest = artifactId.slice(CONNECTED_DATA_ARTIFACT_PREFIX.length);
+  const owner = rest.split("/")[0];
+  return owner || null;
+}
+function connectedDataArtifactExpiresAt(artifactId) {
+  const timestamp = artifactTimestamp(artifactId);
+  return timestamp === null ? null : new Date(timestamp + CONNECTED_DATA_ARTIFACT_TTL_MS);
+}
+function connectedDataArtifactIsExpired(artifactId, now = Date.now()) {
+  const timestamp = artifactTimestamp(artifactId);
+  return timestamp === null || timestamp + CONNECTED_DATA_ARTIFACT_TTL_MS <= now;
+}
+async function signedDownloadUrl(pathname, expiresAt) {
+  const token = privateBlobToken();
+  if (!token) return null;
+  const validUntil = Math.min(Date.now() + CONNECTED_DATA_DOWNLOAD_TTL_MS, expiresAt.getTime());
+  if (validUntil <= Date.now()) return null;
+  const { issueSignedToken, presignUrl } = await import("@vercel/blob");
+  const signedToken = await issueSignedToken({
+    token,
+    pathname,
+    operations: ["get"],
+    validUntil
+  });
+  const { presignedUrl } = await presignUrl(signedToken, {
+    access: "private",
+    operation: "get",
+    pathname,
+    validUntil
+  });
+  return { url: presignedUrl, expiresAt: new Date(validUntil).toISOString() };
+}
+async function createConnectedDataArtifact(args) {
+  return createConnectedDataBinaryArtifact({
+    ...args,
+    contentType: "application/x-ndjson"
+  });
+}
+async function createConnectedDataBinaryArtifact(args) {
+  const requestedCreatedAt = args.stableCreatedAt ? Date.parse(args.stableCreatedAt) : Number.NaN;
+  const createdAt = Number.isFinite(requestedCreatedAt) ? requestedCreatedAt : Date.now();
+  const filename = safeFilename(args.filename);
+  const extension = filename.includes(".") ? `.${filename.split(".").at(-1)}` : ".bin";
+  const stableSuffix = args.stableKey ? safeFilename(args.stableKey) : randomUUID();
+  const requestedPathname = `${CONNECTED_DATA_ARTIFACT_PREFIX}${args.ownerId}/${createdAt}-${safeFilename(args.exportId)}-${stableSuffix}--${filename}`;
+  const content = typeof args.content === "string" ? Buffer.from(args.content) : Buffer.from(args.content);
+  const bytes = content.byteLength;
+  const sha256 = createHash("sha256").update(content).digest("hex");
+  const expiresAt = new Date(createdAt + CONNECTED_DATA_ARTIFACT_TTL_MS);
+  const token = privateBlobToken();
+  let artifactId = requestedPathname;
+  if (token) {
+    const { put } = await import("@vercel/blob");
+    const stored = await put(requestedPathname, content, {
+      access: "private",
+      token,
+      contentType: args.contentType,
+      addRandomSuffix: !args.stableKey,
+      ...args.stableKey ? { allowOverwrite: true } : {},
+      cacheControlMaxAge: 60,
+      multipart: bytes > 100 * 1024 * 1024
+    });
+    artifactId = stored.pathname;
+  } else {
+    if (isHosted()) throw new Error("connected_data_private_blob_not_configured");
+    const path = join(localBaseDir(), "blobs", requestedPathname);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, content);
+  }
+  const download = await signedDownloadUrl(artifactId, expiresAt);
+  return {
+    artifactId,
+    filename,
+    contentType: args.contentType,
+    bytes,
+    sha256,
+    createdAt: new Date(createdAt).toISOString(),
+    expiresAt: expiresAt.toISOString(),
+    downloadUrl: download?.url ?? null,
+    downloadUrlExpiresAt: download?.expiresAt ?? null,
+    readback: {
+      tool: "report_artifact_read",
+      arguments: { artifactId, offset: 0, maxBytes: 2e4 },
+      continuation: "Repeat with offset set to the previous result nextOffset until nextOffset is null."
+    }
+  };
+}
+async function renewConnectedDataArtifactDownload(args) {
+  if (connectedDataArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
+  const expiresAt = connectedDataArtifactExpiresAt(args.artifactId);
+  if (!expiresAt || expiresAt.getTime() <= Date.now()) return null;
+  const download = await signedDownloadUrl(args.artifactId, expiresAt);
+  if (!download) return null;
+  return {
+    downloadUrl: download.url,
+    downloadUrlExpiresAt: download.expiresAt,
+    expiresAt: expiresAt.toISOString()
+  };
+}
+async function streamToBuffer(stream, maxBytes = Number.POSITIVE_INFINITY) {
+  const reader = stream.getReader();
+  const chunks = [];
+  let totalBytes = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value) {
+        const chunk = Buffer.from(value);
+        totalBytes += chunk.byteLength;
+        if (totalBytes > maxBytes) {
+          await reader.cancel("artifact_too_large").catch(() => void 0);
+          return null;
+        }
+        chunks.push(chunk);
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return Buffer.concat(chunks);
+}
+async function readConnectedDataArtifactWindow(artifactId, offset, maxBytes) {
+  if (connectedDataArtifactIsExpired(artifactId)) return null;
+  const token = privateBlobToken();
+  let buffer;
+  if (token) {
+    const { get } = await import("@vercel/blob");
+    const result = await get(artifactId, { access: "private", token, useCache: false });
+    if (!result || result.statusCode !== 200) return null;
+    const streamed = await streamToBuffer(result.stream);
+    if (!streamed) return null;
+    buffer = streamed;
+  } else {
+    if (isHosted()) return null;
+    try {
+      buffer = await readFile(join(localBaseDir(), "blobs", artifactId));
+    } catch {
+      return null;
+    }
+  }
+  const totalBytes = buffer.length;
+  const start = Math.max(0, offset);
+  const end = Math.min(totalBytes, start + maxBytes);
+  return {
+    text: buffer.subarray(start, end).toString("utf8"),
+    totalBytes,
+    nextOffset: end < totalBytes ? end : null
+  };
+}
+async function readConnectedDataArtifactBytes(args) {
+  if (connectedDataArtifactOwnerId(args.artifactId) !== args.ownerId || connectedDataArtifactIsExpired(args.artifactId)) return null;
+  const token = privateBlobToken();
+  let buffer;
+  if (token) {
+    const { get } = await import("@vercel/blob");
+    const result = await get(args.artifactId, { access: "private", token, useCache: false });
+    if (!result || result.statusCode !== 200) return null;
+    const streamed = await streamToBuffer(result.stream, args.maxBytes ?? 25 * 1024 * 1024);
+    if (!streamed) return null;
+    buffer = streamed;
+  } else {
+    if (isHosted()) return null;
+    try {
+      const path = join(localBaseDir(), "blobs", args.artifactId);
+      const info = await stat(path);
+      if (info.size > (args.maxBytes ?? 25 * 1024 * 1024)) return null;
+      buffer = await readFile(path);
+    } catch {
+      return null;
+    }
+  }
+  if (buffer.byteLength > (args.maxBytes ?? 25 * 1024 * 1024)) return null;
+  return buffer;
+}
+async function cleanupExpiredConnectedDataArtifacts(args = {}) {
+  const now = args.now ?? /* @__PURE__ */ new Date();
+  const token = privateBlobToken();
+  if (!token) return { deleted: 0, store: isHosted() ? "none" : "local" };
+  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 17)) {
+    return { deleted: 0, store: "private-vercel-blob", skipped: true };
+  }
+  const cutoff = now.getTime() - CONNECTED_DATA_ARTIFACT_TTL_MS;
+  const { list, del } = await import("@vercel/blob");
+  let cursor;
+  let deleted = 0;
+  for (let page = 0; page < 20; page++) {
+    const result = await list({ prefix: CONNECTED_DATA_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
+    const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
+    if (expired.length > 0) {
+      await del(expired.map((blob) => blob.pathname), { token });
+      deleted += expired.length;
+    }
+    if (!result.hasMore || !result.cursor) break;
+    cursor = result.cursor;
+  }
+  return { deleted, store: "private-vercel-blob" };
+}
+
+export {
+  CONNECTED_DATA_ARTIFACT_PREFIX,
+  connectedDataArtifactOwnerId,
+  createConnectedDataArtifact,
+  createConnectedDataBinaryArtifact,
+  renewConnectedDataArtifactDownload,
+  readConnectedDataArtifactWindow,
+  readConnectedDataArtifactBytes,
+  cleanupExpiredConnectedDataArtifacts
+};

package/dist/{chunk-65FTMB7G.js → chunk-WEFPBAAG.js}

@@ -379,7 +379,7 @@ var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
 var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
 var _db = null;
 var _rateLimitSchemaReady = false;
-var CORE_SCHEMA_VERSION = "2026-08-23.1";
+var CORE_SCHEMA_VERSION = "2026-08-27.1";
 var SITE_EXTRACT_COUNTER_REPAIR_VERSION = "data-2026-07-28-site-extract-counters.1";
 var CORE_SCHEMA_TABLES = [
   "admin_credit_adjustment_events",
@@ -437,6 +437,7 @@ var CORE_SCHEMA_INDEXES = [
   "credit_lots_user_active",
   "credit_lot_repairs_user_applied",
   "harvest_attempts_job_id",
+  "harvest_attempts_provider_lookup",
   "harvest_attempts_user_created_at",
   "inbox_messages_user_sent_at",
   "jobs_created_at_status_user",
@@ -480,7 +481,23 @@ var CORE_SCHEMA_TRIGGERS = [
 ];
 var CORE_SCHEMA_COLUMNS = {
   credit_lots: ["never_expires"],
-  harvest_attempts: ["debug_json"],
+  harvest_attempts: [
+    "debug_json",
+    "browser_provider",
+    "provider_session_id",
+    "provider_lookup_status",
+    "provider_status",
+    "provider_error_code",
+    "provider_error_message",
+    "provider_reason_source",
+    "provider_duration_ms",
+    "provider_bandwidth_bytes",
+    "provider_captcha",
+    "provider_checked_at",
+    "provider_detail_json",
+    "provider_disconnect_observed",
+    "provider_disconnect_message"
+  ],
   jobs: ["public_error_json"],
   oauth_tokens: ["replaced_by", "rotated_at"],
   site_extract_jobs: [
@@ -689,6 +706,20 @@ async function migrate(options = {}) {
       browser_close_succeeded  INTEGER,
       browser_close_error      TEXT,
       debug_json               TEXT,
+      browser_provider         TEXT,
+      provider_session_id      TEXT,
+      provider_lookup_status   TEXT,
+      provider_status          TEXT,
+      provider_error_code      TEXT,
+      provider_error_message   TEXT,
+      provider_reason_source   TEXT,
+      provider_duration_ms     INTEGER,
+      provider_bandwidth_bytes INTEGER,
+      provider_captcha         INTEGER,
+      provider_checked_at      TEXT,
+      provider_detail_json     TEXT,
+      provider_disconnect_observed INTEGER,
+      provider_disconnect_message TEXT,
       created_at               TEXT NOT NULL DEFAULT (datetime('now')),
       completed_at             TEXT,
       UNIQUE(job_id, attempt_number)
@@ -700,6 +731,28 @@ async function migrate(options = {}) {
     await db.execute(`ALTER TABLE harvest_attempts ADD COLUMN debug_json TEXT`);
   } catch {
   }
+  for (const column of [
+    "browser_provider TEXT",
+    "provider_session_id TEXT",
+    "provider_lookup_status TEXT",
+    "provider_status TEXT",
+    "provider_error_code TEXT",
+    "provider_error_message TEXT",
+    "provider_reason_source TEXT",
+    "provider_duration_ms INTEGER",
+    "provider_bandwidth_bytes INTEGER",
+    "provider_captcha INTEGER",
+    "provider_checked_at TEXT",
+    "provider_detail_json TEXT",
+    "provider_disconnect_observed INTEGER",
+    "provider_disconnect_message TEXT"
+  ]) {
+    try {
+      await db.execute(`ALTER TABLE harvest_attempts ADD COLUMN ${column}`);
+    } catch {
+    }
+  }
+  await db.execute(`CREATE INDEX IF NOT EXISTS harvest_attempts_provider_lookup ON harvest_attempts(browser_provider, provider_lookup_status, provider_checked_at)`);
   try {
     await db.execute(`ALTER TABLE users ADD COLUMN password_hash TEXT`);
   } catch {
@@ -1854,6 +1907,20 @@ function rowToHarvestAttemptLog(row) {
     kernel_delete_error: row.kernel_delete_error != null ? String(row.kernel_delete_error) : null,
     browser_close_succeeded: nullableBoolean(row.browser_close_succeeded),
     browser_close_error: row.browser_close_error != null ? String(row.browser_close_error) : null,
+    browser_provider: row.browser_provider != null ? String(row.browser_provider) : null,
+    provider_session_id: row.provider_session_id != null ? String(row.provider_session_id) : null,
+    provider_lookup_status: row.provider_lookup_status != null ? String(row.provider_lookup_status) : null,
+    provider_status: row.provider_status != null ? String(row.provider_status) : null,
+    provider_error_code: row.provider_error_code != null ? String(row.provider_error_code) : null,
+    provider_error_message: row.provider_error_message != null ? String(row.provider_error_message) : null,
+    provider_reason_source: row.provider_reason_source != null ? String(row.provider_reason_source) : null,
+    provider_duration_ms: row.provider_duration_ms != null ? Number(row.provider_duration_ms) : null,
+    provider_bandwidth_bytes: row.provider_bandwidth_bytes != null ? Number(row.provider_bandwidth_bytes) : null,
+    provider_captcha: nullableBoolean(row.provider_captcha),
+    provider_checked_at: row.provider_checked_at != null ? String(row.provider_checked_at) : null,
+    provider_detail: parseNullableJson(row.provider_detail_json),
+    provider_disconnect_observed: nullableBoolean(row.provider_disconnect_observed),
+    provider_disconnect_message: row.provider_disconnect_message != null ? String(row.provider_disconnect_message) : null,
     debug: parseNullableJson(row.debug_json),
     created_at: String(row.created_at),
     completed_at: row.completed_at != null ? String(row.completed_at) : null
@@ -1906,13 +1973,7 @@ async function startHarvestAttempt(input) {
         id, job_id, user_id, attempt_number, max_attempts, query, location, max_questions, status, created_at
       )
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'running', ?)
-      ON CONFLICT(job_id, attempt_number) DO UPDATE SET
-        status = 'running',
-        created_at = excluded.created_at,
-        completed_at = NULL,
-        outcome = NULL,
-        error = NULL,
-        debug_json = NULL
+      ON CONFLICT(job_id, attempt_number) DO NOTHING
     `,
     args: [
       randomUUID(),
@@ -1943,6 +2004,32 @@ async function finishHarvestAttempt(input) {
         kernel_delete_error = ?,
         browser_close_succeeded = ?,
         browser_close_error = ?,
+        browser_provider = COALESCE(browser_provider, ?),
+        provider_session_id = COALESCE(provider_session_id, ?),
+        provider_lookup_status = CASE
+          WHEN provider_lookup_status = 'complete' THEN provider_lookup_status
+          ELSE COALESCE(?, provider_lookup_status)
+        END,
+        provider_status = COALESCE(provider_status, ?),
+        provider_error_code = CASE
+          WHEN provider_reason_source = 'provider_session_api' THEN provider_error_code
+          ELSE COALESCE(?, provider_error_code)
+        END,
+        provider_error_message = CASE
+          WHEN provider_reason_source = 'provider_session_api' THEN provider_error_message
+          ELSE COALESCE(?, provider_error_message)
+        END,
+        provider_reason_source = CASE
+          WHEN provider_reason_source = 'provider_session_api' THEN provider_reason_source
+          ELSE COALESCE(?, provider_reason_source)
+        END,
+        provider_duration_ms = COALESCE(provider_duration_ms, ?),
+        provider_bandwidth_bytes = COALESCE(provider_bandwidth_bytes, ?),
+        provider_captcha = COALESCE(provider_captcha, ?),
+        provider_checked_at = COALESCE(provider_checked_at, ?),
+        provider_detail_json = COALESCE(provider_detail_json, ?),
+        provider_disconnect_observed = COALESCE(provider_disconnect_observed, ?),
+        provider_disconnect_message = COALESCE(provider_disconnect_message, ?),
         debug_json = ?,
         completed_at = ?
       WHERE job_id = ? AND attempt_number = ?
@@ -1959,6 +2046,20 @@ async function finishHarvestAttempt(input) {
       input.kernelDeleteError,
       input.browserCloseSucceeded == null ? null : input.browserCloseSucceeded ? 1 : 0,
       input.browserCloseError,
+      input.browserProvider ?? null,
+      input.providerSessionId ?? null,
+      input.providerLookupStatus ?? null,
+      input.providerStatus ?? null,
+      input.providerErrorCode ?? null,
+      input.providerErrorMessage ?? null,
+      input.providerReasonSource ?? null,
+      input.providerDurationMs ?? null,
+      input.providerBandwidthBytes ?? null,
+      input.providerCaptcha == null ? null : input.providerCaptcha ? 1 : 0,
+      input.providerCheckedAt ?? null,
+      input.providerDetail == null ? null : boundedJson(input.providerDetail),
+      input.providerDisconnectObserved == null ? null : input.providerDisconnectObserved ? 1 : 0,
+      input.providerDisconnectMessage ?? null,
       input.debug == null ? null : JSON.stringify(input.debug),
       input.completedAt,
       input.jobId,
@@ -1966,6 +2067,65 @@ async function finishHarvestAttempt(input) {
     ]
   });
 }
+function boundedJson(value, maxBytes = 4096) {
+  const serialized = JSON.stringify(value);
+  return serialized.length <= maxBytes ? serialized : JSON.stringify({ truncated: true });
+}
+async function updateHarvestAttemptProviderTelemetry(input) {
+  const result = await getDb().execute({
+    sql: `
+      UPDATE harvest_attempts SET
+        provider_lookup_status = ?,
+        provider_status = ?,
+        provider_error_code = ?,
+        provider_error_message = ?,
+        provider_reason_source = ?,
+        provider_duration_ms = ?,
+        provider_bandwidth_bytes = ?,
+        provider_captcha = ?,
+        provider_checked_at = ?,
+        provider_detail_json = ?
+      WHERE id = ?
+        AND job_id = ?
+        AND user_id = ?
+        AND browser_provider = 'bright_data_browser_api'
+        AND provider_session_id = ?
+        AND COALESCE(provider_lookup_status, '') <> 'complete'
+    `,
+    args: [
+      input.lookupStatus,
+      input.providerStatus ?? null,
+      input.errorCode ?? null,
+      input.errorMessage ?? null,
+      input.reasonSource ?? null,
+      input.durationMs ?? null,
+      input.bandwidthBytes ?? null,
+      input.captcha == null ? null : input.captcha ? 1 : 0,
+      input.checkedAt,
+      input.detail == null ? null : boundedJson(input.detail),
+      input.attemptId,
+      input.jobId,
+      input.userId,
+      input.providerSessionId
+    ]
+  });
+  return result.rowsAffected === 1;
+}
+async function listHarvestAttemptsPendingProviderTelemetry(limit = 25) {
+  const boundedLimit = Math.max(1, Math.min(100, Math.trunc(limit)));
+  const result = await getDb().execute({
+    sql: `
+      SELECT * FROM harvest_attempts
+      WHERE browser_provider = 'bright_data_browser_api'
+        AND provider_session_id IS NOT NULL
+        AND provider_lookup_status IN ('pending', 'temporary_error')
+      ORDER BY COALESCE(provider_checked_at, created_at) ASC
+      LIMIT ?
+    `,
+    args: [boundedLimit]
+  });
+  return result.rows.map((row) => rowToHarvestAttemptLog(row));
+}
 async function listHarvestAttempts(jobId, userId) {
   const res = userId == null ? await getDb().execute({
     sql: `SELECT * FROM harvest_attempts WHERE job_id = ? ORDER BY attempt_number ASC`,
@@ -2502,6 +2662,14 @@ async function completeJob(id, result) {
     args: [JSON.stringify(result), id]
   });
 }
+async function completeRunningJob(id, result) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'done', result = ?, error = NULL, public_error_json = NULL, completed_at = datetime('now')
+          WHERE id = ? AND status = 'running'`,
+    args: [JSON.stringify(result), id]
+  });
+  return updated.rowsAffected === 1;
+}
 async function failJob(id, error, publicError) {
   await getDb().execute({
     sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
@@ -3574,6 +3742,8 @@ export {
   getOrCreateRunningJob,
   startHarvestAttempt,
   finishHarvestAttempt,
+  updateHarvestAttemptProviderTelemetry,
+  listHarvestAttemptsPendingProviderTelemetry,
   listHarvestAttempts,
   getJob,
   listJobs,
@@ -3611,6 +3781,7 @@ export {
   listStaleRunningDurablePaaJobs,
   markJobBillingSettled,
   completeJob,
+  completeRunningJob,
   failJob,
   failRunningJob,
   cancelJob,

package/dist/{chunk-LN6N3YLK.js → chunk-WFQ2E4WE.js}

@@ -8,7 +8,7 @@ import {
 import {
   buildPublicErrorEnvelope,
   getDb
-} from "./chunk-65FTMB7G.js";
+} from "./chunk-WEFPBAAG.js";
 
 // src/lib/media-extractor.ts
 import { createWriteStream, mkdirSync, rmSync } from "fs";