mcp-scraper 0.35.1 → 0.37.0

package/dist/chunk-QZXKQB7Y.js

@@ -0,0 +1,414 @@
+import {
+  createPrivateArtifactFromStream,
+  renewPrivateArtifactDownload,
+  validatePublicHttpUrl
+} from "./chunk-345BQXZH.js";
+
+// src/lib/media-extractor.ts
+import { createWriteStream, mkdirSync, rmSync } from "fs";
+import { homedir } from "os";
+import { join, extname, basename } from "path";
+import { pipeline } from "stream/promises";
+import { Readable, Transform } from "stream";
+var AD_PATTERNS = [
+  "doubleclick.net",
+  "googlesyndication.com",
+  "googletagmanager.com",
+  "google-analytics.com",
+  "googletagservices.com",
+  "adservice.google",
+  "googletag.",
+  "pagead2.googlesyndication",
+  "facebook.net/tr",
+  "connect.facebook.net",
+  "fbcdn.net/rsrc",
+  "analytics.twitter.com",
+  "static.ads-twitter.com",
+  "ads.twitter.com",
+  "t.co/i/adsct",
+  "pixel.advertising.com",
+  "hotjar.com",
+  "clarity.ms",
+  "quantserve.com",
+  "scorecardresearch.com",
+  "newrelic.com",
+  "nr-data.net",
+  "segment.io",
+  "segment.com",
+  "amplitude.com",
+  "mixpanel.com",
+  "heap.io",
+  "fullstory.com",
+  "moatads.com",
+  "criteo.com",
+  "adsrvr.org",
+  "rubiconproject.com",
+  "pubmatic.com",
+  "openx.net",
+  "appnexus.com",
+  "amazon-adsystem.com",
+  "media.net",
+  "yieldmo.com",
+  "triplelift.com",
+  "sharethrough.com",
+  "prebid.",
+  "smaato.net",
+  "indexworm.com",
+  "casalemedia.com",
+  "outbrain.com",
+  "taboola.com",
+  "revcontent.com",
+  "mgid.com",
+  "tawk.to",
+  "intercom.io",
+  "drift.com",
+  "hs-scripts.com",
+  "zopim.com",
+  "livechatinc.com",
+  "userlike.com",
+  "onetrust.com",
+  "cookielaw.org",
+  "cookieinformation.com",
+  "trustarc.com",
+  "/ads/",
+  "/ad/",
+  "/banner/",
+  "/banners/",
+  "/pixel/",
+  "/beacon/",
+  "/tracking/",
+  "/tracker/",
+  "/remarketing/",
+  "/conversion/",
+  "1x1.gif",
+  "spacer.gif",
+  "blank.gif",
+  "transparent.gif"
+];
+var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
+var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
+var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
+function isAdUrl(url) {
+  const lower = url.toLowerCase();
+  return AD_PATTERNS.some((p) => lower.includes(p));
+}
+function isDataUri(url) {
+  return url.startsWith("data:");
+}
+function typeFromUrl(url) {
+  try {
+    const ext = extname(new URL(url).pathname).toLowerCase();
+    if (IMAGE_EXTS.has(ext)) return "image";
+    if (VIDEO_EXTS.has(ext)) return "video";
+    if (AUDIO_EXTS.has(ext)) return "audio";
+  } catch {
+  }
+  return null;
+}
+function typeFromMime(mime) {
+  const lower = mime.toLowerCase();
+  if (lower.startsWith("image/")) return "image";
+  if (lower.startsWith("video/")) return "video";
+  if (lower.startsWith("audio/")) return "audio";
+  return null;
+}
+function resolveUrl(raw, base) {
+  if (!raw || isDataUri(raw)) return null;
+  try {
+    return new URL(raw, base).href;
+  } catch {
+    return null;
+  }
+}
+function safeFilename(url, index) {
+  try {
+    const u = new URL(url);
+    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+    return base || `asset-${index}`;
+  } catch {
+    return `asset-${index}`;
+  }
+}
+function extractMediaUrls(html, baseUrl) {
+  const seen = /* @__PURE__ */ new Set();
+  const urls = [];
+  const add = (raw) => {
+    const resolved = resolveUrl(raw.trim(), baseUrl);
+    if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
+    seen.add(resolved);
+    urls.push(resolved);
+  };
+  for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
+    const tag = m[0];
+    for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
+      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
+      if (v) add(v);
+    }
+    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
+    if (srcset) {
+      for (const part of srcset.split(",")) {
+        const u = part.trim().split(/\s+/)[0];
+        if (u) add(u);
+      }
+    }
+  }
+  for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
+    const tag = m[0];
+    const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
+    if (src) add(src);
+    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
+    if (srcset) {
+      for (const part of srcset.split(",")) {
+        const u = part.trim().split(/\s+/)[0];
+        if (u) add(u);
+      }
+    }
+  }
+  for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
+    const tag = m[0];
+    for (const attr of ["src", "poster"]) {
+      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
+      if (v) add(v);
+    }
+  }
+  for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
+    const tag = m[0];
+    const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
+    if (v) add(v);
+  }
+  for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
+    const tag = m[0];
+    const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
+    if (prop === "og:image" || prop === "twitter:image") {
+      const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
+      if (content) add(content);
+    }
+  }
+  for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
+    add(m[1]);
+  }
+  return urls;
+}
+async function downloadAsset(url, destDir, filename, options = {}) {
+  const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024));
+  let target = url;
+  let res = null;
+  for (let redirects = 0; redirects <= 5; redirects++) {
+    const checked = await validatePublicHttpUrl(target, { field: "media URL" });
+    if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Media URL was rejected");
+    res = await fetch(checked.parsed.href, {
+      headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
+      signal: AbortSignal.timeout(15e3),
+      redirect: "manual"
+    });
+    if (res.status >= 300 && res.status < 400) {
+      const location = res.headers.get("location");
+      if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`);
+      target = new URL(location, checked.parsed.href).href;
+      res = null;
+      continue;
+    }
+    break;
+  }
+  if (!res) throw new Error("Media download exceeded five redirects");
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  if (!res.body) throw new Error("Empty response body");
+  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
+  if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {
+    throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? "no content-type"}`);
+  }
+  const declaredBytes = Number(res.headers.get("content-length") ?? 0);
+  if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
+    throw new Error(`Media asset exceeds ${maxBytes} byte limit`);
+  }
+  let dest = join(destDir, filename);
+  if (mimeType && !extname(filename)) {
+    const mimeExt = {
+      "image/jpeg": ".jpg",
+      "image/png": ".png",
+      "image/webp": ".webp",
+      "image/gif": ".gif",
+      "image/svg+xml": ".svg",
+      "image/avif": ".avif",
+      "video/mp4": ".mp4",
+      "video/webm": ".webm",
+      "audio/mpeg": ".mp3",
+      "audio/ogg": ".ogg",
+      "audio/wav": ".wav"
+    };
+    const ext = mimeExt[mimeType];
+    if (ext) dest = dest + ext;
+  }
+  const writer = createWriteStream(dest);
+  await new Promise((resolve, reject) => {
+    writer.once("open", () => resolve());
+    writer.once("error", reject);
+  });
+  let streamedBytes = 0;
+  const limiter = new Transform({
+    transform(chunk, _encoding, callback) {
+      const bytes = chunk.length;
+      if (streamedBytes + bytes > maxBytes) {
+        callback(new Error(`Media asset exceeds ${maxBytes} byte limit`));
+        return;
+      }
+      if (options.consumeBytes && !options.consumeBytes(bytes)) {
+        callback(new Error("Media export total byte limit exceeded"));
+        return;
+      }
+      streamedBytes += bytes;
+      callback(null, chunk);
+    }
+  });
+  try {
+    await pipeline(Readable.fromWeb(res.body), limiter, writer);
+  } catch (error) {
+    writer.destroy();
+    rmSync(dest, { force: true });
+    throw error;
+  }
+  const { statSync } = await import("fs");
+  const sizeBytes = statSync(dest).size;
+  return { savedPath: dest, sizeBytes, mimeType };
+}
+async function harvestPageMedia(html, pageUrl, options = {}) {
+  const types = options.types ?? ["image", "video", "audio"];
+  const typesSet = new Set(types);
+  const rawUrls = extractMediaUrls(html, pageUrl);
+  const totalFound = rawUrls.length;
+  const filteredUrls = [];
+  const kept = [];
+  for (const url of rawUrls) {
+    const type = typeFromUrl(url);
+    if (!type || !typesSet.has(type)) {
+      filteredUrls.push(url);
+      continue;
+    }
+    kept.push({ url, type });
+  }
+  if (options.outputDir === null) {
+    return {
+      pageUrl,
+      outputDir: null,
+      assets: kept.map(({ url, type }, i) => ({
+        url,
+        type,
+        mimeType: null,
+        filename: safeFilename(url, i),
+        savedPath: null,
+        sizeBytes: null
+      })),
+      filteredCount: filteredUrls.length,
+      totalFound
+    };
+  }
+  const domain = (() => {
+    try {
+      return new URL(pageUrl).hostname.replace(/^www\./, "");
+    } catch {
+      return "unknown";
+    }
+  })();
+  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
+  const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
+  mkdirSync(outDir, { recursive: true });
+  const assets = [];
+  await Promise.allSettled(
+    kept.map(async ({ url, type }, i) => {
+      const filename = safeFilename(url, i);
+      try {
+        const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
+        const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
+        assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
+      } catch {
+        assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
+      }
+    })
+  );
+  assets.sort((a, b) => {
+    if (a.savedPath && !b.savedPath) return -1;
+    if (!a.savedPath && b.savedPath) return 1;
+    return a.url.localeCompare(b.url);
+  });
+  return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
+}
+
+// src/api/site-extract-artifacts.ts
+var SITE_EXTRACT_ARTIFACT_PREFIX = "site-extracts/";
+var SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
+var SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
+function siteExtractArtifactToken() {
+  return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
+}
+function hostedByEnvironment() {
+  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
+}
+function policy() {
+  return {
+    prefix: SITE_EXTRACT_ARTIFACT_PREFIX,
+    artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,
+    downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,
+    token: siteExtractArtifactToken()
+  };
+}
+async function createSiteExtractBundleArtifactStream(args) {
+  const pointer = await createPrivateArtifactFromStream({
+    policy: policy(),
+    ownerId: args.ownerId,
+    artifactKey: `${args.jobId}.zip`,
+    createdAt: args.createdAt,
+    filename: `${args.jobId}-site-export.zip`,
+    contentType: "application/zip",
+    content: args.content
+  });
+  return {
+    key: pointer.artifactId,
+    url: pointer.downloadUrl ?? "",
+    bytes: pointer.bytes,
+    contentType: pointer.contentType,
+    filename: pointer.filename,
+    sha256: pointer.sha256,
+    expiresAt: pointer.expiresAt,
+    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt
+  };
+}
+async function renewSiteExtractArtifactDownload(args) {
+  return renewPrivateArtifactDownload({
+    policy: policy(),
+    artifactId: args.artifactId,
+    ownerId: args.ownerId
+  });
+}
+async function cleanupExpiredSiteExtractArtifacts(args = {}) {
+  const now = args.now ?? /* @__PURE__ */ new Date();
+  const token = siteExtractArtifactToken();
+  if (!token) return { deleted: 0, store: hostedByEnvironment() ? "none" : "local" };
+  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {
+    return { deleted: 0, store: "private-vercel-blob", skipped: true };
+  }
+  const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS;
+  const { list, del } = await import("@vercel/blob");
+  let cursor;
+  let deleted = 0;
+  for (let page = 0; page < 20; page += 1) {
+    const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
+    const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
+    if (expired.length > 0) {
+      await del(expired.map((blob) => blob.pathname), { token });
+      deleted += expired.length;
+    }
+    if (!result.hasMore || !result.cursor) break;
+    cursor = result.cursor;
+  }
+  return { deleted, store: "private-vercel-blob" };
+}
+
+export {
+  downloadAsset,
+  harvestPageMedia,
+  SITE_EXTRACT_ARTIFACT_PREFIX,
+  createSiteExtractBundleArtifactStream,
+  renewSiteExtractArtifactDownload,
+  cleanupExpiredSiteExtractArtifacts
+};
+//# sourceMappingURL=chunk-QZXKQB7Y.js.map

package/dist/chunk-QZXKQB7Y.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/site-extract-artifacts.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\n\nconst AD_PATTERNS: string[] = [\n  'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n  'google-analytics.com', 'googletagservices.com', 'adservice.google',\n  'googletag.', 'pagead2.googlesyndication',\n  'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n  'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n  't.co/i/adsct', 'pixel.advertising.com',\n  'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n  'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n  'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n  'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n  'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n  'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n  'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n  'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n  'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n  'zopim.com', 'livechatinc.com', 'userlike.com',\n  'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n  '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n  '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n  '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS  = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS  = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS  = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n  url:        string\n  type:       MediaType\n  mimeType:   string | null\n  filename:   string\n  savedPath:  string | null\n  sizeBytes:  number | null\n}\n\nexport interface MediaManifest {\n  pageUrl:       string\n  outputDir:     string | null\n  assets:        MediaAsset[]\n  filteredCount: number\n  totalFound:    number\n}\n\nexport interface MediaExtractOptions {\n  types?:      MediaType[]\n  outputDir?:  string | null\n}\n\nfunction isAdUrl(url: string): boolean {\n  const lower = url.toLowerCase()\n  return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n  return url.startsWith('data:')\n}\n\nfunction typeFromUrl(url: string): MediaType | null {\n  try {\n    const ext = extname(new URL(url).pathname).toLowerCase()\n    if (IMAGE_EXTS.has(ext)) return 'image'\n    if (VIDEO_EXTS.has(ext)) return 'video'\n    if (AUDIO_EXTS.has(ext)) return 'audio'\n  } catch { /* non-parseable URL */ }\n  return null\n}\n\nfunction typeFromMime(mime: string): MediaType | null {\n  const lower = mime.toLowerCase()\n  if (lower.startsWith('image/')) return 'image'\n  if (lower.startsWith('video/')) return 'video'\n  if (lower.startsWith('audio/')) return 'audio'\n  return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n  if (!raw || isDataUri(raw)) return null\n  try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n  try {\n    const u = new URL(url)\n    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n    return base || `asset-${index}`\n  } catch {\n    return `asset-${index}`\n  }\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n  const seen = new Set<string>()\n  const urls: string[] = []\n\n  const add = (raw: string) => {\n    const resolved = resolveUrl(raw.trim(), baseUrl)\n    if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return\n    seen.add(resolved)\n    urls.push(resolved)\n  }\n\n  for (const m of html.matchAll(/<img\\s[^>]*>/gi)) {\n    const tag = m[0]\n    for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original']) {\n      const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n      if (v) add(v)\n    }\n    const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n    if (srcset) {\n      for (const part of srcset.split(',')) {\n        const u = part.trim().split(/\\s+/)[0]\n        if (u) add(u)\n      }\n    }\n  }\n\n  for (const m of html.matchAll(/<source\\s[^>]*>/gi)) {\n    const tag = m[0]\n    const src = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n    if (src) add(src)\n    const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n    if (srcset) {\n      for (const part of srcset.split(',')) {\n        const u = part.trim().split(/\\s+/)[0]\n        if (u) add(u)\n      }\n    }\n  }\n\n  for (const m of html.matchAll(/<video\\s[^>]*>/gi)) {\n    const tag = m[0]\n    for (const attr of ['src', 'poster']) {\n      const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n      if (v) add(v)\n    }\n  }\n\n  for (const m of html.matchAll(/<audio\\s[^>]*>/gi)) {\n    const tag = m[0]\n    const v = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n    if (v) add(v)\n  }\n\n  for (const m of html.matchAll(/<meta\\s[^>]*>/gi)) {\n    const tag = m[0]\n    const prop = (tag.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]?.toLowerCase()\n    if (prop === 'og:image' || prop === 'twitter:image') {\n      const content = (tag.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n      if (content) add(content)\n    }\n  }\n\n  for (const m of html.matchAll(/background(?:-image)?\\s*:\\s*url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)/gi)) {\n    add(m[1])\n  }\n\n  return urls\n}\n\nexport async function downloadAsset(\n  url: string,\n  destDir: string,\n  filename: string,\n  options: {\n    expectedType?: MediaType\n    maxBytes?: number\n    consumeBytes?: (bytes: number) => boolean\n  } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n  const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n  let target = url\n  let res: Response | null = null\n  for (let redirects = 0; redirects <= 5; redirects++) {\n    const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n    if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n    res = await fetch(checked.parsed.href, {\n      headers: { 'User-Agent': 'Mozilla/5.0 (compatible; ThorbitBot/1.0)' },\n      signal: AbortSignal.timeout(15_000),\n      redirect: 'manual',\n    })\n    if (res.status >= 300 && res.status < 400) {\n      const location = res.headers.get('location')\n      if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n      target = new URL(location, checked.parsed.href).href\n      res = null\n      continue\n    }\n    break\n  }\n  if (!res) throw new Error('Media download exceeded five redirects')\n  if (!res.ok) throw new Error(`HTTP ${res.status}`)\n  if (!res.body) throw new Error('Empty response body')\n\n  const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n  if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n    throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n  }\n  const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n  if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n    throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n  }\n\n  let dest = join(destDir, filename)\n  if (mimeType && !extname(filename)) {\n    const mimeExt: Record<string, string> = {\n      'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n      'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n      'video/mp4': '.mp4', 'video/webm': '.webm',\n      'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n    }\n    const ext = mimeExt[mimeType]\n    if (ext) dest = dest + ext\n  }\n\n  const writer = createWriteStream(dest)\n  await new Promise<void>((resolve, reject) => {\n    writer.once('open', () => resolve())\n    writer.once('error', reject)\n  })\n  let streamedBytes = 0\n  const limiter = new Transform({\n    transform(chunk: Buffer, _encoding, callback) {\n      const bytes = chunk.length\n      if (streamedBytes + bytes > maxBytes) {\n        callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n        return\n      }\n      if (options.consumeBytes && !options.consumeBytes(bytes)) {\n        callback(new Error('Media export total byte limit exceeded'))\n        return\n      }\n      streamedBytes += bytes\n      callback(null, chunk)\n    },\n  })\n  try {\n    await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n  } catch (error) {\n    writer.destroy()\n    rmSync(dest, { force: true })\n    throw error\n  }\n\n  const { statSync } = await import('node:fs')\n  const sizeBytes = statSync(dest).size\n\n  return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n  html: string,\n  pageUrl: string,\n  options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n  const types    = options.types ?? ['image', 'video', 'audio']\n  const typesSet = new Set(types)\n  const rawUrls  = extractMediaUrls(html, pageUrl)\n  const totalFound = rawUrls.length\n\n  const filteredUrls: string[] = []\n  const kept: Array<{ url: string; type: MediaType }> = []\n\n  for (const url of rawUrls) {\n    const type = typeFromUrl(url)\n    if (!type || !typesSet.has(type)) { filteredUrls.push(url); continue }\n    kept.push({ url, type })\n  }\n\n  if (options.outputDir === null) {\n    return {\n      pageUrl,\n      outputDir: null,\n      assets: kept.map(({ url, type }, i) => ({\n        url, type, mimeType: null, filename: safeFilename(url, i), savedPath: null, sizeBytes: null,\n      })),\n      filteredCount: filteredUrls.length,\n      totalFound,\n    }\n  }\n\n  const domain  = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n  const stamp   = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n  const outDir  = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n  mkdirSync(outDir, { recursive: true })\n\n  const assets: MediaAsset[] = []\n\n  await Promise.allSettled(\n    kept.map(async ({ url, type }, i) => {\n      const filename = safeFilename(url, i)\n      try {\n        const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename)\n        const resolvedType = mimeType ? (typeFromMime(mimeType) ?? type) : type\n        assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes })\n      } catch {\n        assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null })\n      }\n    })\n  )\n\n  assets.sort((a, b) => {\n    if (a.savedPath && !b.savedPath) return -1\n    if (!a.savedPath && b.savedPath) return 1\n    return a.url.localeCompare(b.url)\n  })\n\n  return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound }\n}\n","import {\n  createPrivateArtifact,\n  createPrivateArtifactFromStream,\n  privateArtifactOwnerId,\n  renewPrivateArtifactDownload,\n  type PrivateArtifactPolicy,\n} from './private-artifacts.js'\nimport type { ExtractJobArtifact } from './site-extract-repository.js'\nimport type { Readable } from 'node:stream'\n\nexport const SITE_EXTRACT_ARTIFACT_PREFIX = 'site-extracts/'\nexport const SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1_000\nexport const SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1_000\n\nfunction siteExtractArtifactToken(): string | null {\n  return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()\n    || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()\n    || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()\n    || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()\n    || null\n}\n\nfunction hostedByEnvironment(): boolean {\n  return process.env.VERCEL === '1' || process.env.NODE_ENV === 'production'\n}\n\nfunction policy(): PrivateArtifactPolicy {\n  return {\n    prefix: SITE_EXTRACT_ARTIFACT_PREFIX,\n    artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,\n    downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,\n    token: siteExtractArtifactToken(),\n  }\n}\n\nexport function siteExtractArtifactOwnerId(artifactId: string): string | null {\n  return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX)\n}\n\nexport async function createSiteExtractBundleArtifact(args: {\n  ownerId: string\n  jobId: string\n  createdAt: Date | string | number\n  content: Buffer\n}): Promise<ExtractJobArtifact> {\n  const pointer = await createPrivateArtifact({\n    policy: policy(),\n    ownerId: args.ownerId,\n    artifactKey: `${args.jobId}.zip`,\n    createdAt: args.createdAt,\n    filename: `${args.jobId}-site-export.zip`,\n    contentType: 'application/zip',\n    content: args.content,\n  })\n  return {\n    key: pointer.artifactId,\n    url: pointer.downloadUrl ?? '',\n    bytes: pointer.bytes,\n    contentType: pointer.contentType,\n    filename: pointer.filename,\n    sha256: pointer.sha256,\n    expiresAt: pointer.expiresAt,\n    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n  }\n}\n\nexport async function createSiteExtractBundleArtifactStream(args: {\n  ownerId: string\n  jobId: string\n  createdAt: Date | string | number\n  content: Readable\n}): Promise<ExtractJobArtifact> {\n  const pointer = await createPrivateArtifactFromStream({\n    policy: policy(),\n    ownerId: args.ownerId,\n    artifactKey: `${args.jobId}.zip`,\n    createdAt: args.createdAt,\n    filename: `${args.jobId}-site-export.zip`,\n    contentType: 'application/zip',\n    content: args.content,\n  })\n  return {\n    key: pointer.artifactId,\n    url: pointer.downloadUrl ?? '',\n    bytes: pointer.bytes,\n    contentType: pointer.contentType,\n    filename: pointer.filename,\n    sha256: pointer.sha256,\n    expiresAt: pointer.expiresAt,\n    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n  }\n}\n\nexport async function renewSiteExtractArtifactDownload(args: {\n  artifactId: string\n  ownerId: string\n}): Promise<{ downloadUrl: string; downloadUrlExpiresAt: string; expiresAt: string } | null> {\n  return renewPrivateArtifactDownload({\n    policy: policy(),\n    artifactId: args.artifactId,\n    ownerId: args.ownerId,\n  })\n}\n\nexport async function cleanupExpiredSiteExtractArtifacts(args: {\n  now?: Date\n  force?: boolean\n} = {}): Promise<{ deleted: number; store: 'private-vercel-blob' | 'local' | 'none'; skipped?: boolean }> {\n  const now = args.now ?? new Date()\n  const token = siteExtractArtifactToken()\n  if (!token) return { deleted: 0, store: hostedByEnvironment() ? 'none' : 'local' }\n  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {\n    return { deleted: 0, store: 'private-vercel-blob', skipped: true }\n  }\n  const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS\n  const { list, del } = await import('@vercel/blob')\n  let cursor: string | undefined\n  let deleted = 0\n  for (let page = 0; page < 20; page += 1) {\n    const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1_000, cursor })\n    const expired = result.blobs.filter(blob => new Date(blob.uploadedAt).getTime() <= cutoff)\n    if (expired.length > 0) {\n      await del(expired.map(blob => blob.pathname), { token })\n      deleted += expired.length\n    }\n    if (!result.hasMore || !result.cursor) break\n    cursor = result.cursor\n  }\n  return { deleted, store: 'private-vercel-blob' }\n}\n"],"mappings":";;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAGpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AA0BtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEA,SAAS,YAAY,KAA+B;AAClD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,iBAAiB,MAAc,SAA2B;AACxE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,OAAiB,CAAC;AAExB,QAAM,MAAM,CAAC,QAAgB;AAC3B,UAAM,WAAW,WAAW,IAAI,KAAK,GAAG,OAAO;AAC/C,QAAI,CAAC,YAAY,KAAK,IAAI,QAAQ,KAAK,QAAQ,QAAQ,EAAG;AAC1D,SAAK,IAAI,QAAQ;AACjB,SAAK,KAAK,QAAQ;AAAA,EACpB;AAEA,aAAW,KAAK,KAAK,SAAS,gBAAgB,GAAG;AAC/C,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,eAAe,GAAG;AACxE,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AACA,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,mBAAmB,GAAG;AAClD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,OAAO,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAChE,QAAI,IAAK,KAAI,GAAG;AAChB,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,QAAQ,GAAG;AACpC,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,KAAK,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAC9D,QAAI,EAAG,KAAI,CAAC;AAAA,EACd;AAEA,aAAW,KAAK,KAAK,SAAS,iBAAiB,GAAG;AAChD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,QAAQ,IAAI,MAAM,6CAA6C,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY;AAC9F,QAAI,SAAS,cAAc,SAAS,iBAAiB;AACnD,YAAM,WAAW,IAAI,MAAM,mCAAmC,KAAK,CAAC,GAAG,CAAC;AACxE,UAAI,QAAS,KAAI,OAAO;AAAA,IAC1B;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,gEAAgE,GAAG;AAC/F,QAAI,EAAE,CAAC,CAAC;AAAA,EACV;AAEA,SAAO;AACT;AAEA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,UAAW,iBAAiB,MAAM,OAAO;AAC/C,QAAM,aAAa,QAAQ;AAE3B,QAAM,eAAyB,CAAC;AAChC,QAAM,OAAgD,CAAC;AAEvD,aAAW,OAAO,SAAS;AACzB,UAAM,OAAO,YAAY,GAAG;AAC5B,QAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,GAAG;AAAE,mBAAa,KAAK,GAAG;AAAG;AAAA,IAAS;AACrE,SAAK,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,EACzB;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,EAAE,KAAK,KAAK,GAAG,OAAO;AAAA,QACtC;AAAA,QAAK;AAAA,QAAM,UAAU;AAAA,QAAM,UAAU,aAAa,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,MACzF,EAAE;AAAA,MACF,eAAe,aAAa;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,GAAG,MAAM;AACnC,YAAM,WAAW,aAAa,KAAK,CAAC;AACpC,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,KAAK,QAAQ,QAAQ;AACpF,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,OAAQ;AACnE,eAAO,KAAK,EAAE,KAAK,MAAM,cAAc,UAAU,UAAU,SAAS,SAAS,GAAG,WAAW,UAAU,CAAC;AAAA,MACxG,QAAQ;AACN,eAAO,KAAK,EAAE,KAAK,MAAM,UAAU,MAAM,UAAU,WAAW,MAAM,WAAW,KAAK,CAAC;AAAA,MACvF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,eAAe,aAAa,QAAQ,WAAW;AAC9F;;;ACnTO,IAAM,+BAA+B;AACrC,IAAM,+BAA+B,IAAI,KAAK,KAAK,KAAK;AACxD,IAAM,+BAA+B,KAAK,KAAK;AAEtD,SAAS,2BAA0C;AACjD,SAAO,QAAQ,IAAI,wCAAwC,KAAK,KAC3D,QAAQ,IAAI,mCAAmC,KAAK,KACpD,QAAQ,IAAI,iCAAiC,KAAK,KAClD,QAAQ,IAAI,sCAAsC,KAAK,KACvD;AACP;AAEA,SAAS,sBAA+B;AACtC,SAAO,QAAQ,IAAI,WAAW,OAAO,QAAQ,IAAI,aAAa;AAChE;AAEA,SAAS,SAAgC;AACvC,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,eAAe;AAAA,IACf,OAAO,yBAAyB;AAAA,EAClC;AACF;AAiCA,eAAsB,sCAAsC,MAK5B;AAC9B,QAAM,UAAU,MAAM,gCAAgC;AAAA,IACpD,QAAQ,OAAO;AAAA,IACf,SAAS,KAAK;AAAA,IACd,aAAa,GAAG,KAAK,KAAK;AAAA,IAC1B,WAAW,KAAK;AAAA,IAChB,UAAU,GAAG,KAAK,KAAK;AAAA,IACvB,aAAa;AAAA,IACb,SAAS,KAAK;AAAA,EAChB,CAAC;AACD,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ,eAAe;AAAA,IAC5B,OAAO,QAAQ;AAAA,IACf,aAAa,QAAQ;AAAA,IACrB,UAAU,QAAQ;AAAA,IAClB,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB,sBAAsB,QAAQ;AAAA,EAChC;AACF;AAEA,eAAsB,iCAAiC,MAGsC;AAC3F,SAAO,6BAA6B;AAAA,IAClC,QAAQ,OAAO;AAAA,IACf,YAAY,KAAK;AAAA,IACjB,SAAS,KAAK;AAAA,EAChB,CAAC;AACH;AAEA,eAAsB,mCAAmC,OAGrD,CAAC,GAAqG;AACxG,QAAM,MAAM,KAAK,OAAO,oBAAI,KAAK;AACjC,QAAM,QAAQ,yBAAyB;AACvC,MAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,OAAO,oBAAoB,IAAI,SAAS,QAAQ;AACjF,MAAI,CAAC,KAAK,SAAS,EAAE,IAAI,YAAY,MAAM,KAAK,IAAI,cAAc,MAAM,KAAK;AAC3E,WAAO,EAAE,SAAS,GAAG,OAAO,uBAAuB,SAAS,KAAK;AAAA,EACnE;AACA,QAAM,SAAS,IAAI,QAAQ,IAAI;AAC/B,QAAM,EAAE,MAAM,IAAI,IAAI,MAAM,OAAO,cAAc;AACjD,MAAI;AACJ,MAAI,UAAU;AACd,WAAS,OAAO,GAAG,OAAO,IAAI,QAAQ,GAAG;AACvC,UAAM,SAAS,MAAM,KAAK,EAAE,QAAQ,8BAA8B,OAAO,OAAO,KAAO,OAAO,CAAC;AAC/F,UAAM,UAAU,OAAO,MAAM,OAAO,UAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,QAAQ,KAAK,MAAM;AACzF,QAAI,QAAQ,SAAS,GAAG;AACtB,YAAM,IAAI,QAAQ,IAAI,UAAQ,KAAK,QAAQ,GAAG,EAAE,MAAM,CAAC;AACvD,iBAAW,QAAQ;AAAA,IACrB;AACA,QAAI,CAAC,OAAO,WAAW,CAAC,OAAO,OAAQ;AACvC,aAAS,OAAO;AAAA,EAClB;AACA,SAAO,EAAE,SAAS,OAAO,sBAAsB;AACjD;","names":[]}

package/dist/{db-YAI5AQOI.js → db-N3YECFWR.js}

@@ -1,4 +1,5 @@
 import {
+  CORE_SCHEMA_VERSION,
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
@@ -24,6 +25,7 @@ import {
   createWorkflowRun,
   createWorkflowSchedule,
   creditMc,
+  creditMcIdempotent,
   deactivateUser,
   debitMc,
   debitMcIdempotent,
@@ -36,6 +38,7 @@ import {
   generateApiKey,
   getBillingLedgerSummary,
   getClient,
+  getConnectedAccountBillingState,
   getDb,
   getInboxMessage,
   getJob,
@@ -69,6 +72,7 @@ import {
   listKpoJobs,
   listRequestEvents,
   listUsers,
+  listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
   listWorkflowRuns,
   listWorkflowSchedules,
@@ -93,6 +97,7 @@ import {
   rotateApiKey,
   rotateRefresh,
   setConcurrencySubId,
+  setConnectedAccountBillingState,
   setDbForTesting,
   setExtraConcurrencySlots,
   setMemoryPlan,
@@ -108,8 +113,9 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-62DQAWPF.js";
+} from "./chunk-CSCD2HNS.js";
 export {
+  CORE_SCHEMA_VERSION,
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
@@ -135,6 +141,7 @@ export {
   createWorkflowRun,
   createWorkflowSchedule,
   creditMc,
+  creditMcIdempotent,
   deactivateUser,
   debitMc,
   debitMcIdempotent,
@@ -147,6 +154,7 @@ export {
   generateApiKey,
   getBillingLedgerSummary,
   getClient,
+  getConnectedAccountBillingState,
   getDb,
   getInboxMessage,
   getJob,
@@ -180,6 +188,7 @@ export {
   listKpoJobs,
   listRequestEvents,
   listUsers,
+  listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
   listWorkflowRuns,
   listWorkflowSchedules,
@@ -204,6 +213,7 @@ export {
   rotateApiKey,
   rotateRefresh,
   setConcurrencySubId,
+  setConnectedAccountBillingState,
   setDbForTesting,
   setExtraConcurrencySlots,
   setMemoryPlan,
@@ -220,4 +230,4 @@ export {
   upsertPageSnapshot,
   verifyPassword
 };
-//# sourceMappingURL=db-YAI5AQOI.js.map
+//# sourceMappingURL=db-N3YECFWR.js.map