mcp-scraper 0.34.1 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +10 -9
  2. package/dist/bin/api-server.cjs +24342 -17931
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +51 -7
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +48 -5
  8. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +2 -2
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +2997 -2070
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +8 -8
  15. package/dist/bin/paa-harvest.cjs +125 -70
  16. package/dist/bin/paa-harvest.cjs.map +1 -1
  17. package/dist/bin/paa-harvest.js +4 -4
  18. package/dist/chunk-345BQXZH.js +712 -0
  19. package/dist/chunk-345BQXZH.js.map +1 -0
  20. package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
  21. package/dist/chunk-44HZLHDV.js.map +1 -0
  22. package/dist/chunk-4HO66323.js +7 -0
  23. package/dist/chunk-4HO66323.js.map +1 -0
  24. package/dist/{chunk-BWXLTWF7.js → chunk-4ZIJ3BKZ.js} +6 -4
  25. package/dist/chunk-4ZIJ3BKZ.js.map +1 -0
  26. package/dist/{chunk-6OPHG76G.js → chunk-5RULXBJ7.js} +3166 -2322
  27. package/dist/chunk-5RULXBJ7.js.map +1 -0
  28. package/dist/chunk-AN3VQARU.js +684 -0
  29. package/dist/chunk-AN3VQARU.js.map +1 -0
  30. package/dist/{chunk-XVVNKASZ.js → chunk-ANCGXUQJ.js} +118 -73
  31. package/dist/chunk-ANCGXUQJ.js.map +1 -0
  32. package/dist/{chunk-3HBPKR5G.js → chunk-D7LM5QZN.js} +3 -3
  33. package/dist/{chunk-4ZB3X6BQ.js → chunk-E5UEELA7.js} +16 -2
  34. package/dist/{chunk-4ZB3X6BQ.js.map → chunk-E5UEELA7.js.map} +1 -1
  35. package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
  36. package/dist/chunk-FQI5PFE7.js.map +1 -0
  37. package/dist/chunk-G3P3ZDB4.js +69 -0
  38. package/dist/chunk-G3P3ZDB4.js.map +1 -0
  39. package/dist/{chunk-7AYRWAEK.js → chunk-G7KAVJ3F.js} +2 -2
  40. package/dist/{chunk-7AYRWAEK.js.map → chunk-G7KAVJ3F.js.map} +1 -1
  41. package/dist/{chunk-62DQAWPF.js → chunk-IFYER7O4.js} +367 -42
  42. package/dist/chunk-IFYER7O4.js.map +1 -0
  43. package/dist/chunk-O2MCWFXQ.js +499 -0
  44. package/dist/chunk-O2MCWFXQ.js.map +1 -0
  45. package/dist/{chunk-JWIE5NCR.js → chunk-QZXKQB7Y.js} +140 -10
  46. package/dist/chunk-QZXKQB7Y.js.map +1 -0
  47. package/dist/{db-YAI5AQOI.js → db-N6MPVMEF.js} +8 -2
  48. package/dist/extract-bundle-M4SDJG3V.js +568 -0
  49. package/dist/extract-bundle-M4SDJG3V.js.map +1 -0
  50. package/dist/index.cjs +129 -70
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +11 -0
  53. package/dist/index.d.ts +11 -0
  54. package/dist/index.js +4 -4
  55. package/dist/location-data-repository-Z4NQOU5Y.js +35 -0
  56. package/dist/{server-IWDHTES2.js → server-ZIGAFKOL.js} +10844 -7480
  57. package/dist/server-ZIGAFKOL.js.map +1 -0
  58. package/dist/site-extract-repository-PGQZNW6V.js +62 -0
  59. package/dist/site-extract-repository-PGQZNW6V.js.map +1 -0
  60. package/dist/{worker-645BZPEK.js → worker-EBB6CTGW.js} +7 -7
  61. package/docs/hosted-location-data.md +108 -0
  62. package/docs/mcp-tool-craft-lint.generated.md +6 -3
  63. package/docs/mcp-tool-manifest.generated.json +2073 -551
  64. package/docs/mcp-tool-quality-spec.md +1 -1
  65. package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
  66. package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
  67. package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
  68. package/docs/specs/query-fanout-transport-contract-fix.md +9 -1
  69. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
  70. package/package.json +6 -5
  71. package/dist/chunk-5PZ6N2QM.js +0 -130
  72. package/dist/chunk-5PZ6N2QM.js.map +0 -1
  73. package/dist/chunk-62DQAWPF.js.map +0 -1
  74. package/dist/chunk-6OPHG76G.js.map +0 -1
  75. package/dist/chunk-7N2KYL4U.js +0 -7
  76. package/dist/chunk-7N2KYL4U.js.map +0 -1
  77. package/dist/chunk-BWXLTWF7.js.map +0 -1
  78. package/dist/chunk-JWIE5NCR.js.map +0 -1
  79. package/dist/chunk-M2S27J6Z.js.map +0 -1
  80. package/dist/chunk-R7EETU7Z.js +0 -419
  81. package/dist/chunk-R7EETU7Z.js.map +0 -1
  82. package/dist/chunk-XVVNKASZ.js.map +0 -1
  83. package/dist/chunk-ZID3WQID.js.map +0 -1
  84. package/dist/extract-bundle-GPZRLBVY.js +0 -331
  85. package/dist/extract-bundle-GPZRLBVY.js.map +0 -1
  86. package/dist/server-IWDHTES2.js.map +0 -1
  87. package/dist/site-extract-repository-OLVWMOU2.js +0 -30
  88. /package/dist/{chunk-3HBPKR5G.js.map → chunk-D7LM5QZN.js.map} +0 -0
  89. /package/dist/{db-YAI5AQOI.js.map → db-N6MPVMEF.js.map} +0 -0
  90. /package/dist/{site-extract-repository-OLVWMOU2.js.map → location-data-repository-Z4NQOU5Y.js.map} +0 -0
  91. /package/dist/{worker-645BZPEK.js.map → worker-EBB6CTGW.js.map} +0 -0
@@ -1,9 +1,15 @@
1
+ import {
2
+ createPrivateArtifactFromStream,
3
+ renewPrivateArtifactDownload,
4
+ validatePublicHttpUrl
5
+ } from "./chunk-345BQXZH.js";
6
+
1
7
  // src/lib/media-extractor.ts
2
- import { createWriteStream, mkdirSync } from "fs";
8
+ import { createWriteStream, mkdirSync, rmSync } from "fs";
3
9
  import { homedir } from "os";
4
10
  import { join, extname, basename } from "path";
5
11
  import { pipeline } from "stream/promises";
6
- import { Readable } from "stream";
12
+ import { Readable, Transform } from "stream";
7
13
  var AD_PATTERNS = [
8
14
  "doubleclick.net",
9
15
  "googlesyndication.com",
@@ -183,14 +189,38 @@ function extractMediaUrls(html, baseUrl) {
183
189
  }
184
190
  return urls;
185
191
  }
186
- async function downloadAsset(url, destDir, filename) {
187
- const res = await fetch(url, {
188
- headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
189
- signal: AbortSignal.timeout(15e3)
190
- });
192
+ async function downloadAsset(url, destDir, filename, options = {}) {
193
+ const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024));
194
+ let target = url;
195
+ let res = null;
196
+ for (let redirects = 0; redirects <= 5; redirects++) {
197
+ const checked = await validatePublicHttpUrl(target, { field: "media URL" });
198
+ if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Media URL was rejected");
199
+ res = await fetch(checked.parsed.href, {
200
+ headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
201
+ signal: AbortSignal.timeout(15e3),
202
+ redirect: "manual"
203
+ });
204
+ if (res.status >= 300 && res.status < 400) {
205
+ const location = res.headers.get("location");
206
+ if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`);
207
+ target = new URL(location, checked.parsed.href).href;
208
+ res = null;
209
+ continue;
210
+ }
211
+ break;
212
+ }
213
+ if (!res) throw new Error("Media download exceeded five redirects");
191
214
  if (!res.ok) throw new Error(`HTTP ${res.status}`);
192
215
  if (!res.body) throw new Error("Empty response body");
193
216
  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
217
+ if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {
218
+ throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? "no content-type"}`);
219
+ }
220
+ const declaredBytes = Number(res.headers.get("content-length") ?? 0);
221
+ if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
222
+ throw new Error(`Media asset exceeds ${maxBytes} byte limit`);
223
+ }
194
224
  let dest = join(destDir, filename);
195
225
  if (mimeType && !extname(filename)) {
196
226
  const mimeExt = {
@@ -210,7 +240,33 @@ async function downloadAsset(url, destDir, filename) {
210
240
  if (ext) dest = dest + ext;
211
241
  }
212
242
  const writer = createWriteStream(dest);
213
- await pipeline(Readable.fromWeb(res.body), writer);
243
+ await new Promise((resolve, reject) => {
244
+ writer.once("open", () => resolve());
245
+ writer.once("error", reject);
246
+ });
247
+ let streamedBytes = 0;
248
+ const limiter = new Transform({
249
+ transform(chunk, _encoding, callback) {
250
+ const bytes = chunk.length;
251
+ if (streamedBytes + bytes > maxBytes) {
252
+ callback(new Error(`Media asset exceeds ${maxBytes} byte limit`));
253
+ return;
254
+ }
255
+ if (options.consumeBytes && !options.consumeBytes(bytes)) {
256
+ callback(new Error("Media export total byte limit exceeded"));
257
+ return;
258
+ }
259
+ streamedBytes += bytes;
260
+ callback(null, chunk);
261
+ }
262
+ });
263
+ try {
264
+ await pipeline(Readable.fromWeb(res.body), limiter, writer);
265
+ } catch (error) {
266
+ writer.destroy();
267
+ rmSync(dest, { force: true });
268
+ throw error;
269
+ }
214
270
  const { statSync } = await import("fs");
215
271
  const sizeBytes = statSync(dest).size;
216
272
  return { savedPath: dest, sizeBytes, mimeType };
@@ -277,8 +333,82 @@ async function harvestPageMedia(html, pageUrl, options = {}) {
277
333
  return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
278
334
  }
279
335
 
336
+ // src/api/site-extract-artifacts.ts
337
+ var SITE_EXTRACT_ARTIFACT_PREFIX = "site-extracts/";
338
+ var SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
339
+ var SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
340
+ function siteExtractArtifactToken() {
341
+ return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
342
+ }
343
+ function hostedByEnvironment() {
344
+ return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
345
+ }
346
+ function policy() {
347
+ return {
348
+ prefix: SITE_EXTRACT_ARTIFACT_PREFIX,
349
+ artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,
350
+ downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,
351
+ token: siteExtractArtifactToken()
352
+ };
353
+ }
354
+ async function createSiteExtractBundleArtifactStream(args) {
355
+ const pointer = await createPrivateArtifactFromStream({
356
+ policy: policy(),
357
+ ownerId: args.ownerId,
358
+ artifactKey: `${args.jobId}.zip`,
359
+ createdAt: args.createdAt,
360
+ filename: `${args.jobId}-site-export.zip`,
361
+ contentType: "application/zip",
362
+ content: args.content
363
+ });
364
+ return {
365
+ key: pointer.artifactId,
366
+ url: pointer.downloadUrl ?? "",
367
+ bytes: pointer.bytes,
368
+ contentType: pointer.contentType,
369
+ filename: pointer.filename,
370
+ sha256: pointer.sha256,
371
+ expiresAt: pointer.expiresAt,
372
+ downloadUrlExpiresAt: pointer.downloadUrlExpiresAt
373
+ };
374
+ }
375
+ async function renewSiteExtractArtifactDownload(args) {
376
+ return renewPrivateArtifactDownload({
377
+ policy: policy(),
378
+ artifactId: args.artifactId,
379
+ ownerId: args.ownerId
380
+ });
381
+ }
382
+ async function cleanupExpiredSiteExtractArtifacts(args = {}) {
383
+ const now = args.now ?? /* @__PURE__ */ new Date();
384
+ const token = siteExtractArtifactToken();
385
+ if (!token) return { deleted: 0, store: hostedByEnvironment() ? "none" : "local" };
386
+ if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {
387
+ return { deleted: 0, store: "private-vercel-blob", skipped: true };
388
+ }
389
+ const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS;
390
+ const { list, del } = await import("@vercel/blob");
391
+ let cursor;
392
+ let deleted = 0;
393
+ for (let page = 0; page < 20; page += 1) {
394
+ const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
395
+ const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
396
+ if (expired.length > 0) {
397
+ await del(expired.map((blob) => blob.pathname), { token });
398
+ deleted += expired.length;
399
+ }
400
+ if (!result.hasMore || !result.cursor) break;
401
+ cursor = result.cursor;
402
+ }
403
+ return { deleted, store: "private-vercel-blob" };
404
+ }
405
+
280
406
  export {
281
407
  downloadAsset,
282
- harvestPageMedia
408
+ harvestPageMedia,
409
+ SITE_EXTRACT_ARTIFACT_PREFIX,
410
+ createSiteExtractBundleArtifactStream,
411
+ renewSiteExtractArtifactDownload,
412
+ cleanupExpiredSiteExtractArtifacts
283
413
  };
284
- //# sourceMappingURL=chunk-JWIE5NCR.js.map
414
+ //# sourceMappingURL=chunk-QZXKQB7Y.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/site-extract-artifacts.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nfunction typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nfunction typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n const seen = new Set<string>()\n const urls: string[] = []\n\n const add = (raw: string) => {\n const resolved = resolveUrl(raw.trim(), baseUrl)\n if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return\n seen.add(resolved)\n urls.push(resolved)\n }\n\n for (const m of html.matchAll(/<img\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<source\\s[^>]*>/gi)) {\n const tag = m[0]\n const src = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (src) add(src)\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<video\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'poster']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n }\n\n for (const m of html.matchAll(/<audio\\s[^>]*>/gi)) {\n const tag = m[0]\n const v = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (v) add(v)\n }\n\n for (const m of html.matchAll(/<meta\\s[^>]*>/gi)) {\n const tag = m[0]\n const prop = (tag.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]?.toLowerCase()\n if (prop === 'og:image' || prop === 'twitter:image') {\n const content = (tag.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (content) add(content)\n }\n }\n\n for (const m of html.matchAll(/background(?:-image)?\\s*:\\s*url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)/gi)) {\n add(m[1])\n }\n\n return urls\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n options: {\n expectedType?: MediaType\n maxBytes?: number\n consumeBytes?: (bytes: number) => boolean\n } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n let target = url\n let res: Response | null = null\n for (let redirects = 0; redirects <= 5; redirects++) {\n const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n res = await fetch(checked.parsed.href, {\n headers: { 'User-Agent': 'Mozilla/5.0 (compatible; ThorbitBot/1.0)' },\n signal: AbortSignal.timeout(15_000),\n redirect: 'manual',\n })\n if (res.status >= 300 && res.status < 400) {\n const location = res.headers.get('location')\n if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n target = new URL(location, checked.parsed.href).href\n res = null\n continue\n }\n break\n }\n if (!res) throw new Error('Media download exceeded five redirects')\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n }\n const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n }\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await new Promise<void>((resolve, reject) => {\n writer.once('open', () => resolve())\n writer.once('error', reject)\n })\n let streamedBytes = 0\n const limiter = new Transform({\n transform(chunk: Buffer, _encoding, callback) {\n const bytes = chunk.length\n if (streamedBytes + bytes > maxBytes) {\n callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n return\n }\n if (options.consumeBytes && !options.consumeBytes(bytes)) {\n callback(new Error('Media export total byte limit exceeded'))\n return\n }\n streamedBytes += bytes\n callback(null, chunk)\n },\n })\n try {\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n } catch (error) {\n writer.destroy()\n rmSync(dest, { force: true })\n throw error\n }\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const rawUrls = extractMediaUrls(html, pageUrl)\n const totalFound = rawUrls.length\n\n const filteredUrls: string[] = []\n const kept: Array<{ url: string; type: MediaType }> = []\n\n for (const url of rawUrls) {\n const type = typeFromUrl(url)\n if (!type || !typesSet.has(type)) { filteredUrls.push(url); continue }\n kept.push({ url, type })\n }\n\n if (options.outputDir === null) {\n return {\n pageUrl,\n outputDir: null,\n assets: kept.map(({ url, type }, i) => ({\n url, type, mimeType: null, filename: safeFilename(url, i), savedPath: null, sizeBytes: null,\n })),\n filteredCount: filteredUrls.length,\n totalFound,\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async ({ url, type }, i) => {\n const filename = safeFilename(url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? type) : type\n assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes })\n } catch {\n assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound }\n}\n","import {\n createPrivateArtifact,\n createPrivateArtifactFromStream,\n privateArtifactOwnerId,\n renewPrivateArtifactDownload,\n type PrivateArtifactPolicy,\n} from './private-artifacts.js'\nimport type { ExtractJobArtifact } from './site-extract-repository.js'\nimport type { Readable } from 'node:stream'\n\nexport const SITE_EXTRACT_ARTIFACT_PREFIX = 'site-extracts/'\nexport const SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1_000\nexport const SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1_000\n\nfunction siteExtractArtifactToken(): string | null {\n return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()\n || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()\n || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()\n || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()\n || null\n}\n\nfunction hostedByEnvironment(): boolean {\n return process.env.VERCEL === '1' || process.env.NODE_ENV === 'production'\n}\n\nfunction policy(): PrivateArtifactPolicy {\n return {\n prefix: SITE_EXTRACT_ARTIFACT_PREFIX,\n artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,\n downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,\n token: siteExtractArtifactToken(),\n }\n}\n\nexport function siteExtractArtifactOwnerId(artifactId: string): string | null {\n return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX)\n}\n\nexport async function createSiteExtractBundleArtifact(args: {\n ownerId: string\n jobId: string\n createdAt: Date | string | number\n content: Buffer\n}): Promise<ExtractJobArtifact> {\n const pointer = await createPrivateArtifact({\n policy: policy(),\n ownerId: args.ownerId,\n artifactKey: `${args.jobId}.zip`,\n createdAt: args.createdAt,\n filename: `${args.jobId}-site-export.zip`,\n contentType: 'application/zip',\n content: args.content,\n })\n return {\n key: pointer.artifactId,\n url: pointer.downloadUrl ?? '',\n bytes: pointer.bytes,\n contentType: pointer.contentType,\n filename: pointer.filename,\n sha256: pointer.sha256,\n expiresAt: pointer.expiresAt,\n downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n }\n}\n\nexport async function createSiteExtractBundleArtifactStream(args: {\n ownerId: string\n jobId: string\n createdAt: Date | string | number\n content: Readable\n}): Promise<ExtractJobArtifact> {\n const pointer = await createPrivateArtifactFromStream({\n policy: policy(),\n ownerId: args.ownerId,\n artifactKey: `${args.jobId}.zip`,\n createdAt: args.createdAt,\n filename: `${args.jobId}-site-export.zip`,\n contentType: 'application/zip',\n content: args.content,\n })\n return {\n key: pointer.artifactId,\n url: pointer.downloadUrl ?? '',\n bytes: pointer.bytes,\n contentType: pointer.contentType,\n filename: pointer.filename,\n sha256: pointer.sha256,\n expiresAt: pointer.expiresAt,\n downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n }\n}\n\nexport async function renewSiteExtractArtifactDownload(args: {\n artifactId: string\n ownerId: string\n}): Promise<{ downloadUrl: string; downloadUrlExpiresAt: string; expiresAt: string } | null> {\n return renewPrivateArtifactDownload({\n policy: policy(),\n artifactId: args.artifactId,\n ownerId: args.ownerId,\n })\n}\n\nexport async function cleanupExpiredSiteExtractArtifacts(args: {\n now?: Date\n force?: boolean\n} = {}): Promise<{ deleted: number; store: 'private-vercel-blob' | 'local' | 'none'; skipped?: boolean }> {\n const now = args.now ?? new Date()\n const token = siteExtractArtifactToken()\n if (!token) return { deleted: 0, store: hostedByEnvironment() ? 'none' : 'local' }\n if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {\n return { deleted: 0, store: 'private-vercel-blob', skipped: true }\n }\n const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS\n const { list, del } = await import('@vercel/blob')\n let cursor: string | undefined\n let deleted = 0\n for (let page = 0; page < 20; page += 1) {\n const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1_000, cursor })\n const expired = result.blobs.filter(blob => new Date(blob.uploadedAt).getTime() <= cutoff)\n if (expired.length > 0) {\n await del(expired.map(blob => blob.pathname), { token })\n deleted += expired.length\n }\n if (!result.hasMore || !result.cursor) break\n cursor = result.cursor\n }\n return { deleted, store: 'private-vercel-blob' }\n}\n"],"mappings":";;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAGpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AA0BtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEA,SAAS,YAAY,KAA+B;AAClD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,iBAAiB,MAAc,SAA2B;AACxE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,OAAiB,CAAC;AAExB,QAAM,MAAM,CAAC,QAAgB;AAC3B,UAAM,WAAW,WAAW,IAAI,KAAK,GAAG,OAAO;AAC/C,QAAI,CAAC,YAAY,KAAK,IAAI,QAAQ,KAAK,QAAQ,QAAQ,EAAG;AAC1D,SAAK,IAAI,QAAQ;AACjB,SAAK,KAAK,QAAQ;AAAA,EACpB;AAEA,aAAW,KAAK,KAAK,SAAS,gBAAgB,GAAG;AAC/C,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,eAAe,GAAG;AACxE,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AACA,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,mBAAmB,GAAG;AAClD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,OAAO,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAChE,QAAI,IAAK,KAAI,GAAG;AAChB,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,QAAQ,GAAG;AACpC,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,KAAK,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAC9D,QAAI,EAAG,KAAI,CAAC;AAAA,EACd;AAEA,aAAW,KAAK,KAAK,SAAS,iBAAiB,GAAG;AAChD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,QAAQ,IAAI,MAAM,6CAA6C,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY;AAC9F,QAAI,SAAS,cAAc,SAAS,iBAAiB;AACnD,YAAM,WAAW,IAAI,MAAM,mCAAmC,KAAK,CAAC,GAAG,CAAC;AACxE,UAAI,QAAS,KAAI,OAAO;AAAA,IAC1B;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,gEAAgE,GAAG;AAC/F,QAAI,EAAE,CAAC,CAAC;AAAA,EACV;AAEA,SAAO;AACT;AAEA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,UAAW,iBAAiB,MAAM,OAAO;AAC/C,QAAM,aAAa,QAAQ;AAE3B,QAAM,eAAyB,CAAC;AAChC,QAAM,OAAgD,CAAC;AAEvD,aAAW,OAAO,SAAS;AACzB,UAAM,OAAO,YAAY,GAAG;AAC5B,QAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,GAAG;AAAE,mBAAa,KAAK,GAAG;AAAG;AAAA,IAAS;AACrE,SAAK,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,EACzB;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,EAAE,KAAK,KAAK,GAAG,OAAO;AAAA,QACtC;AAAA,QAAK;AAAA,QAAM,UAAU;AAAA,QAAM,UAAU,aAAa,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,MACzF,EAAE;AAAA,MACF,eAAe,aAAa;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,GAAG,MAAM;AACnC,YAAM,WAAW,aAAa,KAAK,CAAC;AACpC,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,KAAK,QAAQ,QAAQ;AACpF,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,OAAQ;AACnE,eAAO,KAAK,EAAE,KAAK,MAAM,cAAc,UAAU,UAAU,SAAS,SAAS,GAAG,WAAW,UAAU,CAAC;AAAA,MACxG,QAAQ;AACN,eAAO,KAAK,EAAE,KAAK,MAAM,UAAU,MAAM,UAAU,WAAW,MAAM,WAAW,KAAK,CAAC;AAAA,MACvF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,eAAe,aAAa,QAAQ,WAAW;AAC9F;;;ACnTO,IAAM,+BAA+B;AACrC,IAAM,+BAA+B,IAAI,KAAK,KAAK,KAAK;AACxD,IAAM,+BAA+B,KAAK,KAAK;AAEtD,SAAS,2BAA0C;AACjD,SAAO,QAAQ,IAAI,wCAAwC,KAAK,KAC3D,QAAQ,IAAI,mCAAmC,KAAK,KACpD,QAAQ,IAAI,iCAAiC,KAAK,KAClD,QAAQ,IAAI,sCAAsC,KAAK,KACvD;AACP;AAEA,SAAS,sBAA+B;AACtC,SAAO,QAAQ,IAAI,WAAW,OAAO,QAAQ,IAAI,aAAa;AAChE;AAEA,SAAS,SAAgC;AACvC,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,eAAe;AAAA,IACf,OAAO,yBAAyB;AAAA,EAClC;AACF;AAiCA,eAAsB,sCAAsC,MAK5B;AAC9B,QAAM,UAAU,MAAM,gCAAgC;AAAA,IACpD,QAAQ,OAAO;AAAA,IACf,SAAS,KAAK;AAAA,IACd,aAAa,GAAG,KAAK,KAAK;AAAA,IAC1B,WAAW,KAAK;AAAA,IAChB,UAAU,GAAG,KAAK,KAAK;AAAA,IACvB,aAAa;AAAA,IACb,SAAS,KAAK;AAAA,EAChB,CAAC;AACD,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ,eAAe;AAAA,IAC5B,OAAO,QAAQ;AAAA,IACf,aAAa,QAAQ;AAAA,IACrB,UAAU,QAAQ;AAAA,IAClB,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB,sBAAsB,QAAQ;AAAA,EAChC;AACF;AAEA,eAAsB,iCAAiC,MAGsC;AAC3F,SAAO,6BAA6B;AAAA,IAClC,QAAQ,OAAO;AAAA,IACf,YAAY,KAAK;AAAA,IACjB,SAAS,KAAK;AAAA,EAChB,CAAC;AACH;AAEA,eAAsB,mCAAmC,OAGrD,CAAC,GAAqG;AACxG,QAAM,MAAM,KAAK,OAAO,oBAAI,KAAK;AACjC,QAAM,QAAQ,yBAAyB;AACvC,MAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,OAAO,oBAAoB,IAAI,SAAS,QAAQ;AACjF,MAAI,CAAC,KAAK,SAAS,EAAE,IAAI,YAAY,MAAM,KAAK,IAAI,cAAc,MAAM,KAAK;AAC3E,WAAO,EAAE,SAAS,GAAG,OAAO,uBAAuB,SAAS,KAAK;AAAA,EACnE;AACA,QAAM,SAAS,IAAI,QAAQ,IAAI;AAC/B,QAAM,EAAE,MAAM,IAAI,IAAI,MAAM,OAAO,cAAc;AACjD,MAAI;AACJ,MAAI,UAAU;AACd,WAAS,OAAO,GAAG,OAAO,IAAI,QAAQ,GAAG;AACvC,UAAM,SAAS,MAAM,KAAK,EAAE,QAAQ,8BAA8B,OAAO,OAAO,KAAO,OAAO,CAAC;AAC/F,UAAM,UAAU,OAAO,MAAM,OAAO,UAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,QAAQ,KAAK,MAAM;AACzF,QAAI,QAAQ,SAAS,GAAG;AACtB,YAAM,IAAI,QAAQ,IAAI,UAAQ,KAAK,QAAQ,GAAG,EAAE,MAAM,CAAC;AACvD,iBAAW,QAAQ;AAAA,IACrB;AACA,QAAI,CAAC,OAAO,WAAW,CAAC,OAAO,OAAQ;AACvC,aAAS,OAAO;AAAA,EAClB;AACA,SAAO,EAAE,SAAS,OAAO,sBAAsB;AACjD;","names":[]}
@@ -36,6 +36,7 @@ import {
36
36
  generateApiKey,
37
37
  getBillingLedgerSummary,
38
38
  getClient,
39
+ getConnectedAccountBillingState,
39
40
  getDb,
40
41
  getInboxMessage,
41
42
  getJob,
@@ -69,6 +70,7 @@ import {
69
70
  listKpoJobs,
70
71
  listRequestEvents,
71
72
  listUsers,
73
+ listUsersDueForConnectedAccountBillingReconciliation,
72
74
  listWorkflowArtifacts,
73
75
  listWorkflowRuns,
74
76
  listWorkflowSchedules,
@@ -93,6 +95,7 @@ import {
93
95
  rotateApiKey,
94
96
  rotateRefresh,
95
97
  setConcurrencySubId,
98
+ setConnectedAccountBillingState,
96
99
  setDbForTesting,
97
100
  setExtraConcurrencySlots,
98
101
  setMemoryPlan,
@@ -108,7 +111,7 @@ import {
108
111
  updateKpoJobState,
109
112
  upsertPageSnapshot,
110
113
  verifyPassword
111
- } from "./chunk-62DQAWPF.js";
114
+ } from "./chunk-IFYER7O4.js";
112
115
  export {
113
116
  CREDIT_LOT_TTL,
114
117
  SiteAuditJobRowSchema,
@@ -147,6 +150,7 @@ export {
147
150
  generateApiKey,
148
151
  getBillingLedgerSummary,
149
152
  getClient,
153
+ getConnectedAccountBillingState,
150
154
  getDb,
151
155
  getInboxMessage,
152
156
  getJob,
@@ -180,6 +184,7 @@ export {
180
184
  listKpoJobs,
181
185
  listRequestEvents,
182
186
  listUsers,
187
+ listUsersDueForConnectedAccountBillingReconciliation,
183
188
  listWorkflowArtifacts,
184
189
  listWorkflowRuns,
185
190
  listWorkflowSchedules,
@@ -204,6 +209,7 @@ export {
204
209
  rotateApiKey,
205
210
  rotateRefresh,
206
211
  setConcurrencySubId,
212
+ setConnectedAccountBillingState,
207
213
  setDbForTesting,
208
214
  setExtraConcurrencySlots,
209
215
  setMemoryPlan,
@@ -220,4 +226,4 @@ export {
220
226
  upsertPageSnapshot,
221
227
  verifyPassword
222
228
  };
223
- //# sourceMappingURL=db-YAI5AQOI.js.map
229
+ //# sourceMappingURL=db-N6MPVMEF.js.map