mcp-scraper 0.34.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/bin/api-server.cjs +24342 -17931
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +51 -7
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +48 -5
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2997 -2070
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +8 -8
- package/dist/bin/paa-harvest.cjs +125 -70
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +4 -4
- package/dist/chunk-345BQXZH.js +712 -0
- package/dist/chunk-345BQXZH.js.map +1 -0
- package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
- package/dist/chunk-44HZLHDV.js.map +1 -0
- package/dist/chunk-4HO66323.js +7 -0
- package/dist/chunk-4HO66323.js.map +1 -0
- package/dist/{chunk-BWXLTWF7.js → chunk-4ZIJ3BKZ.js} +6 -4
- package/dist/chunk-4ZIJ3BKZ.js.map +1 -0
- package/dist/{chunk-6OPHG76G.js → chunk-5RULXBJ7.js} +3166 -2322
- package/dist/chunk-5RULXBJ7.js.map +1 -0
- package/dist/chunk-AN3VQARU.js +684 -0
- package/dist/chunk-AN3VQARU.js.map +1 -0
- package/dist/{chunk-XVVNKASZ.js → chunk-ANCGXUQJ.js} +118 -73
- package/dist/chunk-ANCGXUQJ.js.map +1 -0
- package/dist/{chunk-3HBPKR5G.js → chunk-D7LM5QZN.js} +3 -3
- package/dist/{chunk-4ZB3X6BQ.js → chunk-E5UEELA7.js} +16 -2
- package/dist/{chunk-4ZB3X6BQ.js.map → chunk-E5UEELA7.js.map} +1 -1
- package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
- package/dist/chunk-FQI5PFE7.js.map +1 -0
- package/dist/chunk-G3P3ZDB4.js +69 -0
- package/dist/chunk-G3P3ZDB4.js.map +1 -0
- package/dist/{chunk-7AYRWAEK.js → chunk-G7KAVJ3F.js} +2 -2
- package/dist/{chunk-7AYRWAEK.js.map → chunk-G7KAVJ3F.js.map} +1 -1
- package/dist/{chunk-62DQAWPF.js → chunk-IFYER7O4.js} +367 -42
- package/dist/chunk-IFYER7O4.js.map +1 -0
- package/dist/chunk-O2MCWFXQ.js +499 -0
- package/dist/chunk-O2MCWFXQ.js.map +1 -0
- package/dist/{chunk-JWIE5NCR.js → chunk-QZXKQB7Y.js} +140 -10
- package/dist/chunk-QZXKQB7Y.js.map +1 -0
- package/dist/{db-YAI5AQOI.js → db-N6MPVMEF.js} +8 -2
- package/dist/extract-bundle-M4SDJG3V.js +568 -0
- package/dist/extract-bundle-M4SDJG3V.js.map +1 -0
- package/dist/index.cjs +129 -70
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +4 -4
- package/dist/location-data-repository-Z4NQOU5Y.js +35 -0
- package/dist/{server-IWDHTES2.js → server-ZIGAFKOL.js} +10844 -7480
- package/dist/server-ZIGAFKOL.js.map +1 -0
- package/dist/site-extract-repository-PGQZNW6V.js +62 -0
- package/dist/site-extract-repository-PGQZNW6V.js.map +1 -0
- package/dist/{worker-645BZPEK.js → worker-EBB6CTGW.js} +7 -7
- package/docs/hosted-location-data.md +108 -0
- package/docs/mcp-tool-craft-lint.generated.md +6 -3
- package/docs/mcp-tool-manifest.generated.json +2073 -551
- package/docs/mcp-tool-quality-spec.md +1 -1
- package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
- package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
- package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
- package/docs/specs/query-fanout-transport-contract-fix.md +9 -1
- package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
- package/package.json +6 -5
- package/dist/chunk-5PZ6N2QM.js +0 -130
- package/dist/chunk-5PZ6N2QM.js.map +0 -1
- package/dist/chunk-62DQAWPF.js.map +0 -1
- package/dist/chunk-6OPHG76G.js.map +0 -1
- package/dist/chunk-7N2KYL4U.js +0 -7
- package/dist/chunk-7N2KYL4U.js.map +0 -1
- package/dist/chunk-BWXLTWF7.js.map +0 -1
- package/dist/chunk-JWIE5NCR.js.map +0 -1
- package/dist/chunk-M2S27J6Z.js.map +0 -1
- package/dist/chunk-R7EETU7Z.js +0 -419
- package/dist/chunk-R7EETU7Z.js.map +0 -1
- package/dist/chunk-XVVNKASZ.js.map +0 -1
- package/dist/chunk-ZID3WQID.js.map +0 -1
- package/dist/extract-bundle-GPZRLBVY.js +0 -331
- package/dist/extract-bundle-GPZRLBVY.js.map +0 -1
- package/dist/server-IWDHTES2.js.map +0 -1
- package/dist/site-extract-repository-OLVWMOU2.js +0 -30
- /package/dist/{chunk-3HBPKR5G.js.map → chunk-D7LM5QZN.js.map} +0 -0
- /package/dist/{db-YAI5AQOI.js.map → db-N6MPVMEF.js.map} +0 -0
- /package/dist/{site-extract-repository-OLVWMOU2.js.map → location-data-repository-Z4NQOU5Y.js.map} +0 -0
- /package/dist/{worker-645BZPEK.js.map → worker-EBB6CTGW.js.map} +0 -0
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createPrivateArtifactFromStream,
|
|
3
|
+
renewPrivateArtifactDownload,
|
|
4
|
+
validatePublicHttpUrl
|
|
5
|
+
} from "./chunk-345BQXZH.js";
|
|
6
|
+
|
|
1
7
|
// src/lib/media-extractor.ts
|
|
2
|
-
import { createWriteStream, mkdirSync } from "fs";
|
|
8
|
+
import { createWriteStream, mkdirSync, rmSync } from "fs";
|
|
3
9
|
import { homedir } from "os";
|
|
4
10
|
import { join, extname, basename } from "path";
|
|
5
11
|
import { pipeline } from "stream/promises";
|
|
6
|
-
import { Readable } from "stream";
|
|
12
|
+
import { Readable, Transform } from "stream";
|
|
7
13
|
var AD_PATTERNS = [
|
|
8
14
|
"doubleclick.net",
|
|
9
15
|
"googlesyndication.com",
|
|
@@ -183,14 +189,38 @@ function extractMediaUrls(html, baseUrl) {
|
|
|
183
189
|
}
|
|
184
190
|
return urls;
|
|
185
191
|
}
|
|
186
|
-
async function downloadAsset(url, destDir, filename) {
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
192
|
+
async function downloadAsset(url, destDir, filename, options = {}) {
|
|
193
|
+
const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024));
|
|
194
|
+
let target = url;
|
|
195
|
+
let res = null;
|
|
196
|
+
for (let redirects = 0; redirects <= 5; redirects++) {
|
|
197
|
+
const checked = await validatePublicHttpUrl(target, { field: "media URL" });
|
|
198
|
+
if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Media URL was rejected");
|
|
199
|
+
res = await fetch(checked.parsed.href, {
|
|
200
|
+
headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
|
|
201
|
+
signal: AbortSignal.timeout(15e3),
|
|
202
|
+
redirect: "manual"
|
|
203
|
+
});
|
|
204
|
+
if (res.status >= 300 && res.status < 400) {
|
|
205
|
+
const location = res.headers.get("location");
|
|
206
|
+
if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`);
|
|
207
|
+
target = new URL(location, checked.parsed.href).href;
|
|
208
|
+
res = null;
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
if (!res) throw new Error("Media download exceeded five redirects");
|
|
191
214
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
192
215
|
if (!res.body) throw new Error("Empty response body");
|
|
193
216
|
const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
|
|
217
|
+
if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {
|
|
218
|
+
throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? "no content-type"}`);
|
|
219
|
+
}
|
|
220
|
+
const declaredBytes = Number(res.headers.get("content-length") ?? 0);
|
|
221
|
+
if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
|
|
222
|
+
throw new Error(`Media asset exceeds ${maxBytes} byte limit`);
|
|
223
|
+
}
|
|
194
224
|
let dest = join(destDir, filename);
|
|
195
225
|
if (mimeType && !extname(filename)) {
|
|
196
226
|
const mimeExt = {
|
|
@@ -210,7 +240,33 @@ async function downloadAsset(url, destDir, filename) {
|
|
|
210
240
|
if (ext) dest = dest + ext;
|
|
211
241
|
}
|
|
212
242
|
const writer = createWriteStream(dest);
|
|
213
|
-
await
|
|
243
|
+
await new Promise((resolve, reject) => {
|
|
244
|
+
writer.once("open", () => resolve());
|
|
245
|
+
writer.once("error", reject);
|
|
246
|
+
});
|
|
247
|
+
let streamedBytes = 0;
|
|
248
|
+
const limiter = new Transform({
|
|
249
|
+
transform(chunk, _encoding, callback) {
|
|
250
|
+
const bytes = chunk.length;
|
|
251
|
+
if (streamedBytes + bytes > maxBytes) {
|
|
252
|
+
callback(new Error(`Media asset exceeds ${maxBytes} byte limit`));
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (options.consumeBytes && !options.consumeBytes(bytes)) {
|
|
256
|
+
callback(new Error("Media export total byte limit exceeded"));
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
streamedBytes += bytes;
|
|
260
|
+
callback(null, chunk);
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
try {
|
|
264
|
+
await pipeline(Readable.fromWeb(res.body), limiter, writer);
|
|
265
|
+
} catch (error) {
|
|
266
|
+
writer.destroy();
|
|
267
|
+
rmSync(dest, { force: true });
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
214
270
|
const { statSync } = await import("fs");
|
|
215
271
|
const sizeBytes = statSync(dest).size;
|
|
216
272
|
return { savedPath: dest, sizeBytes, mimeType };
|
|
@@ -277,8 +333,82 @@ async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
|
277
333
|
return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
|
|
278
334
|
}
|
|
279
335
|
|
|
336
|
+
// src/api/site-extract-artifacts.ts
|
|
337
|
+
var SITE_EXTRACT_ARTIFACT_PREFIX = "site-extracts/";
|
|
338
|
+
var SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
339
|
+
var SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
340
|
+
function siteExtractArtifactToken() {
|
|
341
|
+
return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
342
|
+
}
|
|
343
|
+
function hostedByEnvironment() {
|
|
344
|
+
return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
|
|
345
|
+
}
|
|
346
|
+
function policy() {
|
|
347
|
+
return {
|
|
348
|
+
prefix: SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
349
|
+
artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,
|
|
350
|
+
downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,
|
|
351
|
+
token: siteExtractArtifactToken()
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
async function createSiteExtractBundleArtifactStream(args) {
|
|
355
|
+
const pointer = await createPrivateArtifactFromStream({
|
|
356
|
+
policy: policy(),
|
|
357
|
+
ownerId: args.ownerId,
|
|
358
|
+
artifactKey: `${args.jobId}.zip`,
|
|
359
|
+
createdAt: args.createdAt,
|
|
360
|
+
filename: `${args.jobId}-site-export.zip`,
|
|
361
|
+
contentType: "application/zip",
|
|
362
|
+
content: args.content
|
|
363
|
+
});
|
|
364
|
+
return {
|
|
365
|
+
key: pointer.artifactId,
|
|
366
|
+
url: pointer.downloadUrl ?? "",
|
|
367
|
+
bytes: pointer.bytes,
|
|
368
|
+
contentType: pointer.contentType,
|
|
369
|
+
filename: pointer.filename,
|
|
370
|
+
sha256: pointer.sha256,
|
|
371
|
+
expiresAt: pointer.expiresAt,
|
|
372
|
+
downloadUrlExpiresAt: pointer.downloadUrlExpiresAt
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
async function renewSiteExtractArtifactDownload(args) {
|
|
376
|
+
return renewPrivateArtifactDownload({
|
|
377
|
+
policy: policy(),
|
|
378
|
+
artifactId: args.artifactId,
|
|
379
|
+
ownerId: args.ownerId
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
async function cleanupExpiredSiteExtractArtifacts(args = {}) {
|
|
383
|
+
const now = args.now ?? /* @__PURE__ */ new Date();
|
|
384
|
+
const token = siteExtractArtifactToken();
|
|
385
|
+
if (!token) return { deleted: 0, store: hostedByEnvironment() ? "none" : "local" };
|
|
386
|
+
if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {
|
|
387
|
+
return { deleted: 0, store: "private-vercel-blob", skipped: true };
|
|
388
|
+
}
|
|
389
|
+
const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS;
|
|
390
|
+
const { list, del } = await import("@vercel/blob");
|
|
391
|
+
let cursor;
|
|
392
|
+
let deleted = 0;
|
|
393
|
+
for (let page = 0; page < 20; page += 1) {
|
|
394
|
+
const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
|
|
395
|
+
const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
|
|
396
|
+
if (expired.length > 0) {
|
|
397
|
+
await del(expired.map((blob) => blob.pathname), { token });
|
|
398
|
+
deleted += expired.length;
|
|
399
|
+
}
|
|
400
|
+
if (!result.hasMore || !result.cursor) break;
|
|
401
|
+
cursor = result.cursor;
|
|
402
|
+
}
|
|
403
|
+
return { deleted, store: "private-vercel-blob" };
|
|
404
|
+
}
|
|
405
|
+
|
|
280
406
|
export {
|
|
281
407
|
downloadAsset,
|
|
282
|
-
harvestPageMedia
|
|
408
|
+
harvestPageMedia,
|
|
409
|
+
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
410
|
+
createSiteExtractBundleArtifactStream,
|
|
411
|
+
renewSiteExtractArtifactDownload,
|
|
412
|
+
cleanupExpiredSiteExtractArtifacts
|
|
283
413
|
};
|
|
284
|
-
//# sourceMappingURL=chunk-
|
|
414
|
+
//# sourceMappingURL=chunk-QZXKQB7Y.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/site-extract-artifacts.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nfunction typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nfunction typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n const seen = new Set<string>()\n const urls: string[] = []\n\n const add = (raw: string) => {\n const resolved = resolveUrl(raw.trim(), baseUrl)\n if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return\n seen.add(resolved)\n urls.push(resolved)\n }\n\n for (const m of html.matchAll(/<img\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<source\\s[^>]*>/gi)) {\n const tag = m[0]\n const src = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (src) add(src)\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<video\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'poster']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n }\n\n for (const m of html.matchAll(/<audio\\s[^>]*>/gi)) {\n const tag = m[0]\n const v = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (v) add(v)\n }\n\n for (const m of html.matchAll(/<meta\\s[^>]*>/gi)) {\n const tag = m[0]\n const prop = (tag.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]?.toLowerCase()\n if (prop === 'og:image' || prop === 'twitter:image') {\n const content = (tag.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (content) add(content)\n }\n }\n\n for (const m of html.matchAll(/background(?:-image)?\\s*:\\s*url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)/gi)) {\n add(m[1])\n }\n\n return urls\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n options: {\n expectedType?: MediaType\n maxBytes?: number\n consumeBytes?: (bytes: number) => boolean\n } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n let target = url\n let res: Response | null = null\n for (let redirects = 0; redirects <= 5; redirects++) {\n const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n res = await fetch(checked.parsed.href, {\n headers: { 'User-Agent': 'Mozilla/5.0 (compatible; ThorbitBot/1.0)' },\n signal: AbortSignal.timeout(15_000),\n redirect: 'manual',\n })\n if (res.status >= 300 && res.status < 400) {\n const location = res.headers.get('location')\n if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n target = new URL(location, checked.parsed.href).href\n res = null\n continue\n }\n break\n }\n if (!res) throw new Error('Media download exceeded five redirects')\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n }\n const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n }\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await new Promise<void>((resolve, reject) => {\n writer.once('open', () => resolve())\n writer.once('error', reject)\n })\n let streamedBytes = 0\n const limiter = new Transform({\n transform(chunk: Buffer, _encoding, callback) {\n const bytes = chunk.length\n if (streamedBytes + bytes > maxBytes) {\n callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n return\n }\n if (options.consumeBytes && !options.consumeBytes(bytes)) {\n callback(new Error('Media export total byte limit exceeded'))\n return\n }\n streamedBytes += bytes\n callback(null, chunk)\n },\n })\n try {\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n } catch (error) {\n writer.destroy()\n rmSync(dest, { force: true })\n throw error\n }\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const rawUrls = extractMediaUrls(html, pageUrl)\n const totalFound = rawUrls.length\n\n const filteredUrls: string[] = []\n const kept: Array<{ url: string; type: MediaType }> = []\n\n for (const url of rawUrls) {\n const type = typeFromUrl(url)\n if (!type || !typesSet.has(type)) { filteredUrls.push(url); continue }\n kept.push({ url, type })\n }\n\n if (options.outputDir === null) {\n return {\n pageUrl,\n outputDir: null,\n assets: kept.map(({ url, type }, i) => ({\n url, type, mimeType: null, filename: safeFilename(url, i), savedPath: null, sizeBytes: null,\n })),\n filteredCount: filteredUrls.length,\n totalFound,\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async ({ url, type }, i) => {\n const filename = safeFilename(url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? type) : type\n assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes })\n } catch {\n assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound }\n}\n","import {\n createPrivateArtifact,\n createPrivateArtifactFromStream,\n privateArtifactOwnerId,\n renewPrivateArtifactDownload,\n type PrivateArtifactPolicy,\n} from './private-artifacts.js'\nimport type { ExtractJobArtifact } from './site-extract-repository.js'\nimport type { Readable } from 'node:stream'\n\nexport const SITE_EXTRACT_ARTIFACT_PREFIX = 'site-extracts/'\nexport const SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1_000\nexport const SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1_000\n\nfunction siteExtractArtifactToken(): string | null {\n return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()\n || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()\n || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()\n || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()\n || null\n}\n\nfunction hostedByEnvironment(): boolean {\n return process.env.VERCEL === '1' || process.env.NODE_ENV === 'production'\n}\n\nfunction policy(): PrivateArtifactPolicy {\n return {\n prefix: SITE_EXTRACT_ARTIFACT_PREFIX,\n artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,\n downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,\n token: siteExtractArtifactToken(),\n }\n}\n\nexport function siteExtractArtifactOwnerId(artifactId: string): string | null {\n return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX)\n}\n\nexport async function createSiteExtractBundleArtifact(args: {\n ownerId: string\n jobId: string\n createdAt: Date | string | number\n content: Buffer\n}): Promise<ExtractJobArtifact> {\n const pointer = await createPrivateArtifact({\n policy: policy(),\n ownerId: args.ownerId,\n artifactKey: `${args.jobId}.zip`,\n createdAt: args.createdAt,\n filename: `${args.jobId}-site-export.zip`,\n contentType: 'application/zip',\n content: args.content,\n })\n return {\n key: pointer.artifactId,\n url: pointer.downloadUrl ?? '',\n bytes: pointer.bytes,\n contentType: pointer.contentType,\n filename: pointer.filename,\n sha256: pointer.sha256,\n expiresAt: pointer.expiresAt,\n downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n }\n}\n\nexport async function createSiteExtractBundleArtifactStream(args: {\n ownerId: string\n jobId: string\n createdAt: Date | string | number\n content: Readable\n}): Promise<ExtractJobArtifact> {\n const pointer = await createPrivateArtifactFromStream({\n policy: policy(),\n ownerId: args.ownerId,\n artifactKey: `${args.jobId}.zip`,\n createdAt: args.createdAt,\n filename: `${args.jobId}-site-export.zip`,\n contentType: 'application/zip',\n content: args.content,\n })\n return {\n key: pointer.artifactId,\n url: pointer.downloadUrl ?? '',\n bytes: pointer.bytes,\n contentType: pointer.contentType,\n filename: pointer.filename,\n sha256: pointer.sha256,\n expiresAt: pointer.expiresAt,\n downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n }\n}\n\nexport async function renewSiteExtractArtifactDownload(args: {\n artifactId: string\n ownerId: string\n}): Promise<{ downloadUrl: string; downloadUrlExpiresAt: string; expiresAt: string } | null> {\n return renewPrivateArtifactDownload({\n policy: policy(),\n artifactId: args.artifactId,\n ownerId: args.ownerId,\n })\n}\n\nexport async function cleanupExpiredSiteExtractArtifacts(args: {\n now?: Date\n force?: boolean\n} = {}): Promise<{ deleted: number; store: 'private-vercel-blob' | 'local' | 'none'; skipped?: boolean }> {\n const now = args.now ?? new Date()\n const token = siteExtractArtifactToken()\n if (!token) return { deleted: 0, store: hostedByEnvironment() ? 'none' : 'local' }\n if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {\n return { deleted: 0, store: 'private-vercel-blob', skipped: true }\n }\n const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS\n const { list, del } = await import('@vercel/blob')\n let cursor: string | undefined\n let deleted = 0\n for (let page = 0; page < 20; page += 1) {\n const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1_000, cursor })\n const expired = result.blobs.filter(blob => new Date(blob.uploadedAt).getTime() <= cutoff)\n if (expired.length > 0) {\n await del(expired.map(blob => blob.pathname), { token })\n deleted += expired.length\n }\n if (!result.hasMore || !result.cursor) break\n cursor = result.cursor\n }\n return { deleted, store: 'private-vercel-blob' }\n}\n"],"mappings":";;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAGpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AA0BtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEA,SAAS,YAAY,KAA+B;AAClD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,iBAAiB,MAAc,SAA2B;AACxE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,OAAiB,CAAC;AAExB,QAAM,MAAM,CAAC,QAAgB;AAC3B,UAAM,WAAW,WAAW,IAAI,KAAK,GAAG,OAAO;AAC/C,QAAI,CAAC,YAAY,KAAK,IAAI,QAAQ,KAAK,QAAQ,QAAQ,EAAG;AAC1D,SAAK,IAAI,QAAQ;AACjB,SAAK,KAAK,QAAQ;AAAA,EACpB;AAEA,aAAW,KAAK,KAAK,SAAS,gBAAgB,GAAG;AAC/C,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,eAAe,GAAG;AACxE,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AACA,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,mBAAmB,GAAG;AAClD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,OAAO,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAChE,QAAI,IAAK,KAAI,GAAG;AAChB,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,QAAQ,GAAG;AACpC,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,KAAK,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAC9D,QAAI,EAAG,KAAI,CAAC;AAAA,EACd;AAEA,aAAW,KAAK,KAAK,SAAS,iBAAiB,GAAG;AAChD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,QAAQ,IAAI,MAAM,6CAA6C,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY;AAC9F,QAAI,SAAS,cAAc,SAAS,iBAAiB;AACnD,YAAM,WAAW,IAAI,MAAM,mCAAmC,KAAK,CAAC,GAAG,CAAC;AACxE,UAAI,QAAS,KAAI,OAAO;AAAA,IAC1B;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,gEAAgE,GAAG;AAC/F,QAAI,EAAE,CAAC,CAAC;AAAA,EACV;AAEA,SAAO;AACT;AAEA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,UAAW,iBAAiB,MAAM,OAAO;AAC/C,QAAM,aAAa,QAAQ;AAE3B,QAAM,eAAyB,CAAC;AAChC,QAAM,OAAgD,CAAC;AAEvD,aAAW,OAAO,SAAS;AACzB,UAAM,OAAO,YAAY,GAAG;AAC5B,QAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,GAAG;AAAE,mBAAa,KAAK,GAAG;AAAG;AAAA,IAAS;AACrE,SAAK,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,EACzB;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,EAAE,KAAK,KAAK,GAAG,OAAO;AAAA,QACtC;AAAA,QAAK;AAAA,QAAM,UAAU;AAAA,QAAM,UAAU,aAAa,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,MACzF,EAAE;AAAA,MACF,eAAe,aAAa;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,GAAG,MAAM;AACnC,YAAM,WAAW,aAAa,KAAK,CAAC;AACpC,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,KAAK,QAAQ,QAAQ;AACpF,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,OAAQ;AACnE,eAAO,KAAK,EAAE,KAAK,MAAM,cAAc,UAAU,UAAU,SAAS,SAAS,GAAG,WAAW,UAAU,CAAC;AAAA,MACxG,QAAQ;AACN,eAAO,KAAK,EAAE,KAAK,MAAM,UAAU,MAAM,UAAU,WAAW,MAAM,WAAW,KAAK,CAAC;AAAA,MACvF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,eAAe,aAAa,QAAQ,WAAW;AAC9F;;;ACnTO,IAAM,+BAA+B;AACrC,IAAM,+BAA+B,IAAI,KAAK,KAAK,KAAK;AACxD,IAAM,+BAA+B,KAAK,KAAK;AAEtD,SAAS,2BAA0C;AACjD,SAAO,QAAQ,IAAI,wCAAwC,KAAK,KAC3D,QAAQ,IAAI,mCAAmC,KAAK,KACpD,QAAQ,IAAI,iCAAiC,KAAK,KAClD,QAAQ,IAAI,sCAAsC,KAAK,KACvD;AACP;AAEA,SAAS,sBAA+B;AACtC,SAAO,QAAQ,IAAI,WAAW,OAAO,QAAQ,IAAI,aAAa;AAChE;AAEA,SAAS,SAAgC;AACvC,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,eAAe;AAAA,IACf,OAAO,yBAAyB;AAAA,EAClC;AACF;AAiCA,eAAsB,sCAAsC,MAK5B;AAC9B,QAAM,UAAU,MAAM,gCAAgC;AAAA,IACpD,QAAQ,OAAO;AAAA,IACf,SAAS,KAAK;AAAA,IACd,aAAa,GAAG,KAAK,KAAK;AAAA,IAC1B,WAAW,KAAK;AAAA,IAChB,UAAU,GAAG,KAAK,KAAK;AAAA,IACvB,aAAa;AAAA,IACb,SAAS,KAAK;AAAA,EAChB,CAAC;AACD,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ,eAAe;AAAA,IAC5B,OAAO,QAAQ;AAAA,IACf,aAAa,QAAQ;AAAA,IACrB,UAAU,QAAQ;AAAA,IAClB,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB,sBAAsB,QAAQ;AAAA,EAChC;AACF;AAEA,eAAsB,iCAAiC,MAGsC;AAC3F,SAAO,6BAA6B;AAAA,IAClC,QAAQ,OAAO;AAAA,IACf,YAAY,KAAK;AAAA,IACjB,SAAS,KAAK;AAAA,EAChB,CAAC;AACH;AAEA,eAAsB,mCAAmC,OAGrD,CAAC,GAAqG;AACxG,QAAM,MAAM,KAAK,OAAO,oBAAI,KAAK;AACjC,QAAM,QAAQ,yBAAyB;AACvC,MAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,OAAO,oBAAoB,IAAI,SAAS,QAAQ;AACjF,MAAI,CAAC,KAAK,SAAS,EAAE,IAAI,YAAY,MAAM,KAAK,IAAI,cAAc,MAAM,KAAK;AAC3E,WAAO,EAAE,SAAS,GAAG,OAAO,uBAAuB,SAAS,KAAK;AAAA,EACnE;AACA,QAAM,SAAS,IAAI,QAAQ,IAAI;AAC/B,QAAM,EAAE,MAAM,IAAI,IAAI,MAAM,OAAO,cAAc;AACjD,MAAI;AACJ,MAAI,UAAU;AACd,WAAS,OAAO,GAAG,OAAO,IAAI,QAAQ,GAAG;AACvC,UAAM,SAAS,MAAM,KAAK,EAAE,QAAQ,8BAA8B,OAAO,OAAO,KAAO,OAAO,CAAC;AAC/F,UAAM,UAAU,OAAO,MAAM,OAAO,UAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,QAAQ,KAAK,MAAM;AACzF,QAAI,QAAQ,SAAS,GAAG;AACtB,YAAM,IAAI,QAAQ,IAAI,UAAQ,KAAK,QAAQ,GAAG,EAAE,MAAM,CAAC;AACvD,iBAAW,QAAQ;AAAA,IACrB;AACA,QAAI,CAAC,OAAO,WAAW,CAAC,OAAO,OAAQ;AACvC,aAAS,OAAO;AAAA,EAClB;AACA,SAAO,EAAE,SAAS,OAAO,sBAAsB;AACjD;","names":[]}
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
generateApiKey,
|
|
37
37
|
getBillingLedgerSummary,
|
|
38
38
|
getClient,
|
|
39
|
+
getConnectedAccountBillingState,
|
|
39
40
|
getDb,
|
|
40
41
|
getInboxMessage,
|
|
41
42
|
getJob,
|
|
@@ -69,6 +70,7 @@ import {
|
|
|
69
70
|
listKpoJobs,
|
|
70
71
|
listRequestEvents,
|
|
71
72
|
listUsers,
|
|
73
|
+
listUsersDueForConnectedAccountBillingReconciliation,
|
|
72
74
|
listWorkflowArtifacts,
|
|
73
75
|
listWorkflowRuns,
|
|
74
76
|
listWorkflowSchedules,
|
|
@@ -93,6 +95,7 @@ import {
|
|
|
93
95
|
rotateApiKey,
|
|
94
96
|
rotateRefresh,
|
|
95
97
|
setConcurrencySubId,
|
|
98
|
+
setConnectedAccountBillingState,
|
|
96
99
|
setDbForTesting,
|
|
97
100
|
setExtraConcurrencySlots,
|
|
98
101
|
setMemoryPlan,
|
|
@@ -108,7 +111,7 @@ import {
|
|
|
108
111
|
updateKpoJobState,
|
|
109
112
|
upsertPageSnapshot,
|
|
110
113
|
verifyPassword
|
|
111
|
-
} from "./chunk-
|
|
114
|
+
} from "./chunk-IFYER7O4.js";
|
|
112
115
|
export {
|
|
113
116
|
CREDIT_LOT_TTL,
|
|
114
117
|
SiteAuditJobRowSchema,
|
|
@@ -147,6 +150,7 @@ export {
|
|
|
147
150
|
generateApiKey,
|
|
148
151
|
getBillingLedgerSummary,
|
|
149
152
|
getClient,
|
|
153
|
+
getConnectedAccountBillingState,
|
|
150
154
|
getDb,
|
|
151
155
|
getInboxMessage,
|
|
152
156
|
getJob,
|
|
@@ -180,6 +184,7 @@ export {
|
|
|
180
184
|
listKpoJobs,
|
|
181
185
|
listRequestEvents,
|
|
182
186
|
listUsers,
|
|
187
|
+
listUsersDueForConnectedAccountBillingReconciliation,
|
|
183
188
|
listWorkflowArtifacts,
|
|
184
189
|
listWorkflowRuns,
|
|
185
190
|
listWorkflowSchedules,
|
|
@@ -204,6 +209,7 @@ export {
|
|
|
204
209
|
rotateApiKey,
|
|
205
210
|
rotateRefresh,
|
|
206
211
|
setConcurrencySubId,
|
|
212
|
+
setConnectedAccountBillingState,
|
|
207
213
|
setDbForTesting,
|
|
208
214
|
setExtraConcurrencySlots,
|
|
209
215
|
setMemoryPlan,
|
|
@@ -220,4 +226,4 @@ export {
|
|
|
220
226
|
upsertPageSnapshot,
|
|
221
227
|
verifyPassword
|
|
222
228
|
};
|
|
223
|
-
//# sourceMappingURL=db-
|
|
229
|
+
//# sourceMappingURL=db-N6MPVMEF.js.map
|