mcp-scraper 0.76.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +4 -4
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-JNFOBQVH.js → chunk-7HF3T7P2.js} +1 -1
- package/dist/{chunk-BOGRXBH2.js → chunk-DTJ56S5B.js} +1 -1
- package/dist/{chunk-XSYEDC25.js → chunk-GT6MP4SU.js} +1 -1
- package/dist/{chunk-SDX6TRJ6.js → chunk-IP7UDXMU.js} +3 -3
- package/dist/{server-IAGYCPN6.js → server-EKRNKKLU.js} +1126 -898
- package/package.json +1 -1
|
@@ -314,7 +314,7 @@ import {
|
|
|
314
314
|
resolveDeploymentProfile,
|
|
315
315
|
resolveLocalSourcebookSchemaType,
|
|
316
316
|
transcribeMediaUrl
|
|
317
|
-
} from "./chunk-
|
|
317
|
+
} from "./chunk-DTJ56S5B.js";
|
|
318
318
|
import {
|
|
319
319
|
auditImageUrls,
|
|
320
320
|
auditImages,
|
|
@@ -433,7 +433,7 @@ import {
|
|
|
433
433
|
} from "./chunk-GGZEC22A.js";
|
|
434
434
|
import {
|
|
435
435
|
PACKAGE_VERSION
|
|
436
|
-
} from "./chunk-
|
|
436
|
+
} from "./chunk-7HF3T7P2.js";
|
|
437
437
|
import {
|
|
438
438
|
CaptchaError,
|
|
439
439
|
MapsSearchIncompletePageError,
|
|
@@ -3801,8 +3801,8 @@ function el(type, style, ...children) {
|
|
|
3801
3801
|
const ch = children.flat().filter((c) => c != null);
|
|
3802
3802
|
return { type, props: { style, children: ch.length === 0 ? void 0 : ch.length === 1 ? ch[0] : ch } };
|
|
3803
3803
|
}
|
|
3804
|
-
function clean(
|
|
3805
|
-
return
|
|
3804
|
+
function clean(text2) {
|
|
3805
|
+
return text2.replace(/[\u{1F000}-\u{1FFFF}\u{2600}-\u{27BF}]/gu, "").replace(/[().,!?]/g, "").trim();
|
|
3806
3806
|
}
|
|
3807
3807
|
function tagline(post) {
|
|
3808
3808
|
if (post.titleDisplayItalic) return clean(post.titleDisplayItalic).slice(0, 38);
|
|
@@ -4099,8 +4099,8 @@ async function runKernelAttempt(client2, auditedUrl, mode) {
|
|
|
4099
4099
|
});
|
|
4100
4100
|
}
|
|
4101
4101
|
}
|
|
4102
|
-
async function fetchWithKernelDetailed(
|
|
4103
|
-
const checked = await validatePublicHttpUrl(
|
|
4102
|
+
async function fetchWithKernelDetailed(url2) {
|
|
4103
|
+
const checked = await validatePublicHttpUrl(url2, { field: "browser URL" });
|
|
4104
4104
|
if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
|
|
4105
4105
|
const auditedUrl = checked.parsed.href;
|
|
4106
4106
|
const apiKey = browserServiceApiKey();
|
|
@@ -4158,8 +4158,8 @@ async function fetchWithKernelDetailed(url) {
|
|
|
4158
4158
|
error.code = "browser_challenge_unresolved";
|
|
4159
4159
|
throw error;
|
|
4160
4160
|
}
|
|
4161
|
-
async function fetchWithKernel(
|
|
4162
|
-
return (await fetchWithKernelDetailed(
|
|
4161
|
+
async function fetchWithKernel(url2) {
|
|
4162
|
+
return (await fetchWithKernelDetailed(url2)).html;
|
|
4163
4163
|
}
|
|
4164
4164
|
|
|
4165
4165
|
// src/lib/http-user-agent.ts
|
|
@@ -4188,12 +4188,12 @@ async function extractKpo(opts) {
|
|
|
4188
4188
|
continue;
|
|
4189
4189
|
}
|
|
4190
4190
|
if (!res.ok) return null;
|
|
4191
|
-
const
|
|
4192
|
-
if (
|
|
4193
|
-
const document3 = loadHtmlDocument(
|
|
4194
|
-
if (detectBrowserChallenge({ status: res.status, title: document3.title, html:
|
|
4191
|
+
const text2 = await res.text();
|
|
4192
|
+
if (text2.length < 2e3) return null;
|
|
4193
|
+
const document3 = loadHtmlDocument(text2);
|
|
4194
|
+
if (detectBrowserChallenge({ status: res.status, title: document3.title, html: text2 }).detected) return null;
|
|
4195
4195
|
if (document3.mainText.length < 300) return null;
|
|
4196
|
-
return
|
|
4196
|
+
return text2;
|
|
4197
4197
|
}
|
|
4198
4198
|
return null;
|
|
4199
4199
|
} catch {
|
|
@@ -4498,9 +4498,9 @@ function selectRankedLogoCandidates(candidates) {
|
|
|
4498
4498
|
}
|
|
4499
4499
|
function logoFamilyKey(value) {
|
|
4500
4500
|
try {
|
|
4501
|
-
const
|
|
4502
|
-
const host2 =
|
|
4503
|
-
const path5 = decodeURIComponent(
|
|
4501
|
+
const url2 = new URL(value);
|
|
4502
|
+
const host2 = url2.hostname.replace(/^www\./, "").toLowerCase();
|
|
4503
|
+
const path5 = decodeURIComponent(url2.pathname).toLowerCase().replace(/-\d+x\d+(?=\.[a-z0-9]+$)/, "");
|
|
4504
4504
|
return `${host2}${path5}`;
|
|
4505
4505
|
} catch {
|
|
4506
4506
|
return value.toLowerCase();
|
|
@@ -4902,8 +4902,8 @@ async function extractBrandingFromPage(page2) {
|
|
|
4902
4902
|
const primaryChoice = cssPrimary ? { value: cssPrimary.hex, source: "css_variable", confidence: "high", detail: cssPrimary.name } : themeColor ? { value: themeColor, source: "theme_color", confidence: "high", detail: 'meta[name="theme-color"]' } : navBgUsable && isChromatic(navBgUsable) ? { value: navBgUsable, source: "navigation_background", confidence: "medium", detail: "nav, header, or banner computed background" } : topBg && isChromatic(topBg) ? { value: topBg, source: "navigation_background", confidence: "medium", detail: "dominant wide rendered background near the top of the page" } : svgPrimary && isChromatic(svgPrimary) ? { value: svgPrimary, source: "header_svg", confidence: "medium", detail: "dominant rendered identity SVG fill near the top of the page" } : navChildBg && isChromatic(navChildBg) ? { value: navChildBg, source: "navigation_background", confidence: "medium", detail: "dominant rendered navigation child background" } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : visibleBg && isChromatic(visibleBg) ? { value: visibleBg, source: "visible_background", confidence: "low", detail: "area-weighted rendered element background" } : bodyBgHex && !isTransparentOrWhite(bodyBgHex) ? { value: bodyBgHex, source: "body_background", confidence: "low", detail: "computed body background fallback" } : null;
|
|
4903
4903
|
const accentChoice = cssAccent ? { value: cssAccent.hex, source: "css_variable", confidence: "high", detail: cssAccent.name } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : null;
|
|
4904
4904
|
const primary = primaryChoice?.value ?? null;
|
|
4905
|
-
const
|
|
4906
|
-
const
|
|
4905
|
+
const accent2 = accentChoice?.value ?? null;
|
|
4906
|
+
const text2 = raw.bodyColor;
|
|
4907
4907
|
const heading = raw.h1Color;
|
|
4908
4908
|
const background = bodyBgHex;
|
|
4909
4909
|
const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
|
|
@@ -4911,16 +4911,16 @@ async function extractBrandingFromPage(page2) {
|
|
|
4911
4911
|
colorScheme: scheme,
|
|
4912
4912
|
colors: {
|
|
4913
4913
|
primary: primary && !isTransparentOrWhite(primary) ? primary : null,
|
|
4914
|
-
accent:
|
|
4914
|
+
accent: accent2 && !isTransparentOrWhite(accent2) ? accent2 : null,
|
|
4915
4915
|
background: background ?? null,
|
|
4916
|
-
text:
|
|
4916
|
+
text: text2 ?? null,
|
|
4917
4917
|
heading: heading ?? null
|
|
4918
4918
|
},
|
|
4919
4919
|
colorEvidence: {
|
|
4920
4920
|
primary: primaryChoice,
|
|
4921
4921
|
accent: accentChoice,
|
|
4922
4922
|
background: background ? { value: background, source: "body_background", confidence: "high", detail: "computed body background" } : null,
|
|
4923
|
-
text:
|
|
4923
|
+
text: text2 ? { value: text2, source: "body_text", confidence: "high", detail: "computed body text color" } : null,
|
|
4924
4924
|
heading: heading ? { value: heading, source: "heading_text", confidence: "high", detail: "computed first H1 text color" } : null
|
|
4925
4925
|
},
|
|
4926
4926
|
fonts: {
|
|
@@ -4984,10 +4984,10 @@ async function readRenderedSnapshot(page2) {
|
|
|
4984
4984
|
return contexts;
|
|
4985
4985
|
};
|
|
4986
4986
|
const add = (raw, type, method, element, dimensions, alt) => {
|
|
4987
|
-
const
|
|
4988
|
-
if (!
|
|
4989
|
-
const existing = rows.get(
|
|
4990
|
-
url,
|
|
4987
|
+
const url2 = resolve2(raw);
|
|
4988
|
+
if (!url2) return;
|
|
4989
|
+
const existing = rows.get(url2) ?? {
|
|
4990
|
+
url: url2,
|
|
4991
4991
|
type,
|
|
4992
4992
|
discoveryMethods: [],
|
|
4993
4993
|
altTexts: [],
|
|
@@ -5006,7 +5006,7 @@ async function readRenderedSnapshot(page2) {
|
|
|
5006
5006
|
existing.height = dimensions.height;
|
|
5007
5007
|
}
|
|
5008
5008
|
}
|
|
5009
|
-
rows.set(
|
|
5009
|
+
rows.set(url2, existing);
|
|
5010
5010
|
};
|
|
5011
5011
|
const srcsetUrls = (value) => (value ?? "").split(",").map((part) => part.trim().split(/\s+/)[0] ?? "").filter(Boolean);
|
|
5012
5012
|
for (const img of Array.from(document.querySelectorAll("img"))) {
|
|
@@ -5017,7 +5017,7 @@ async function readRenderedSnapshot(page2) {
|
|
|
5017
5017
|
for (const attr of ["data-src", "data-lazy-src", "data-original", "data-bg", "data-background"]) {
|
|
5018
5018
|
add(image.getAttribute(attr), "image", `rendered-${attr}`, image, dimensions, image.alt);
|
|
5019
5019
|
}
|
|
5020
|
-
for (const
|
|
5020
|
+
for (const url2 of srcsetUrls(image.getAttribute("srcset"))) add(url2, "image", "rendered-srcset", image, dimensions, image.alt);
|
|
5021
5021
|
const anchor = image.closest("a[href]");
|
|
5022
5022
|
const anchorAttrs = anchor ? Array.from(anchor.attributes).map((attribute) => `${attribute.name}=${attribute.value}`).join(" ") : "";
|
|
5023
5023
|
if (anchor && (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(anchor.href))) add(anchor.href, "image", "rendered-lightbox-target", image, void 0, image.alt);
|
|
@@ -5026,7 +5026,7 @@ async function readRenderedSnapshot(page2) {
|
|
|
5026
5026
|
const parent = source.closest("picture,video,audio");
|
|
5027
5027
|
const type = parent?.tagName === "VIDEO" ? "video" : parent?.tagName === "AUDIO" ? "audio" : "image";
|
|
5028
5028
|
add(source.getAttribute("src"), type, "rendered-source", source);
|
|
5029
|
-
for (const
|
|
5029
|
+
for (const url2 of srcsetUrls(source.getAttribute("srcset"))) add(url2, type, "rendered-source-srcset", source);
|
|
5030
5030
|
}
|
|
5031
5031
|
for (const video of Array.from(document.querySelectorAll("video"))) {
|
|
5032
5032
|
const media = video;
|
|
@@ -5140,7 +5140,7 @@ async function scrollAndStitch(page2, device = "desktop") {
|
|
|
5140
5140
|
await page2.waitForTimeout(LAZY_SETTLE_MS);
|
|
5141
5141
|
return page2.screenshot({ type: "png", fullPage: true, animations: "disabled" });
|
|
5142
5142
|
}
|
|
5143
|
-
async function captureLocal(
|
|
5143
|
+
async function captureLocal(url2, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
5144
5144
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
5145
5145
|
const browser = await chromium2.launch({ headless: true });
|
|
5146
5146
|
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
@@ -5151,7 +5151,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia
|
|
|
5151
5151
|
}, runtimeUserAgent));
|
|
5152
5152
|
const page2 = await context.newPage();
|
|
5153
5153
|
try {
|
|
5154
|
-
await page2.goto(
|
|
5154
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
5155
5155
|
await page2.waitForTimeout(1500);
|
|
5156
5156
|
const result2 = {};
|
|
5157
5157
|
if (wantBranding) result2.branding = await extractBrandingFromPage(page2);
|
|
@@ -5162,7 +5162,7 @@ async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia
|
|
|
5162
5162
|
await browser.close();
|
|
5163
5163
|
}
|
|
5164
5164
|
}
|
|
5165
|
-
async function captureKernel(
|
|
5165
|
+
async function captureKernel(url2, kernelApiKey, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
5166
5166
|
const kernel = new Kernel2({ apiKey: kernelApiKey });
|
|
5167
5167
|
const openedAtMs = Date.now();
|
|
5168
5168
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
@@ -5175,7 +5175,7 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
5175
5175
|
}, runtimeUserAgent));
|
|
5176
5176
|
const page2 = context.pages()[0] ?? await context.newPage();
|
|
5177
5177
|
try {
|
|
5178
|
-
await page2.goto(
|
|
5178
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
5179
5179
|
await page2.waitForTimeout(1500);
|
|
5180
5180
|
const result2 = {};
|
|
5181
5181
|
if (wantBranding) result2.branding = await extractBrandingFromPage(page2);
|
|
@@ -5187,26 +5187,26 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
5187
5187
|
void recordKernelSession({ kernelSessionId: session.session_id, source: "screenshot", stealth: typeof session.stealth === "boolean" ? session.stealth : null, headlessSent: null, proxyUsed: typeof session.proxy_id === "string" ? Boolean(session.proxy_id) : null, openedAtMs, closedAtMs: Date.now() });
|
|
5188
5188
|
}
|
|
5189
5189
|
}
|
|
5190
|
-
async function captureScreenshot(
|
|
5191
|
-
const result2 = await capturePageData(
|
|
5190
|
+
async function captureScreenshot(url2, kernelApiKey, device = "desktop") {
|
|
5191
|
+
const result2 = await capturePageData(url2, { kernelApiKey, device, screenshot: true, branding: false });
|
|
5192
5192
|
return result2.screenshot;
|
|
5193
5193
|
}
|
|
5194
|
-
async function extractBranding(
|
|
5195
|
-
const result2 = await capturePageData(
|
|
5194
|
+
async function extractBranding(url2, kernelApiKey, device = "desktop") {
|
|
5195
|
+
const result2 = await capturePageData(url2, { kernelApiKey, device, screenshot: false, branding: true });
|
|
5196
5196
|
return result2.branding;
|
|
5197
5197
|
}
|
|
5198
|
-
async function capturePageData(
|
|
5198
|
+
async function capturePageData(url2, opts = {}) {
|
|
5199
5199
|
const device = opts.device ?? "desktop";
|
|
5200
5200
|
const wantShot = opts.screenshot ?? false;
|
|
5201
5201
|
const wantBranding = opts.branding ?? false;
|
|
5202
5202
|
const wantMedia = opts.media ?? false;
|
|
5203
5203
|
const maxMediaAssets = opts.maxMediaAssets ?? 1e3;
|
|
5204
5204
|
try {
|
|
5205
|
-
return await captureLocal(
|
|
5205
|
+
return await captureLocal(url2, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
5206
5206
|
} catch (localErr) {
|
|
5207
5207
|
const msg = localErr instanceof Error ? localErr.message : String(localErr);
|
|
5208
5208
|
if (!opts.kernelApiKey) throw new Error(`Local capture failed (${msg}) and no headless browser backend is configured`);
|
|
5209
|
-
return await captureKernel(
|
|
5209
|
+
return await captureKernel(url2, opts.kernelApiKey, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
5210
5210
|
}
|
|
5211
5211
|
}
|
|
5212
5212
|
|
|
@@ -5343,11 +5343,11 @@ function extractLinks(html, base) {
|
|
|
5343
5343
|
}
|
|
5344
5344
|
return links;
|
|
5345
5345
|
}
|
|
5346
|
-
function sameAuditedHost(
|
|
5347
|
-
return
|
|
5346
|
+
function sameAuditedHost(url2, auditedHost) {
|
|
5347
|
+
return url2.hostname.replace(/^www\./, "").toLowerCase() === auditedHost.replace(/^www\./, "").toLowerCase();
|
|
5348
5348
|
}
|
|
5349
|
-
async function fetchText(
|
|
5350
|
-
let target =
|
|
5349
|
+
async function fetchText(url2, auditedHost, timeoutMs = 1e4) {
|
|
5350
|
+
let target = url2;
|
|
5351
5351
|
try {
|
|
5352
5352
|
for (let redirects = 0; redirects <= 5; redirects++) {
|
|
5353
5353
|
const checked = await validatePublicHttpUrl(target, { field: "sitemap URL" });
|
|
@@ -5410,17 +5410,17 @@ async function discoverSitemapUrls(startUrl, startDomain, maxUrls = 1e4, maxSite
|
|
|
5410
5410
|
const seenPages = /* @__PURE__ */ new Set();
|
|
5411
5411
|
const urls = [];
|
|
5412
5412
|
let retainedUrlBytes = 0;
|
|
5413
|
-
const reserveUrl = (
|
|
5414
|
-
if (!boundedDiscoveredUrl(
|
|
5415
|
-
const bytes = Buffer.byteLength(
|
|
5413
|
+
const reserveUrl = (url2) => {
|
|
5414
|
+
if (!boundedDiscoveredUrl(url2)) return false;
|
|
5415
|
+
const bytes = Buffer.byteLength(url2);
|
|
5416
5416
|
if (retainedUrlBytes + bytes > MAX_DISCOVERY_FRONTIER_BYTES) return false;
|
|
5417
5417
|
retainedUrlBytes += bytes;
|
|
5418
5418
|
return true;
|
|
5419
5419
|
};
|
|
5420
|
-
for (const
|
|
5421
|
-
if (queuedSitemaps.has(
|
|
5422
|
-
queuedSitemaps.add(
|
|
5423
|
-
sitemapQueue.push({ url, depth: 0 });
|
|
5420
|
+
for (const url2 of candidates.slice(0, 8)) {
|
|
5421
|
+
if (queuedSitemaps.has(url2) || !reserveUrl(url2)) continue;
|
|
5422
|
+
queuedSitemaps.add(url2);
|
|
5423
|
+
sitemapQueue.push({ url: url2, depth: 0 });
|
|
5424
5424
|
}
|
|
5425
5425
|
while (sitemapQueue.length > 0 && visitedSitemaps.size < documentLimit && urls.length < pageLimit) {
|
|
5426
5426
|
const entry = sitemapQueue.shift();
|
|
@@ -5464,9 +5464,9 @@ async function discoverSitemapUrls(startUrl, startDomain, maxUrls = 1e4, maxSite
|
|
|
5464
5464
|
}
|
|
5465
5465
|
return urls;
|
|
5466
5466
|
}
|
|
5467
|
-
async function fetchPage(
|
|
5468
|
-
let target =
|
|
5469
|
-
const auditedHost = baseDomain(
|
|
5467
|
+
async function fetchPage(url2, timeoutMs) {
|
|
5468
|
+
let target = url2;
|
|
5469
|
+
const auditedHost = baseDomain(url2);
|
|
5470
5470
|
try {
|
|
5471
5471
|
for (let redirects = 0; redirects <= 5; redirects++) {
|
|
5472
5472
|
const checked = await validatePublicHttpUrl(target, { field: "page URL" });
|
|
@@ -5493,9 +5493,9 @@ async function fetchPage(url, timeoutMs) {
|
|
|
5493
5493
|
return { html: null, status: null };
|
|
5494
5494
|
}
|
|
5495
5495
|
}
|
|
5496
|
-
function baseDomain(
|
|
5496
|
+
function baseDomain(url2) {
|
|
5497
5497
|
try {
|
|
5498
|
-
return new URL(
|
|
5498
|
+
return new URL(url2).hostname.replace(/^www\./, "");
|
|
5499
5499
|
} catch {
|
|
5500
5500
|
return "";
|
|
5501
5501
|
}
|
|
@@ -5533,17 +5533,17 @@ async function spiderSite(opts) {
|
|
|
5533
5533
|
while (queue.length > 0 && results.length < maxUrls && Date.now() - startMs < budgetMs) {
|
|
5534
5534
|
const batch = queue.splice(0, concurrency);
|
|
5535
5535
|
const settled = await Promise.all(
|
|
5536
|
-
batch.map(async ({ url, from }) => {
|
|
5537
|
-
const { html, status } = await fetchPage(
|
|
5538
|
-
const discovered = { url, status, linkedFrom: from, via: "fetch" };
|
|
5536
|
+
batch.map(async ({ url: url2, from }) => {
|
|
5537
|
+
const { html, status } = await fetchPage(url2, timeoutMs);
|
|
5538
|
+
const discovered = { url: url2, status, linkedFrom: from, via: "fetch" };
|
|
5539
5539
|
const children = [];
|
|
5540
5540
|
if (html) {
|
|
5541
5541
|
try {
|
|
5542
|
-
for (const link of extractLinks(html,
|
|
5542
|
+
for (const link of extractLinks(html, url2)) {
|
|
5543
5543
|
if (baseDomain(link) !== startDomain) continue;
|
|
5544
5544
|
if (!visited.has(link)) {
|
|
5545
5545
|
visited.add(link);
|
|
5546
|
-
children.push({ url: link, from:
|
|
5546
|
+
children.push({ url: link, from: url2 });
|
|
5547
5547
|
}
|
|
5548
5548
|
}
|
|
5549
5549
|
} catch {
|
|
@@ -5667,15 +5667,15 @@ async function captureInertRenderedDom(page2) {
|
|
|
5667
5667
|
return { renderedDom: html.slice(0, low), renderedDomTruncated: true, renderedDomSanitized: true };
|
|
5668
5668
|
}, MAX_RENDERED_DOM_BYTES);
|
|
5669
5669
|
}
|
|
5670
|
-
async function crawlPage(context,
|
|
5670
|
+
async function crawlPage(context, url2, headless, telemetry, options) {
|
|
5671
5671
|
const startedAt = Date.now();
|
|
5672
5672
|
let page2 = null;
|
|
5673
5673
|
let guardState = null;
|
|
5674
5674
|
try {
|
|
5675
5675
|
page2 = await context.newPage();
|
|
5676
|
-
guardState = await installBrowserRequestGuard(page2,
|
|
5676
|
+
guardState = await installBrowserRequestGuard(page2, url2);
|
|
5677
5677
|
const work = (async (p) => {
|
|
5678
|
-
const resp = await p.goto(
|
|
5678
|
+
const resp = await p.goto(url2, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
5679
5679
|
if (options.renderJavaScript || options.captureRenderedDom) await settleRenderedPage(p);
|
|
5680
5680
|
let html = await p.evaluate((maxBytes) => {
|
|
5681
5681
|
const value = document.documentElement?.outerHTML ?? "";
|
|
@@ -5696,13 +5696,13 @@ async function crawlPage(context, url, headless, telemetry, options) {
|
|
|
5696
5696
|
if (challenge.detected) {
|
|
5697
5697
|
const captchaTelemetry = await readCaptchaTelemetry(telemetry.client, telemetry.sessionId);
|
|
5698
5698
|
return {
|
|
5699
|
-
url,
|
|
5699
|
+
url: url2,
|
|
5700
5700
|
html: null,
|
|
5701
5701
|
status: resp?.status() ?? 0,
|
|
5702
5702
|
via: "browser",
|
|
5703
5703
|
headers,
|
|
5704
5704
|
responseTimeMs: Date.now() - startedAt,
|
|
5705
|
-
redirectUrl: finalUrl && finalUrl !==
|
|
5705
|
+
redirectUrl: finalUrl && finalUrl !== url2 ? finalUrl : null,
|
|
5706
5706
|
errorCode: "browser_challenge_unresolved",
|
|
5707
5707
|
error: `Browser challenge remained unresolved (${challenge.reason ?? "unknown challenge"}).`,
|
|
5708
5708
|
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
@@ -5714,13 +5714,13 @@ async function crawlPage(context, url, headless, telemetry, options) {
|
|
|
5714
5714
|
const isHttpError = resp != null && (resp.status() < 200 || resp.status() >= 300);
|
|
5715
5715
|
const rendered = options.captureRenderedDom ? await captureInertRenderedDom(p) : null;
|
|
5716
5716
|
return {
|
|
5717
|
-
url,
|
|
5717
|
+
url: url2,
|
|
5718
5718
|
html,
|
|
5719
5719
|
status: resp?.status() ?? 0,
|
|
5720
5720
|
via: "browser",
|
|
5721
5721
|
headers,
|
|
5722
5722
|
responseTimeMs: Date.now() - startedAt,
|
|
5723
|
-
redirectUrl: finalUrl && finalUrl !==
|
|
5723
|
+
redirectUrl: finalUrl && finalUrl !== url2 ? finalUrl : null,
|
|
5724
5724
|
...isHttpError ? { errorCode: `browser_http_${resp.status()}`, error: `Browser fetch returned HTTP ${resp.status()}` } : {},
|
|
5725
5725
|
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5726
5726
|
challengeDetected: false,
|
|
@@ -5735,7 +5735,7 @@ async function crawlPage(context, url, headless, telemetry, options) {
|
|
|
5735
5735
|
const message = guardState?.blockedReason ?? (error instanceof Error ? error.message : String(error));
|
|
5736
5736
|
const errorCode2 = guardState?.blockedReason ? "browser_navigation_blocked" : /exceeds \d+ byte limit/i.test(message) ? "browser_response_too_large" : /timeout/i.test(message) ? "browser_timeout" : "browser_navigation_failed";
|
|
5737
5737
|
return {
|
|
5738
|
-
url,
|
|
5738
|
+
url: url2,
|
|
5739
5739
|
html: null,
|
|
5740
5740
|
status: 0,
|
|
5741
5741
|
via: "browser",
|
|
@@ -5788,7 +5788,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5788
5788
|
while (true) {
|
|
5789
5789
|
const i = next++;
|
|
5790
5790
|
if (i >= targets.length) break;
|
|
5791
|
-
const
|
|
5791
|
+
const url2 = targets[i];
|
|
5792
5792
|
let r;
|
|
5793
5793
|
try {
|
|
5794
5794
|
if (!browser || inBrowser >= urlsPerBrowser) {
|
|
@@ -5801,7 +5801,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5801
5801
|
}
|
|
5802
5802
|
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
5803
5803
|
try {
|
|
5804
|
-
r = await crawlPage(context,
|
|
5804
|
+
r = await crawlPage(context, url2, passHeadless, { client: client2, sessionId: kb.session_id }, opts);
|
|
5805
5805
|
} finally {
|
|
5806
5806
|
await context.close().catch(() => {
|
|
5807
5807
|
});
|
|
@@ -5810,7 +5810,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5810
5810
|
} catch (error) {
|
|
5811
5811
|
const message = error instanceof Error ? error.message : String(error);
|
|
5812
5812
|
r = {
|
|
5813
|
-
url,
|
|
5813
|
+
url: url2,
|
|
5814
5814
|
html: null,
|
|
5815
5815
|
status: 0,
|
|
5816
5816
|
via: "browser",
|
|
@@ -5822,7 +5822,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5822
5822
|
};
|
|
5823
5823
|
await retire();
|
|
5824
5824
|
}
|
|
5825
|
-
const previous = byUrl.get(
|
|
5825
|
+
const previous = byUrl.get(url2);
|
|
5826
5826
|
const attemptErrors = [
|
|
5827
5827
|
...previous?.attemptErrors ?? [],
|
|
5828
5828
|
...r.error ? [`${phase}: ${r.error}`] : []
|
|
@@ -5832,9 +5832,9 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5832
5832
|
attemptCount: (previous?.attemptCount ?? 0) + 1,
|
|
5833
5833
|
attemptErrors
|
|
5834
5834
|
};
|
|
5835
|
-
byUrl.set(
|
|
5835
|
+
byUrl.set(url2, r);
|
|
5836
5836
|
done++;
|
|
5837
|
-
opts.onPage?.({ url, status: r.status, ok: !!(r.html && r.status >= 200 && r.status < 300), ms: r.responseTimeMs ?? 0, done, total: targets.length, phase });
|
|
5837
|
+
opts.onPage?.({ url: url2, status: r.status, ok: !!(r.html && r.status >= 200 && r.status < 300), ms: r.responseTimeMs ?? 0, done, total: targets.length, phase });
|
|
5838
5838
|
}
|
|
5839
5839
|
} finally {
|
|
5840
5840
|
await retire();
|
|
@@ -6173,8 +6173,8 @@ async function resolveWaybackTimelineCaptures(input) {
|
|
|
6173
6173
|
if (months.length === 0) throw new Error("At least one Wayback month is required.");
|
|
6174
6174
|
const maxPagesPerSnapshot = Math.max(1, Math.min(500, Math.floor(input.maxPagesPerSnapshot)));
|
|
6175
6175
|
const maxCaptures = Math.max(1, Math.min(1e4, Math.floor(input.maxCaptures ?? 1e4)));
|
|
6176
|
-
const requestedUrls = input.timeline.urls ? [...new Set(input.timeline.urls.map((
|
|
6177
|
-
if (requestedUrls?.some((
|
|
6176
|
+
const requestedUrls = input.timeline.urls ? [...new Set(input.timeline.urls.map((url2) => safeHttpUrl(url2)).filter((url2) => Boolean(url2)))].slice(0, maxPagesPerSnapshot) : null;
|
|
6177
|
+
if (requestedUrls?.some((url2) => siteHost(url2) !== rootHost)) {
|
|
6178
6178
|
throw new Error("Every selected Wayback URL must belong to the same site as the root URL.");
|
|
6179
6179
|
}
|
|
6180
6180
|
if (requestedUrls && requestedUrls.length * months.length > MAX_SELECTED_TIMELINE_CELLS) {
|
|
@@ -6275,9 +6275,9 @@ async function inventoryWaybackSnapshots(input) {
|
|
|
6275
6275
|
const successfulHtmlOnly = input.successfulHtmlOnly !== false;
|
|
6276
6276
|
const maxCaptures = Math.max(1, Math.min(1e5, Math.floor(input.maxCaptures ?? 1e4)));
|
|
6277
6277
|
const maxCaptureRows = Math.max(0, Math.min(1e3, Math.floor(input.maxCaptureRows ?? 500)));
|
|
6278
|
-
const selectedUrls = input.urls?.length ? [...new Set(input.urls.map((
|
|
6278
|
+
const selectedUrls = input.urls?.length ? [...new Set(input.urls.map((url2) => safeHttpUrl(url2)).filter((url2) => Boolean(url2)))] : null;
|
|
6279
6279
|
const rootHost = siteHost(rootUrl);
|
|
6280
|
-
if (selectedUrls?.some((
|
|
6280
|
+
if (selectedUrls?.some((url2) => siteHost(url2) !== rootHost)) {
|
|
6281
6281
|
throw new Error("Every selected Wayback inventory URL must belong to the same site as url.");
|
|
6282
6282
|
}
|
|
6283
6283
|
const targets = selectedUrls ?? [rootUrl];
|
|
@@ -6346,8 +6346,8 @@ async function inventoryWaybackSnapshots(input) {
|
|
|
6346
6346
|
}
|
|
6347
6347
|
const monthlyCounts = [...monthly.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([month, captures]) => ({ month, captures }));
|
|
6348
6348
|
const yearlyCounts = [...yearly.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([year, captures]) => ({ year, captures }));
|
|
6349
|
-
const allPerUrl = [...perUrlState.entries()].map(([
|
|
6350
|
-
url,
|
|
6349
|
+
const allPerUrl = [...perUrlState.entries()].map(([url2, state]) => ({
|
|
6350
|
+
url: url2,
|
|
6351
6351
|
captures: state.captures,
|
|
6352
6352
|
uniqueDigests: state.digests.size,
|
|
6353
6353
|
firstTimestamp: state.firstTimestamp,
|
|
@@ -6391,12 +6391,12 @@ function firstJsonLdImage(value) {
|
|
|
6391
6391
|
return null;
|
|
6392
6392
|
}
|
|
6393
6393
|
if (!value || typeof value !== "object") return null;
|
|
6394
|
-
const
|
|
6394
|
+
const record2 = value;
|
|
6395
6395
|
for (const key2 of ["url", "contentUrl"]) {
|
|
6396
|
-
if (typeof
|
|
6396
|
+
if (typeof record2[key2] === "string") return record2[key2];
|
|
6397
6397
|
}
|
|
6398
6398
|
for (const key2 of ["image", "thumbnailUrl", "@graph"]) {
|
|
6399
|
-
const found = firstJsonLdImage(
|
|
6399
|
+
const found = firstJsonLdImage(record2[key2]);
|
|
6400
6400
|
if (found) return found;
|
|
6401
6401
|
}
|
|
6402
6402
|
return null;
|
|
@@ -6430,12 +6430,12 @@ function resolveFeaturedImageCandidates(input) {
|
|
|
6430
6430
|
const resolved = [];
|
|
6431
6431
|
for (const candidate of candidates) {
|
|
6432
6432
|
if (!candidate.value) continue;
|
|
6433
|
-
const
|
|
6434
|
-
if (!
|
|
6435
|
-
seen.add(
|
|
6433
|
+
const url2 = safeHttpUrl(candidate.value, baseUrl);
|
|
6434
|
+
if (!url2 || seen.has(url2)) continue;
|
|
6435
|
+
seen.add(url2);
|
|
6436
6436
|
resolved.push({
|
|
6437
|
-
url,
|
|
6438
|
-
archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp,
|
|
6437
|
+
url: url2,
|
|
6438
|
+
archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url2, "im_") : null,
|
|
6439
6439
|
source: candidate.source
|
|
6440
6440
|
});
|
|
6441
6441
|
}
|
|
@@ -6452,9 +6452,9 @@ function estimatePixels(s) {
|
|
|
6452
6452
|
for (const ch of s) px += PIXEL_WIDTHS[ch] ?? (ch === ch.toUpperCase() && ch !== ch.toLowerCase() ? 10 : 8);
|
|
6453
6453
|
return px;
|
|
6454
6454
|
}
|
|
6455
|
-
function emptyPageData(
|
|
6455
|
+
function emptyPageData(url2, status, via, failure2) {
|
|
6456
6456
|
const page2 = {
|
|
6457
|
-
url,
|
|
6457
|
+
url: url2,
|
|
6458
6458
|
status,
|
|
6459
6459
|
via,
|
|
6460
6460
|
contentType: null,
|
|
@@ -6506,7 +6506,7 @@ function emptyPageData(url, status, via, failure2) {
|
|
|
6506
6506
|
function makeSeedSpider(startUrl, urls) {
|
|
6507
6507
|
return {
|
|
6508
6508
|
startUrl,
|
|
6509
|
-
urls: urls.map((
|
|
6509
|
+
urls: urls.map((url2) => ({ url: url2, status: 200, linkedFrom: "seed", via: "fetch" })),
|
|
6510
6510
|
totalFound: urls.length,
|
|
6511
6511
|
durationMs: 0,
|
|
6512
6512
|
truncated: false,
|
|
@@ -6532,10 +6532,10 @@ var PAGE_DATA_LIMITS = {
|
|
|
6532
6532
|
urlChars: 4096
|
|
6533
6533
|
};
|
|
6534
6534
|
var turndown = new TurndownService2({ headingStyle: "atx", bulletListMarker: "-", codeBlockStyle: "fenced" });
|
|
6535
|
-
function parsePageData(
|
|
6535
|
+
function parsePageData(url2, html, status, via, resp, fetchAttempts = []) {
|
|
6536
6536
|
const document2 = loadHtmlDocument(html);
|
|
6537
6537
|
const $ = document2.$;
|
|
6538
|
-
const documentUrl = resp?.documentUrl ??
|
|
6538
|
+
const documentUrl = resp?.documentUrl ?? url2;
|
|
6539
6539
|
const origin = new URL(documentUrl).origin;
|
|
6540
6540
|
const headers = resp?.headers ?? {};
|
|
6541
6541
|
const contentType = headers["content-type"] ?? null;
|
|
@@ -6606,14 +6606,14 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6606
6606
|
continue;
|
|
6607
6607
|
}
|
|
6608
6608
|
if (!obj || typeof obj !== "object") continue;
|
|
6609
|
-
const
|
|
6610
|
-
if (typeof
|
|
6611
|
-
else if (Array.isArray(
|
|
6612
|
-
for (const value of
|
|
6609
|
+
const record2 = obj;
|
|
6610
|
+
if (typeof record2["@type"] === "string") addSchemaType(record2["@type"]);
|
|
6611
|
+
else if (Array.isArray(record2["@type"])) {
|
|
6612
|
+
for (const value of record2["@type"]) {
|
|
6613
6613
|
if (typeof value === "string") addSchemaType(value);
|
|
6614
6614
|
}
|
|
6615
6615
|
}
|
|
6616
|
-
if (Array.isArray(
|
|
6616
|
+
if (Array.isArray(record2["@graph"])) stack.push(...record2["@graph"].slice(0, 1e3));
|
|
6617
6617
|
}
|
|
6618
6618
|
if (stack.length > 0 || schemaTypes.length >= PAGE_DATA_LIMITS.schemaTypes) truncatedFields.add("schemaTypes");
|
|
6619
6619
|
} catch {
|
|
@@ -6747,7 +6747,7 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6747
6747
|
return u.replace(/\/+$/, "");
|
|
6748
6748
|
}
|
|
6749
6749
|
};
|
|
6750
|
-
const canonicalSelf = canonicalUrl ? canonNorm(canonicalUrl) === canonNorm(
|
|
6750
|
+
const canonicalSelf = canonicalUrl ? canonNorm(canonicalUrl) === canonNorm(url2) : true;
|
|
6751
6751
|
let indexable = true;
|
|
6752
6752
|
let indexabilityReason = null;
|
|
6753
6753
|
if (status !== 200) {
|
|
@@ -6777,7 +6777,7 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6777
6777
|
}
|
|
6778
6778
|
const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: failureCode, error: failureReason } : attempt);
|
|
6779
6779
|
const page2 = {
|
|
6780
|
-
url,
|
|
6780
|
+
url: url2,
|
|
6781
6781
|
status,
|
|
6782
6782
|
via,
|
|
6783
6783
|
contentType,
|
|
@@ -6785,7 +6785,7 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6785
6785
|
sizeBytes,
|
|
6786
6786
|
redirectUrl,
|
|
6787
6787
|
redirectStatus,
|
|
6788
|
-
finalUrl: documentUrl ===
|
|
6788
|
+
finalUrl: documentUrl === url2 ? null : documentUrl,
|
|
6789
6789
|
lastModified,
|
|
6790
6790
|
xRobotsTag,
|
|
6791
6791
|
title,
|
|
@@ -6822,7 +6822,7 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6822
6822
|
discoveryLinks,
|
|
6823
6823
|
bodyMarkdown,
|
|
6824
6824
|
schema,
|
|
6825
|
-
pageId: createHash("sha256").update(
|
|
6825
|
+
pageId: createHash("sha256").update(url2).digest("hex"),
|
|
6826
6826
|
htmlSha256: createHash("sha256").update(html).digest("hex"),
|
|
6827
6827
|
markdownSha256: createHash("sha256").update(bodyMarkdown).digest("hex"),
|
|
6828
6828
|
contentKind: extractionSucceeded ? "text_page" : "insufficient_text",
|
|
@@ -6875,12 +6875,12 @@ function isPageExtractionSuccessful(page2) {
|
|
|
6875
6875
|
if (page2.extractionStatus != null) return page2.extractionStatus === "successful";
|
|
6876
6876
|
return page2.status != null && page2.status >= 200 && page2.status < 300 && page2.wordCount > 0;
|
|
6877
6877
|
}
|
|
6878
|
-
async function fetchPagePlain(
|
|
6878
|
+
async function fetchPagePlain(url2) {
|
|
6879
6879
|
const startedAt = Date.now();
|
|
6880
|
-
const checkedInitial = await validatePublicHttpUrl(
|
|
6880
|
+
const checkedInitial = await validatePublicHttpUrl(url2, { field: "page URL" });
|
|
6881
6881
|
if (checkedInitial.error || !checkedInitial.parsed) {
|
|
6882
6882
|
const attempt = failedAttempt("fetch", null, Date.now() - startedAt, "unsafe_url", checkedInitial.error ?? "Page URL was rejected.");
|
|
6883
|
-
return emptyPageData(
|
|
6883
|
+
return emptyPageData(url2, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
|
|
6884
6884
|
}
|
|
6885
6885
|
let target = checkedInitial.parsed.href;
|
|
6886
6886
|
const requestedHost = checkedInitial.parsed.hostname.replace(/^www\./, "").toLowerCase();
|
|
@@ -6898,17 +6898,17 @@ async function fetchPagePlain(url) {
|
|
|
6898
6898
|
const location2 = res.headers.get("location");
|
|
6899
6899
|
if (!location2) {
|
|
6900
6900
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "redirect_missing_location", `HTTP ${res.status} redirect did not include a Location header.`);
|
|
6901
|
-
return emptyPageData(
|
|
6901
|
+
return emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6902
6902
|
}
|
|
6903
6903
|
const redirectTarget = new URL(location2, target);
|
|
6904
6904
|
if (redirectTarget.hostname.replace(/^www\./, "").toLowerCase() !== requestedHost) {
|
|
6905
6905
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "cross_site_redirect", "Cross-site redirects are not followed during same-site extraction.");
|
|
6906
|
-
return emptyPageData(
|
|
6906
|
+
return emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6907
6907
|
}
|
|
6908
6908
|
const checked = await validatePublicHttpUrl(redirectTarget.href, { field: "redirect URL" });
|
|
6909
6909
|
if (checked.error || !checked.parsed) {
|
|
6910
6910
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "unsafe_redirect", checked.error ?? "Redirect URL was rejected.");
|
|
6911
|
-
return emptyPageData(
|
|
6911
|
+
return emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6912
6912
|
}
|
|
6913
6913
|
redirectStatus ??= res.status;
|
|
6914
6914
|
target = checked.parsed.href;
|
|
@@ -6916,12 +6916,12 @@ async function fetchPagePlain(url) {
|
|
|
6916
6916
|
}
|
|
6917
6917
|
if (!res.ok) {
|
|
6918
6918
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, `http_${res.status}`, `Plain fetch returned HTTP ${res.status}.`);
|
|
6919
|
-
return emptyPageData(
|
|
6919
|
+
return emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6920
6920
|
}
|
|
6921
6921
|
const contentType = res.headers.get("content-type") ?? "";
|
|
6922
6922
|
if (contentType && !/html|xhtml/i.test(contentType)) {
|
|
6923
6923
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "non_html_response", `Plain fetch returned ${contentType}, not HTML.`);
|
|
6924
|
-
const page2 = emptyPageData(
|
|
6924
|
+
const page2 = emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6925
6925
|
page2.contentType = contentType;
|
|
6926
6926
|
page2.contentKind = "non_html";
|
|
6927
6927
|
return page2;
|
|
@@ -6929,27 +6929,27 @@ async function fetchPagePlain(url) {
|
|
|
6929
6929
|
const html = await readResponseTextWithLimit(res, MAX_PAGE_HTML_BYTES, "Page HTML");
|
|
6930
6930
|
if (!html.trim()) {
|
|
6931
6931
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "empty_html", "Plain fetch returned an empty HTML response.");
|
|
6932
|
-
return emptyPageData(
|
|
6932
|
+
return emptyPageData(url2, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6933
6933
|
}
|
|
6934
6934
|
const attempt2 = successfulAttempt("fetch", res.status, responseTimeMs2);
|
|
6935
|
-
const replay = parseWaybackReplayUrl(
|
|
6936
|
-
return parsePageData(
|
|
6935
|
+
const replay = parseWaybackReplayUrl(url2);
|
|
6936
|
+
return parsePageData(url2, html, res.status, "fetch", {
|
|
6937
6937
|
headers,
|
|
6938
6938
|
responseTimeMs: responseTimeMs2,
|
|
6939
|
-
redirectUrl: target ===
|
|
6939
|
+
redirectUrl: target === url2 ? null : target,
|
|
6940
6940
|
redirectStatus,
|
|
6941
6941
|
documentUrl: replay?.originalUrl ?? target
|
|
6942
6942
|
}, [attempt2]);
|
|
6943
6943
|
}
|
|
6944
6944
|
const responseTimeMs = Date.now() - startedAt;
|
|
6945
6945
|
const attempt = failedAttempt("fetch", null, responseTimeMs, "too_many_redirects", "Plain fetch exceeded five redirects.");
|
|
6946
|
-
return emptyPageData(
|
|
6946
|
+
return emptyPageData(url2, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
|
|
6947
6947
|
} catch (error) {
|
|
6948
6948
|
const responseTimeMs = Date.now() - startedAt;
|
|
6949
6949
|
const code = failureCodeForError(error);
|
|
6950
6950
|
const message = compactError(error) || "Plain fetch failed.";
|
|
6951
6951
|
const attempt = failedAttempt("fetch", null, responseTimeMs, code, message);
|
|
6952
|
-
return emptyPageData(
|
|
6952
|
+
return emptyPageData(url2, null, "fetch", { code, message, attempts: [attempt] });
|
|
6953
6953
|
}
|
|
6954
6954
|
}
|
|
6955
6955
|
async function mapWithConcurrency(items, concurrency, fn) {
|
|
@@ -6966,16 +6966,16 @@ async function mapWithConcurrency(items, concurrency, fn) {
|
|
|
6966
6966
|
}
|
|
6967
6967
|
async function extractPagesRotating(urls, opts) {
|
|
6968
6968
|
const uniqueUrls = [...new Set(urls)];
|
|
6969
|
-
const initialPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (
|
|
6970
|
-
const checked = await validatePublicHttpUrl(
|
|
6969
|
+
const initialPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url2) => {
|
|
6970
|
+
const checked = await validatePublicHttpUrl(url2, { field: "page URL" });
|
|
6971
6971
|
if (checked.error || !checked.parsed) {
|
|
6972
|
-
if (!opts.forceBrowserRender) return fetchPagePlain(
|
|
6972
|
+
if (!opts.forceBrowserRender) return fetchPagePlain(url2);
|
|
6973
6973
|
const attempt2 = failedAttempt("browser", null, 0, "unsafe_url", checked.error ?? "Page URL was rejected.");
|
|
6974
|
-
return emptyPageData(
|
|
6974
|
+
return emptyPageData(url2, null, "browser", { code: attempt2.errorCode, message: attempt2.error, attempts: [attempt2] });
|
|
6975
6975
|
}
|
|
6976
6976
|
if (normalizeUrl(checked.parsed.href, checked.parsed.href)) {
|
|
6977
|
-
if (!opts.forceBrowserRender) return fetchPagePlain(
|
|
6978
|
-
return emptyPageData(
|
|
6977
|
+
if (!opts.forceBrowserRender) return fetchPagePlain(url2);
|
|
6978
|
+
return emptyPageData(url2, null, "browser", {
|
|
6979
6979
|
code: "browser_pending",
|
|
6980
6980
|
message: "Awaiting forced JavaScript rendering.",
|
|
6981
6981
|
attempts: []
|
|
@@ -6983,7 +6983,7 @@ async function extractPagesRotating(urls, opts) {
|
|
|
6983
6983
|
}
|
|
6984
6984
|
const method = opts.forceBrowserRender ? "browser" : "fetch";
|
|
6985
6985
|
const attempt = failedAttempt(method, null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
|
|
6986
|
-
const page2 = emptyPageData(
|
|
6986
|
+
const page2 = emptyPageData(url2, null, method, { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
|
|
6987
6987
|
page2.contentKind = "excluded_url";
|
|
6988
6988
|
return page2;
|
|
6989
6989
|
});
|
|
@@ -7044,15 +7044,15 @@ async function extractPagesRotating(urls, opts) {
|
|
|
7044
7044
|
}
|
|
7045
7045
|
}
|
|
7046
7046
|
if (opts.forceBrowserRender && !opts.kernelApiKey) {
|
|
7047
|
-
for (const
|
|
7048
|
-
byUrl.set(
|
|
7047
|
+
for (const url2 of browserTargets) {
|
|
7048
|
+
byUrl.set(url2, emptyPageData(url2, null, "browser", {
|
|
7049
7049
|
code: "browser_service_unconfigured",
|
|
7050
7050
|
message: "Forced JavaScript rendering requires the hosted browser service.",
|
|
7051
7051
|
attempts: [failedAttempt("browser", null, 0, "browser_service_unconfigured", "Forced JavaScript rendering requires the hosted browser service.")]
|
|
7052
7052
|
}));
|
|
7053
7053
|
}
|
|
7054
7054
|
}
|
|
7055
|
-
return urls.map((
|
|
7055
|
+
return urls.map((url2) => byUrl.get(url2) ?? emptyPageData(url2, null, opts.forceBrowserRender ? "browser" : "fetch", {
|
|
7056
7056
|
code: "result_missing",
|
|
7057
7057
|
message: "No extraction result was recorded."
|
|
7058
7058
|
}));
|
|
@@ -7084,8 +7084,8 @@ async function discoverUrlsWithSitemap(startUrl, maxPages, kernelApiKey, options
|
|
|
7084
7084
|
const spider = await spiderSite({ startUrl: boundedStartUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
|
|
7085
7085
|
return { urls: spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url), sitemapUrls: spider.sitemapUrls };
|
|
7086
7086
|
}
|
|
7087
|
-
async function fetchAndParse(
|
|
7088
|
-
return (await extractPagesRotating([
|
|
7087
|
+
async function fetchAndParse(url2, kernelApiKey) {
|
|
7088
|
+
return (await extractPagesRotating([url2], { kernelApiKey, concurrency: 1, urlsPerBrowser: 1 }))[0];
|
|
7089
7089
|
}
|
|
7090
7090
|
async function runWithConcurrency(items, concurrency, fn) {
|
|
7091
7091
|
let i = 0;
|
|
@@ -7164,8 +7164,8 @@ async function extractSite(opts) {
|
|
|
7164
7164
|
sitemapUrls = spider.sitemapUrls;
|
|
7165
7165
|
const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
|
|
7166
7166
|
browserRetries = spider.browserRetries;
|
|
7167
|
-
await runWithConcurrency(urlsToExtract, concurrency, async (
|
|
7168
|
-
const data = await fetchAndParse(
|
|
7167
|
+
await runWithConcurrency(urlsToExtract, concurrency, async (url2) => {
|
|
7168
|
+
const data = await fetchAndParse(url2, opts.kernelApiKey);
|
|
7169
7169
|
if (data.via === "browser") browserRetries++;
|
|
7170
7170
|
pages.push(data);
|
|
7171
7171
|
});
|
|
@@ -7217,14 +7217,14 @@ var inngest = new Inngest({ id: "mcp-scraper" });
|
|
|
7217
7217
|
// src/lib/llm-parse-with-retry.ts
|
|
7218
7218
|
import Anthropic from "@anthropic-ai/sdk";
|
|
7219
7219
|
function extractJson(raw) {
|
|
7220
|
-
let
|
|
7221
|
-
|
|
7222
|
-
if (
|
|
7223
|
-
const objStart =
|
|
7224
|
-
const arrStart =
|
|
7220
|
+
let text2 = raw.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
7221
|
+
text2 = text2.replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/, "").trim();
|
|
7222
|
+
if (text2.startsWith("{") || text2.startsWith("[")) return text2;
|
|
7223
|
+
const objStart = text2.indexOf("{");
|
|
7224
|
+
const arrStart = text2.indexOf("[");
|
|
7225
7225
|
let start = -1;
|
|
7226
7226
|
let open, close;
|
|
7227
|
-
if (objStart === -1 && arrStart === -1) return
|
|
7227
|
+
if (objStart === -1 && arrStart === -1) return text2;
|
|
7228
7228
|
if (objStart === -1) {
|
|
7229
7229
|
start = arrStart;
|
|
7230
7230
|
open = "[";
|
|
@@ -7245,8 +7245,8 @@ function extractJson(raw) {
|
|
|
7245
7245
|
let depth = 0;
|
|
7246
7246
|
let inStr = false;
|
|
7247
7247
|
let escape = false;
|
|
7248
|
-
for (let i = start; i <
|
|
7249
|
-
const ch =
|
|
7248
|
+
for (let i = start; i < text2.length; i++) {
|
|
7249
|
+
const ch = text2[i];
|
|
7250
7250
|
if (escape) {
|
|
7251
7251
|
escape = false;
|
|
7252
7252
|
continue;
|
|
@@ -7262,10 +7262,10 @@ function extractJson(raw) {
|
|
|
7262
7262
|
if (inStr) continue;
|
|
7263
7263
|
if (ch === open) depth++;
|
|
7264
7264
|
else if (ch === close) {
|
|
7265
|
-
if (--depth === 0) return
|
|
7265
|
+
if (--depth === 0) return text2.slice(start, i + 1);
|
|
7266
7266
|
}
|
|
7267
7267
|
}
|
|
7268
|
-
return
|
|
7268
|
+
return text2.slice(start);
|
|
7269
7269
|
}
|
|
7270
7270
|
var DeepInfraLlmClient = class {
|
|
7271
7271
|
apiKey;
|
|
@@ -7395,13 +7395,13 @@ async function llmParseWithRetry(schema, promptFn, llm) {
|
|
|
7395
7395
|
|
|
7396
7396
|
// src/lib/http-client.ts
|
|
7397
7397
|
var FetchHttpClient = class {
|
|
7398
|
-
async get(
|
|
7399
|
-
const response = await fetch(
|
|
7398
|
+
async get(url2, headers) {
|
|
7399
|
+
const response = await fetch(url2, { headers });
|
|
7400
7400
|
const body = await response.text();
|
|
7401
7401
|
return { status: response.status, body };
|
|
7402
7402
|
}
|
|
7403
|
-
async post(
|
|
7404
|
-
const response = await fetch(
|
|
7403
|
+
async post(url2, body, headers) {
|
|
7404
|
+
const response = await fetch(url2, {
|
|
7405
7405
|
method: "POST",
|
|
7406
7406
|
headers: { "Content-Type": "application/json", ...headers },
|
|
7407
7407
|
body: JSON.stringify(body)
|
|
@@ -9567,13 +9567,13 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9567
9567
|
const queue = [];
|
|
9568
9568
|
const seen = /* @__PURE__ */ new Set();
|
|
9569
9569
|
let seenBytes = 0;
|
|
9570
|
-
for (const
|
|
9571
|
-
const normalized = norm(
|
|
9570
|
+
for (const url2 of seed.urls) {
|
|
9571
|
+
const normalized = norm(url2);
|
|
9572
9572
|
const bytes = Buffer.byteLength(normalized);
|
|
9573
9573
|
if (normalized.length > MAX_DISCOVERED_URL_CHARS || bytes > MAX_DISCOVERED_URL_CHARS || seen.has(normalized) || seenBytes + bytes > MAX_DISCOVERY_FRONTIER_BYTES) continue;
|
|
9574
9574
|
seen.add(normalized);
|
|
9575
9575
|
seenBytes += bytes;
|
|
9576
|
-
queue.push(
|
|
9576
|
+
queue.push(url2);
|
|
9577
9577
|
}
|
|
9578
9578
|
let cursor = 0;
|
|
9579
9579
|
for (let i = 0; cursor < maxPages && cursor < queue.length; i++) {
|
|
@@ -9824,8 +9824,8 @@ function buildGoogleOrganicSearchUrl(input) {
|
|
|
9824
9824
|
const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
|
|
9825
9825
|
return `https://www.google.com/search?${params.toString()}`;
|
|
9826
9826
|
}
|
|
9827
|
-
function mapsIdentifiersFromUrl(
|
|
9828
|
-
const fid =
|
|
9827
|
+
function mapsIdentifiersFromUrl(url2) {
|
|
9828
|
+
const fid = url2.match(/!1s(0x[0-9a-f]+):(0x[0-9a-f]+)/i) ?? url2.match(/(0x[0-9a-f]+):(0x[0-9a-f]+)/i);
|
|
9829
9829
|
if (fid) {
|
|
9830
9830
|
try {
|
|
9831
9831
|
return { cid: `${fid[1]}:${fid[2]}`, cidDecimal: BigInt(fid[2]).toString() };
|
|
@@ -9834,7 +9834,7 @@ function mapsIdentifiersFromUrl(url) {
|
|
|
9834
9834
|
}
|
|
9835
9835
|
}
|
|
9836
9836
|
try {
|
|
9837
|
-
const decimal = new URL(
|
|
9837
|
+
const decimal = new URL(url2).searchParams.get("cid");
|
|
9838
9838
|
return { cid: null, cidDecimal: decimal && /^\d+$/.test(decimal) ? decimal : null };
|
|
9839
9839
|
} catch {
|
|
9840
9840
|
return { cid: null, cidDecimal: null };
|
|
@@ -9901,8 +9901,8 @@ var MapsSearchExtractor = class {
|
|
|
9901
9901
|
}
|
|
9902
9902
|
async detectBlock(page2) {
|
|
9903
9903
|
return page2.evaluate(() => {
|
|
9904
|
-
const
|
|
9905
|
-
return /unusual traffic|captcha|recaptcha|about this page/i.test(
|
|
9904
|
+
const text2 = document.body.innerText.slice(0, 2e3);
|
|
9905
|
+
return /unusual traffic|captcha|recaptcha|about this page/i.test(text2) || /\/sorry\//.test(location.href);
|
|
9906
9906
|
});
|
|
9907
9907
|
}
|
|
9908
9908
|
async collectResults(page2, options) {
|
|
@@ -9949,14 +9949,14 @@ var MapsSearchExtractor = class {
|
|
|
9949
9949
|
const fallbackLocation = location2 ?? "";
|
|
9950
9950
|
return page2.evaluate((locationHint) => {
|
|
9951
9951
|
const normalize = (value) => {
|
|
9952
|
-
const
|
|
9953
|
-
return
|
|
9952
|
+
const text2 = value?.replace(/\s+/g, " ").trim() ?? "";
|
|
9953
|
+
return text2 || null;
|
|
9954
9954
|
};
|
|
9955
9955
|
const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
|
|
9956
9956
|
const addressPattern = /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i;
|
|
9957
9957
|
const headings = Array.from(document.querySelectorAll('div[role="heading"]')).filter((heading) => {
|
|
9958
|
-
const
|
|
9959
|
-
return Boolean(
|
|
9958
|
+
const text2 = normalize(heading.textContent);
|
|
9959
|
+
return Boolean(text2) && !/^(?:businesses|places|sponsored|people also ask|discussions and forums|more businesses|more places|custom date range|choose what you)/i.test(text2 ?? "");
|
|
9960
9960
|
});
|
|
9961
9961
|
const seen = /* @__PURE__ */ new Set();
|
|
9962
9962
|
const out = [];
|
|
@@ -10038,19 +10038,19 @@ var MapsSearchExtractor = class {
|
|
|
10038
10038
|
});
|
|
10039
10039
|
if (await dialog.count() === 0 || !await dialog.isVisible().catch(() => false)) return null;
|
|
10040
10040
|
const details = await dialog.evaluate((root, includeServicesValue) => {
|
|
10041
|
-
const
|
|
10041
|
+
const text2 = (value) => value?.replace(/\s+/g, " ").trim() ?? "";
|
|
10042
10042
|
const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
|
|
10043
10043
|
const buttons = Array.from(root.querySelectorAll("button"));
|
|
10044
|
-
const buttonTexts = buttons.map((button) =>
|
|
10044
|
+
const buttonTexts = buttons.map((button) => text2(button.innerText)).filter(Boolean);
|
|
10045
10045
|
const servicesText = includeServicesValue ? buttonTexts.find((value) => /^services\s*:/i.test(value)) : null;
|
|
10046
10046
|
const services = servicesText ? [...new Set(servicesText.replace(/^services\s*:\s*/i, "").split(/\s*[,·]\s*/).map((value) => value.trim()).filter(Boolean))] : [];
|
|
10047
10047
|
const areaText = includeServicesValue ? buttonTexts.find((value) => /^serves\b/i.test(value)) : null;
|
|
10048
10048
|
const areasServed = areaText ? [areaText.replace(/^serves\s*/i, "").trim()].filter(Boolean) : [];
|
|
10049
10049
|
const externalLinks = Array.from(root.querySelectorAll("a[href]"));
|
|
10050
|
-
const website = externalLinks.find((link) => /^website$/i.test(
|
|
10050
|
+
const website = externalLinks.find((link) => /^website$/i.test(text2(link.innerText)) || /^website$/i.test(link.getAttribute("aria-label") ?? ""));
|
|
10051
10051
|
const mapLink = externalLinks.find((link) => /google\.[^/]+\/maps|\/maps\//i.test(link.href));
|
|
10052
|
-
const addressLink = externalLinks.find((link) => /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i.test(
|
|
10053
|
-
const allText =
|
|
10052
|
+
const addressLink = externalLinks.find((link) => /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i.test(text2(link.innerText)));
|
|
10053
|
+
const allText = text2(root.innerText);
|
|
10054
10054
|
const rating = allText.match(/\b(?:rated\s+)?(\d(?:\.\d)?)\s+out of 5\b/i)?.[1] ?? allText.match(/\b(\d(?:\.\d)?)\s+rated\b/i)?.[1] ?? null;
|
|
10055
10055
|
const reviewCount = allText.match(/([\d,]+)\s+(?:user\s+)?reviews?\b/i)?.[1]?.replace(/,/g, "") ?? null;
|
|
10056
10056
|
const phoneButton = buttonTexts.find((value) => /^call\b/i.test(value) || phonePattern.test(value)) ?? "";
|
|
@@ -10077,7 +10077,7 @@ var MapsSearchExtractor = class {
|
|
|
10077
10077
|
rating,
|
|
10078
10078
|
reviewCount,
|
|
10079
10079
|
category: null,
|
|
10080
|
-
address:
|
|
10080
|
+
address: text2(addressLink?.innerText) || null,
|
|
10081
10081
|
phone,
|
|
10082
10082
|
websiteUrl: website?.href ?? null,
|
|
10083
10083
|
hoursStatus,
|
|
@@ -10644,11 +10644,11 @@ async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
|
10644
10644
|
return { path: null, groups: /* @__PURE__ */ new Map(), source: "none", version: null, updatedAt: null };
|
|
10645
10645
|
}
|
|
10646
10646
|
const groups = /* @__PURE__ */ new Map();
|
|
10647
|
-
for (const
|
|
10648
|
-
const state = (
|
|
10649
|
-
const zip = (
|
|
10650
|
-
const city = (
|
|
10651
|
-
const county = (
|
|
10647
|
+
for (const record2 of records) {
|
|
10648
|
+
const state = (record2.state_abbr ?? record2.state ?? "").trim().toUpperCase();
|
|
10649
|
+
const zip = (record2.zipcode ?? record2.zip ?? record2.zip_code ?? "").trim();
|
|
10650
|
+
const city = (record2.city ?? "").trim();
|
|
10651
|
+
const county = (record2.county ?? "").trim();
|
|
10652
10652
|
if (state !== stateAbbr || !zip || !city) continue;
|
|
10653
10653
|
const key2 = normalizeCityKey(city);
|
|
10654
10654
|
if (!groups.has(key2)) groups.set(key2, { zips: /* @__PURE__ */ new Set(), counties: /* @__PURE__ */ new Set() });
|
|
@@ -10727,10 +10727,10 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10727
10727
|
};
|
|
10728
10728
|
const cityQuery = normalizeCityKey(options.cityQuery ?? "");
|
|
10729
10729
|
const zipFilter = options.zip?.trim();
|
|
10730
|
-
const markets = records.filter((
|
|
10731
|
-
const population = numberOrNull(
|
|
10730
|
+
const markets = records.filter((record2) => record2.SUMLEV === "162").map((record2) => {
|
|
10731
|
+
const population = numberOrNull(record2[populationField]);
|
|
10732
10732
|
if (population === null || population < options.minPopulation) return null;
|
|
10733
|
-
const censusName =
|
|
10733
|
+
const censusName = record2.NAME?.trim() ?? "";
|
|
10734
10734
|
if (!censusName) return null;
|
|
10735
10735
|
const city = displayCityFromCensus(censusName);
|
|
10736
10736
|
const zipGroup = zipData.groups.get(normalizeCityKey(city));
|
|
@@ -10742,7 +10742,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10742
10742
|
censusName,
|
|
10743
10743
|
population,
|
|
10744
10744
|
populationYear: options.populationYear,
|
|
10745
|
-
estimatesBase2020: numberOrNull(
|
|
10745
|
+
estimatesBase2020: numberOrNull(record2.ESTIMATESBASE2020),
|
|
10746
10746
|
zips: zipGroup ? [...zipGroup.zips].sort() : [],
|
|
10747
10747
|
counties: zipGroup ? [...zipGroup.counties].sort() : []
|
|
10748
10748
|
};
|
|
@@ -11999,11 +11999,11 @@ import ExcelJS from "exceljs";
|
|
|
11999
11999
|
import Papa from "papaparse";
|
|
12000
12000
|
var OCCURRENCE_DISCLAIMER = "Public occurrence is not email deliverability, consent, or mailbox verification.";
|
|
12001
12001
|
function csvSafeValue(value) {
|
|
12002
|
-
const
|
|
12003
|
-
const first =
|
|
12004
|
-
if (first >= 0 && ["=", "+", "-", "@"].includes(
|
|
12002
|
+
const text2 = value == null ? "" : String(value);
|
|
12003
|
+
const first = text2.search(/\S/);
|
|
12004
|
+
if (first >= 0 && ["=", "+", "-", "@"].includes(text2[first])) return `${text2.slice(0, first)}'${text2.slice(first)}`;
|
|
12005
12005
|
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
12006
|
-
return
|
|
12006
|
+
return text2;
|
|
12007
12007
|
}
|
|
12008
12008
|
function renderLeadListCsv(table) {
|
|
12009
12009
|
if (!table.headers.length) throw new Error("Lead list output requires at least one header");
|
|
@@ -12879,8 +12879,8 @@ var MapsNavigator = class {
|
|
|
12879
12879
|
}
|
|
12880
12880
|
async isGoogleChallenge() {
|
|
12881
12881
|
return this.page.evaluate(() => {
|
|
12882
|
-
const
|
|
12883
|
-
return /unusual traffic|captcha|recaptcha|about this page/i.test(
|
|
12882
|
+
const text2 = (document.body?.innerText ?? "").slice(0, 2e3);
|
|
12883
|
+
return /unusual traffic|captcha|recaptcha|about this page/i.test(text2) || /\/sorry\//.test(location.href);
|
|
12884
12884
|
}).catch(() => true);
|
|
12885
12885
|
}
|
|
12886
12886
|
async waitForNamedBusinessCard(businessName, timeout) {
|
|
@@ -13087,9 +13087,9 @@ var MAX_SCROLL_ROUNDS2 = 80;
|
|
|
13087
13087
|
var OWNER_DISCOVERY_LIMIT = 500;
|
|
13088
13088
|
function mapsImageIdentity(rawUrl) {
|
|
13089
13089
|
try {
|
|
13090
|
-
const
|
|
13091
|
-
const path5 =
|
|
13092
|
-
return `${
|
|
13090
|
+
const url2 = new URL(rawUrl);
|
|
13091
|
+
const path5 = url2.hostname.toLowerCase().endsWith("googleusercontent.com") ? url2.pathname.replace(/=[^/]+$/, "") : url2.pathname;
|
|
13092
|
+
return `${url2.hostname.toLowerCase()}${path5}`;
|
|
13093
13093
|
} catch {
|
|
13094
13094
|
return rawUrl.split("=")[0] ?? rawUrl;
|
|
13095
13095
|
}
|
|
@@ -13469,10 +13469,10 @@ var MapsExtractor = class {
|
|
|
13469
13469
|
await this.driver.close();
|
|
13470
13470
|
}
|
|
13471
13471
|
}
|
|
13472
|
-
parseUrlIdentifiers(
|
|
13473
|
-
const pin =
|
|
13474
|
-
const fid =
|
|
13475
|
-
const kg = decodeURIComponent(
|
|
13472
|
+
parseUrlIdentifiers(url2) {
|
|
13473
|
+
const pin = url2.match(/!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)/);
|
|
13474
|
+
const fid = url2.match(/!1s(0x[0-9a-f]+):(0x[0-9a-f]+)/i);
|
|
13475
|
+
const kg = decodeURIComponent(url2).match(/!16s(\/g\/[a-z0-9_]+)/i);
|
|
13476
13476
|
let cidDecimal = null;
|
|
13477
13477
|
if (fid) {
|
|
13478
13478
|
try {
|
|
@@ -13626,20 +13626,20 @@ var MapsExtractor = class {
|
|
|
13626
13626
|
if (el2.tagName === "H2") {
|
|
13627
13627
|
currentSection = el2.innerText?.trim() || currentSection;
|
|
13628
13628
|
} else if (el2.tagName === "LI") {
|
|
13629
|
-
const
|
|
13630
|
-
const key2 = `${currentSection}|${
|
|
13631
|
-
if (
|
|
13629
|
+
const text2 = el2.innerText?.replace(/^[\s\S]*?\n/, "").trim();
|
|
13630
|
+
const key2 = `${currentSection}|${text2}`;
|
|
13631
|
+
if (text2 && !seen.has(key2)) {
|
|
13632
13632
|
seen.add(key2);
|
|
13633
|
-
results.push({ section: currentSection, attribute:
|
|
13633
|
+
results.push({ section: currentSection, attribute: text2 });
|
|
13634
13634
|
}
|
|
13635
13635
|
}
|
|
13636
13636
|
});
|
|
13637
13637
|
main.querySelectorAll('[href*="businessownership"], [data-item-id^="place-info-links:"]').forEach((el2) => {
|
|
13638
|
-
const
|
|
13639
|
-
const key2 = `Ownership|${
|
|
13640
|
-
if (
|
|
13638
|
+
const text2 = el2.innerText?.trim();
|
|
13639
|
+
const key2 = `Ownership|${text2}`;
|
|
13640
|
+
if (text2 && !seen.has(key2)) {
|
|
13641
13641
|
seen.add(key2);
|
|
13642
|
-
results.push({ section: "Ownership", attribute:
|
|
13642
|
+
results.push({ section: "Ownership", attribute: text2 });
|
|
13643
13643
|
}
|
|
13644
13644
|
});
|
|
13645
13645
|
return results;
|
|
@@ -13914,14 +13914,14 @@ function splitAddress(raw, state) {
|
|
|
13914
13914
|
}
|
|
13915
13915
|
function absoluteImageUrl(src, pageUrl) {
|
|
13916
13916
|
try {
|
|
13917
|
-
const
|
|
13918
|
-
if (!/^https?:$/.test(
|
|
13919
|
-
if (/\.(svg|ico)(?:$|\?)/i.test(
|
|
13920
|
-
if (/(favicon|sprite|spacer|tracking|pixel|logo(?:[-_.]|$)|placeholder|\/cta\/|no-cache\.hubspot\.com\/cta)/i.test(
|
|
13921
|
-
const width = Number.parseInt(
|
|
13922
|
-
const height = Number.parseInt(
|
|
13917
|
+
const url2 = new URL(src.replace(/&/gi, "&"), pageUrl);
|
|
13918
|
+
if (!/^https?:$/.test(url2.protocol)) return null;
|
|
13919
|
+
if (/\.(svg|ico)(?:$|\?)/i.test(url2.pathname)) return null;
|
|
13920
|
+
if (/(favicon|sprite|spacer|tracking|pixel|logo(?:[-_.]|$)|placeholder|\/cta\/|no-cache\.hubspot\.com\/cta)/i.test(url2.href)) return null;
|
|
13921
|
+
const width = Number.parseInt(url2.searchParams.get("width") ?? "", 10);
|
|
13922
|
+
const height = Number.parseInt(url2.searchParams.get("height") ?? "", 10);
|
|
13923
13923
|
if (Number.isFinite(width) && width < 240 || Number.isFinite(height) && height < 180) return null;
|
|
13924
|
-
return
|
|
13924
|
+
return url2.href;
|
|
13925
13925
|
} catch {
|
|
13926
13926
|
return null;
|
|
13927
13927
|
}
|
|
@@ -13990,10 +13990,10 @@ function serviceAreaCandidates(pages, state) {
|
|
|
13990
13990
|
return normalized.filter((item) => item.includes(",") || !normalized.some((other) => other.toLowerCase() === `${item}${stateSuffix}`.toLowerCase()));
|
|
13991
13991
|
}
|
|
13992
13992
|
function likelyOfferingHeading(value) {
|
|
13993
|
-
const
|
|
13994
|
-
if (
|
|
13995
|
-
if (/^(?:\d|are|can|careers?|do|does|how|is|jobs?|should|what|when|where|which|why|who|our|your|choose|schedule|never|helpful|related)\b/i.test(
|
|
13996
|
-
if (/\?|\b(?:faq|frequently asked|resources?|articles?|years? serving|licensed|insured|guarantee|headline|right .* for you|company you can count on|sales process|best .* companies|codes?|make the best decisions|select shingle master)\b/i.test(
|
|
13993
|
+
const text2 = value.replace(/\s+/g, " ").trim();
|
|
13994
|
+
if (text2.length < 3 || text2.length > 70 || GENERIC_HEADINGS.test(text2)) return false;
|
|
13995
|
+
if (/^(?:\d|are|can|careers?|do|does|how|is|jobs?|should|what|when|where|which|why|who|our|your|choose|schedule|never|helpful|related)\b/i.test(text2)) return false;
|
|
13996
|
+
if (/\?|\b(?:faq|frequently asked|resources?|articles?|years? serving|licensed|insured|guarantee|headline|right .* for you|company you can count on|sales process|best .* companies|codes?|make the best decisions|select shingle master)\b/i.test(text2)) return false;
|
|
13997
13997
|
return true;
|
|
13998
13998
|
}
|
|
13999
13999
|
function providerOfferingCandidates(pages) {
|
|
@@ -14025,17 +14025,17 @@ function reviewRating(review) {
|
|
|
14025
14025
|
return Number.isFinite(parsed) ? Math.max(1, Math.min(5, parsed)) : 0;
|
|
14026
14026
|
}
|
|
14027
14027
|
function reviewThemes(review) {
|
|
14028
|
-
const
|
|
14029
|
-
const matches = REVIEW_THEME_RULES.filter(([, pattern]) => pattern.test(
|
|
14028
|
+
const text2 = `${review.text ?? ""} ${review.ownerResponse ?? ""}`;
|
|
14029
|
+
const matches = REVIEW_THEME_RULES.filter(([, pattern]) => pattern.test(text2)).map(([label]) => label);
|
|
14030
14030
|
return matches.length ? matches : ["Overall experience"];
|
|
14031
14031
|
}
|
|
14032
14032
|
function buildReviews(maps, collectedAt) {
|
|
14033
14033
|
const seen = /* @__PURE__ */ new Set();
|
|
14034
14034
|
const reviews = [];
|
|
14035
14035
|
for (const review of maps?.reviews ?? []) {
|
|
14036
|
-
const
|
|
14037
|
-
if (!
|
|
14038
|
-
const key2 = review.reviewId || digest(`${review.author}|${review.date}|${
|
|
14036
|
+
const text2 = review.text?.trim();
|
|
14037
|
+
if (!text2) continue;
|
|
14038
|
+
const key2 = review.reviewId || digest(`${review.author}|${review.date}|${text2}`.toLowerCase());
|
|
14039
14039
|
if (seen.has(key2)) continue;
|
|
14040
14040
|
seen.add(key2);
|
|
14041
14041
|
reviews.push({
|
|
@@ -14046,7 +14046,7 @@ function buildReviews(maps, collectedAt) {
|
|
|
14046
14046
|
rating: reviewRating(review),
|
|
14047
14047
|
ratingScale: 5,
|
|
14048
14048
|
publishedAt: review.date || "",
|
|
14049
|
-
text,
|
|
14049
|
+
text: text2,
|
|
14050
14050
|
...review.ownerResponse ? { providerResponse: { text: review.ownerResponse } } : {},
|
|
14051
14051
|
themes: reviewThemes(review),
|
|
14052
14052
|
collectedAt
|
|
@@ -15744,9 +15744,9 @@ import { provisionDefaultsTool } from "mcpscraper-memory-tools/tools/vaults/prov
|
|
|
15744
15744
|
import { setScheduleEntitlementTool } from "mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";
|
|
15745
15745
|
var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
|
|
15746
15746
|
var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
|
|
15747
|
-
function parseMemoryRpcBody(
|
|
15748
|
-
const dataLine =
|
|
15749
|
-
const payload = dataLine ? dataLine.slice(5).trim() :
|
|
15747
|
+
function parseMemoryRpcBody(text2) {
|
|
15748
|
+
const dataLine = text2.split("\n").find((line) => line.startsWith("data:"));
|
|
15749
|
+
const payload = dataLine ? dataLine.slice(5).trim() : text2;
|
|
15750
15750
|
return JSON.parse(payload);
|
|
15751
15751
|
}
|
|
15752
15752
|
function isMemoryOperator(email) {
|
|
@@ -15790,9 +15790,9 @@ async function memoryCall(toolName, args, userMemoryKey) {
|
|
|
15790
15790
|
params: { name: toolName, arguments: args }
|
|
15791
15791
|
})
|
|
15792
15792
|
});
|
|
15793
|
-
const
|
|
15793
|
+
const text2 = await res.text();
|
|
15794
15794
|
if (!res.ok) return { ok: false, error: `memory ${toolName} failed (${res.status})` };
|
|
15795
|
-
const body = parseMemoryRpcBody(
|
|
15795
|
+
const body = parseMemoryRpcBody(text2);
|
|
15796
15796
|
if (body.error) return { ok: false, error: body.error.message ?? `memory ${toolName} failed` };
|
|
15797
15797
|
if (body.result?.structuredContent) return body.result.structuredContent;
|
|
15798
15798
|
const textContent = body.result?.content?.find((c) => c.type === "text")?.text;
|
|
@@ -16541,13 +16541,13 @@ var YouTubeExtractor = class {
|
|
|
16541
16541
|
{ timeout: 2e4 }
|
|
16542
16542
|
);
|
|
16543
16543
|
} catch {
|
|
16544
|
-
const
|
|
16544
|
+
const url2 = page2.url();
|
|
16545
16545
|
const title = await page2.title().catch(() => "");
|
|
16546
|
-
const isBlocked = /consent\.youtube\.|accounts\.google\.|sorry\.google\./i.test(
|
|
16546
|
+
const isBlocked = /consent\.youtube\.|accounts\.google\.|sorry\.google\./i.test(url2) || /before you continue|sign in|unusual traffic|are you a robot/i.test(title);
|
|
16547
16547
|
if (isBlocked) {
|
|
16548
16548
|
throw new CaptchaError("YouTube blocked this session \u2014 retrying with a fresh session.");
|
|
16549
16549
|
}
|
|
16550
|
-
throw new CaptchaError(`ytInitialData not found after 20s (url=${
|
|
16550
|
+
throw new CaptchaError(`ytInitialData not found after 20s (url=${url2}, title="${title}") \u2014 retrying.`);
|
|
16551
16551
|
}
|
|
16552
16552
|
return page2.evaluate(() => window.ytInitialData);
|
|
16553
16553
|
}
|
|
@@ -16784,8 +16784,8 @@ var YouTubeExtractor = class {
|
|
|
16784
16784
|
const allChannels = [];
|
|
16785
16785
|
let channelMeta = null;
|
|
16786
16786
|
if (options.mode === "search") {
|
|
16787
|
-
const
|
|
16788
|
-
await this.driver.navigateTo(
|
|
16787
|
+
const url2 = `https://www.youtube.com/results?search_query=${encodeURIComponent(options.query)}`;
|
|
16788
|
+
await this.driver.navigateTo(url2);
|
|
16789
16789
|
const first = await this.extractSearchResults(page2);
|
|
16790
16790
|
allVideos.push(...first.videos);
|
|
16791
16791
|
allChannels.push(...first.channels);
|
|
@@ -16871,7 +16871,7 @@ var MP3Downloader = class {
|
|
|
16871
16871
|
}
|
|
16872
16872
|
}
|
|
16873
16873
|
async downloadOne(video) {
|
|
16874
|
-
const
|
|
16874
|
+
const url2 = video.url;
|
|
16875
16875
|
const outputTemplate = path3.join(this.outputDir, "%(title)s.%(ext)s");
|
|
16876
16876
|
try {
|
|
16877
16877
|
const { stdout } = await execFileAsync("yt-dlp", [
|
|
@@ -16888,7 +16888,7 @@ var MP3Downloader = class {
|
|
|
16888
16888
|
"after_move:filepath",
|
|
16889
16889
|
"--no-playlist",
|
|
16890
16890
|
"--no-warnings",
|
|
16891
|
-
|
|
16891
|
+
url2
|
|
16892
16892
|
]);
|
|
16893
16893
|
const outputPath = stdout.trim().split("\n").pop() ?? null;
|
|
16894
16894
|
this.reporter.onDownload(video.videoId, true, outputPath);
|
|
@@ -17068,14 +17068,14 @@ async function fetchViaYoutubeTranscript(videoId) {
|
|
|
17068
17068
|
timestamp: [e.offset / 1e3, (e.offset + e.duration) / 1e3],
|
|
17069
17069
|
text: e.text.trim()
|
|
17070
17070
|
}));
|
|
17071
|
-
const
|
|
17071
|
+
const text2 = chunks.map((c) => c.text).join(" ");
|
|
17072
17072
|
const last = chunks[chunks.length - 1];
|
|
17073
17073
|
const now = Date.now();
|
|
17074
17074
|
await runWithCostContext(
|
|
17075
17075
|
{ ...currentCostContext(), op: "yt_transcription", subOp: "youtube-transcript" },
|
|
17076
17076
|
() => recordKernelSession({ kernelSessionId: null, source: "yt_caption_tier", headlessSent: null, openedAtMs: now, closedAtMs: now })
|
|
17077
17077
|
);
|
|
17078
|
-
return { videoId, text, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
|
|
17078
|
+
return { videoId, text: text2, chunks, durationMs: last ? last.timestamp[1] * 1e3 : 0, method: "youtube-transcript" };
|
|
17079
17079
|
} catch {
|
|
17080
17080
|
return null;
|
|
17081
17081
|
}
|
|
@@ -17090,12 +17090,12 @@ function parseTimedtextJson3(jsonText) {
|
|
|
17090
17090
|
const results = [];
|
|
17091
17091
|
for (const ev of data.events ?? []) {
|
|
17092
17092
|
if (!ev.segs) continue;
|
|
17093
|
-
const
|
|
17094
|
-
if (!
|
|
17093
|
+
const text2 = ev.segs.map((s) => s.utf8 ?? "").join("").replace(/\s+/g, " ").trim();
|
|
17094
|
+
if (!text2) continue;
|
|
17095
17095
|
const start = typeof ev.tStartMs === "number" ? ev.tStartMs : NaN;
|
|
17096
17096
|
const dur = typeof ev.dDurationMs === "number" ? ev.dDurationMs : 0;
|
|
17097
17097
|
if (!Number.isFinite(start)) continue;
|
|
17098
|
-
results.push({ start, dur, text });
|
|
17098
|
+
results.push({ start, dur, text: text2 });
|
|
17099
17099
|
}
|
|
17100
17100
|
return results;
|
|
17101
17101
|
}
|
|
@@ -17107,22 +17107,22 @@ function parseTimedtextXml(xml) {
|
|
|
17107
17107
|
const startMs = parseInt(m[1], 10);
|
|
17108
17108
|
const durMs = parseInt(m[2], 10);
|
|
17109
17109
|
const inner = m[3];
|
|
17110
|
-
let
|
|
17111
|
-
if (!
|
|
17110
|
+
let text2 = inner.replace(/<[^>]+>/g, "").trim();
|
|
17111
|
+
if (!text2) {
|
|
17112
17112
|
const sMatch = /<s[^>]*>([^<]*)<\/s>/g;
|
|
17113
17113
|
let sm;
|
|
17114
17114
|
const parts = [];
|
|
17115
17115
|
while ((sm = sMatch.exec(inner)) !== null) parts.push(sm[1]);
|
|
17116
|
-
|
|
17116
|
+
text2 = parts.join("").trim();
|
|
17117
17117
|
}
|
|
17118
|
-
|
|
17119
|
-
if (
|
|
17118
|
+
text2 = text2.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
17119
|
+
if (text2) results.push({ start: startMs, dur: durMs, text: text2 });
|
|
17120
17120
|
}
|
|
17121
17121
|
if (results.length > 0) return results;
|
|
17122
17122
|
const classicRegex = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
|
|
17123
17123
|
while ((m = classicRegex.exec(xml)) !== null) {
|
|
17124
|
-
const
|
|
17125
|
-
results.push({ start: parseFloat(m[1]) * 1e3, dur: parseFloat(m[2]) * 1e3, text });
|
|
17124
|
+
const text2 = m[3].replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
|
17125
|
+
results.push({ start: parseFloat(m[1]) * 1e3, dur: parseFloat(m[2]) * 1e3, text: text2 });
|
|
17126
17126
|
}
|
|
17127
17127
|
return results;
|
|
17128
17128
|
}
|
|
@@ -17247,8 +17247,8 @@ async function fetchViaKernelSession(videoId) {
|
|
|
17247
17247
|
timestamp: [e.start / 1e3, (e.start + e.dur) / 1e3],
|
|
17248
17248
|
text: e.text
|
|
17249
17249
|
}));
|
|
17250
|
-
const
|
|
17251
|
-
return { videoId, text, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
17250
|
+
const text2 = chunks.map((c) => c.text).join(" ");
|
|
17251
|
+
return { videoId, text: text2, chunks, durationMs: Date.now() - start, method: "browser-innertube" };
|
|
17252
17252
|
}
|
|
17253
17253
|
async function fetchViaResolvedAudio(videoId, language) {
|
|
17254
17254
|
if (!mediaResolverConfigured()) return null;
|
|
@@ -18363,9 +18363,9 @@ var FacebookAdExtractor = class {
|
|
|
18363
18363
|
|
|
18364
18364
|
// src/extractor/FacebookAdGraphql.ts
|
|
18365
18365
|
var AD_LIBRARY_QUERY = "AdLibrarySearchPaginationQuery";
|
|
18366
|
-
function parseFbGraphqlJson(
|
|
18366
|
+
function parseFbGraphqlJson(text2) {
|
|
18367
18367
|
const out = [];
|
|
18368
|
-
const body =
|
|
18368
|
+
const body = text2.replace(/^for\s*\(;;\);/, "").trim();
|
|
18369
18369
|
try {
|
|
18370
18370
|
out.push(JSON.parse(body));
|
|
18371
18371
|
return out;
|
|
@@ -18405,7 +18405,7 @@ function extractCollatedResults(payload) {
|
|
|
18405
18405
|
}
|
|
18406
18406
|
return results;
|
|
18407
18407
|
}
|
|
18408
|
-
async function collectAdLibraryResults(page2,
|
|
18408
|
+
async function collectAdLibraryResults(page2, url2, maxResults, opts = {}) {
|
|
18409
18409
|
const captureMs = opts.captureMs ?? 3e4;
|
|
18410
18410
|
const collected = [];
|
|
18411
18411
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -18413,8 +18413,8 @@ async function collectAdLibraryResults(page2, url, maxResults, opts = {}) {
|
|
|
18413
18413
|
if (!resp.url().includes("/api/graphql")) return;
|
|
18414
18414
|
const friendlyName = (resp.request().postData() ?? "").match(/fb_api_req_friendly_name=([^&]+)/)?.[1];
|
|
18415
18415
|
if (friendlyName !== AD_LIBRARY_QUERY) return;
|
|
18416
|
-
void resp.text().then((
|
|
18417
|
-
for (const payload of parseFbGraphqlJson(
|
|
18416
|
+
void resp.text().then((text2) => {
|
|
18417
|
+
for (const payload of parseFbGraphqlJson(text2)) {
|
|
18418
18418
|
for (const result2 of extractCollatedResults(payload)) {
|
|
18419
18419
|
if (seen.has(result2.ad_archive_id)) continue;
|
|
18420
18420
|
seen.add(result2.ad_archive_id);
|
|
@@ -18425,7 +18425,7 @@ async function collectAdLibraryResults(page2, url, maxResults, opts = {}) {
|
|
|
18425
18425
|
};
|
|
18426
18426
|
page2.on("response", handler);
|
|
18427
18427
|
try {
|
|
18428
|
-
await page2.goto(
|
|
18428
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
18429
18429
|
const deadline = Date.now() + captureMs;
|
|
18430
18430
|
let lastCount = -1;
|
|
18431
18431
|
let stableRounds = 0;
|
|
@@ -18474,9 +18474,9 @@ function decodeEscapedValue(raw) {
|
|
|
18474
18474
|
}
|
|
18475
18475
|
return htmlDecode(out);
|
|
18476
18476
|
}
|
|
18477
|
-
function facebookVideoIdFromUrl(
|
|
18477
|
+
function facebookVideoIdFromUrl(url2) {
|
|
18478
18478
|
try {
|
|
18479
|
-
const parsed = new URL(
|
|
18479
|
+
const parsed = new URL(url2);
|
|
18480
18480
|
const path5 = parsed.pathname;
|
|
18481
18481
|
const direct = path5.match(/\/(?:reel|videos|watch)\/(?:[^/]+\/)?(\d{8,})/i);
|
|
18482
18482
|
if (direct) return direct[1];
|
|
@@ -18488,9 +18488,9 @@ function facebookVideoIdFromUrl(url) {
|
|
|
18488
18488
|
}
|
|
18489
18489
|
return null;
|
|
18490
18490
|
}
|
|
18491
|
-
function parseEfg(
|
|
18491
|
+
function parseEfg(url2) {
|
|
18492
18492
|
try {
|
|
18493
|
-
const efg = new URL(
|
|
18493
|
+
const efg = new URL(url2).searchParams.get("efg");
|
|
18494
18494
|
if (!efg) return null;
|
|
18495
18495
|
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
18496
18496
|
} catch {
|
|
@@ -18524,18 +18524,18 @@ function extractFacebookOrganicVideo(html, pageUrl, sourceUrl = pageUrl, quality
|
|
|
18524
18524
|
const re = /browser_native_(sd|hd)_url\\?"\s*:\s*\\?"([^"<]+?)\\?"/g;
|
|
18525
18525
|
let match;
|
|
18526
18526
|
while ((match = re.exec(searchWindow)) !== null) {
|
|
18527
|
-
const
|
|
18528
|
-
if (!
|
|
18529
|
-
seen.add(
|
|
18530
|
-
const parsed = parseEfg(
|
|
18527
|
+
const url2 = decodeEscapedValue(match[2]);
|
|
18528
|
+
if (!url2 || seen.has(url2)) continue;
|
|
18529
|
+
seen.add(url2);
|
|
18530
|
+
const parsed = parseEfg(url2);
|
|
18531
18531
|
let bitrate = null;
|
|
18532
18532
|
try {
|
|
18533
|
-
const value = new URL(
|
|
18533
|
+
const value = new URL(url2).searchParams.get("bitrate");
|
|
18534
18534
|
bitrate = value ? Number(value) : null;
|
|
18535
18535
|
} catch {
|
|
18536
18536
|
}
|
|
18537
18537
|
candidates.push({
|
|
18538
|
-
url,
|
|
18538
|
+
url: url2,
|
|
18539
18539
|
quality: match[1],
|
|
18540
18540
|
bitrate: Number.isFinite(bitrate) ? bitrate : null,
|
|
18541
18541
|
durationSec: typeof parsed?.duration_s === "number" ? parsed.duration_s : null,
|
|
@@ -18617,8 +18617,8 @@ var FacebookMediaBodySchema = z17.object({
|
|
|
18617
18617
|
function invalidRequest(message) {
|
|
18618
18618
|
return { error_code: "invalid_request", message };
|
|
18619
18619
|
}
|
|
18620
|
-
function isAllowedFacebookOrganicUrl(
|
|
18621
|
-
const hostname =
|
|
18620
|
+
function isAllowedFacebookOrganicUrl(url2) {
|
|
18621
|
+
const hostname = url2.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
18622
18622
|
return hostname === "facebook.com" || hostname === "fb.watch";
|
|
18623
18623
|
}
|
|
18624
18624
|
async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
|
|
@@ -19822,7 +19822,7 @@ function targetFrom(query) {
|
|
|
19822
19822
|
adminCreditsApp.post("/adjust", async (c) => {
|
|
19823
19823
|
try {
|
|
19824
19824
|
const body = await readBody(c);
|
|
19825
|
-
const
|
|
19825
|
+
const record2 = await applyAdjustment({
|
|
19826
19826
|
email: typeof body.email === "string" ? body.email : void 0,
|
|
19827
19827
|
userId: typeof body.userId === "number" ? body.userId : void 0,
|
|
19828
19828
|
direction: body.direction,
|
|
@@ -19832,7 +19832,7 @@ adminCreditsApp.post("/adjust", async (c) => {
|
|
|
19832
19832
|
reference: typeof body.reference === "string" ? body.reference : void 0,
|
|
19833
19833
|
confirmLarge: body.confirmLarge === true
|
|
19834
19834
|
});
|
|
19835
|
-
return c.json({ ok: true, adjustment:
|
|
19835
|
+
return c.json({ ok: true, adjustment: record2 });
|
|
19836
19836
|
} catch (err) {
|
|
19837
19837
|
return fail(c, err);
|
|
19838
19838
|
}
|
|
@@ -19840,14 +19840,14 @@ adminCreditsApp.post("/adjust", async (c) => {
|
|
|
19840
19840
|
adminCreditsApp.post("/adjustments/:reference/edit", async (c) => {
|
|
19841
19841
|
try {
|
|
19842
19842
|
const body = await readBody(c);
|
|
19843
|
-
const
|
|
19843
|
+
const record2 = await editAdjustment({
|
|
19844
19844
|
reference: c.req.param("reference"),
|
|
19845
19845
|
credits: typeof body.credits === "number" ? body.credits : void 0,
|
|
19846
19846
|
reason: typeof body.reason === "string" ? body.reason : void 0,
|
|
19847
19847
|
actor: body.actor,
|
|
19848
19848
|
confirmLarge: body.confirmLarge === true
|
|
19849
19849
|
});
|
|
19850
|
-
return c.json({ ok: true, adjustment:
|
|
19850
|
+
return c.json({ ok: true, adjustment: record2 });
|
|
19851
19851
|
} catch (err) {
|
|
19852
19852
|
return fail(c, err);
|
|
19853
19853
|
}
|
|
@@ -19855,8 +19855,8 @@ adminCreditsApp.post("/adjustments/:reference/edit", async (c) => {
|
|
|
19855
19855
|
adminCreditsApp.post("/adjustments/:reference/reverse", async (c) => {
|
|
19856
19856
|
try {
|
|
19857
19857
|
const body = await readBody(c);
|
|
19858
|
-
const
|
|
19859
|
-
return c.json({ ok: true, adjustment:
|
|
19858
|
+
const record2 = await reverseAdjustment(c.req.param("reference"), body.actor);
|
|
19859
|
+
return c.json({ ok: true, adjustment: record2 });
|
|
19860
19860
|
} catch (err) {
|
|
19861
19861
|
return fail(c, err);
|
|
19862
19862
|
}
|
|
@@ -20019,8 +20019,8 @@ function normalizeInstagramProfileInput(input) {
|
|
|
20019
20019
|
if (!/^[A-Za-z0-9._]{1,30}$/.test(handle)) throw new Error("Invalid Instagram handle");
|
|
20020
20020
|
return { handle, profileUrl: `https://www.instagram.com/${handle}/` };
|
|
20021
20021
|
}
|
|
20022
|
-
function isAllowedInstagramPageUrl(
|
|
20023
|
-
const host2 =
|
|
20022
|
+
function isAllowedInstagramPageUrl(url2) {
|
|
20023
|
+
const host2 = url2.hostname.toLowerCase().replace(/^www\./, "").replace(/^m\./, "");
|
|
20024
20024
|
return host2 === "instagram.com";
|
|
20025
20025
|
}
|
|
20026
20026
|
function normalizeInstagramMediaUrl(raw) {
|
|
@@ -20035,17 +20035,17 @@ function normalizeInstagramMediaUrl(raw) {
|
|
|
20035
20035
|
return null;
|
|
20036
20036
|
}
|
|
20037
20037
|
}
|
|
20038
|
-
function parseInstagramEfg(
|
|
20038
|
+
function parseInstagramEfg(url2) {
|
|
20039
20039
|
try {
|
|
20040
|
-
const efg = new URL(
|
|
20040
|
+
const efg = new URL(url2).searchParams.get("efg");
|
|
20041
20041
|
if (!efg) return null;
|
|
20042
20042
|
return JSON.parse(Buffer.from(efg, "base64").toString("utf8"));
|
|
20043
20043
|
} catch {
|
|
20044
20044
|
return null;
|
|
20045
20045
|
}
|
|
20046
20046
|
}
|
|
20047
|
-
function trackFromUrl(
|
|
20048
|
-
const normalized = normalizeInstagramMediaUrl(
|
|
20047
|
+
function trackFromUrl(url2) {
|
|
20048
|
+
const normalized = normalizeInstagramMediaUrl(url2);
|
|
20049
20049
|
if (!normalized) return null;
|
|
20050
20050
|
const parsed = parseInstagramEfg(normalized);
|
|
20051
20051
|
const tag = typeof parsed?.vencode_tag === "string" ? parsed.vencode_tag : null;
|
|
@@ -20060,7 +20060,7 @@ function trackFromUrl(url) {
|
|
|
20060
20060
|
height: typeof parsed?.height === "number" ? parsed.height : null
|
|
20061
20061
|
};
|
|
20062
20062
|
}
|
|
20063
|
-
function extractInstagramMp4UrlsFromText(
|
|
20063
|
+
function extractInstagramMp4UrlsFromText(text2) {
|
|
20064
20064
|
const urls = [];
|
|
20065
20065
|
const seen = /* @__PURE__ */ new Set();
|
|
20066
20066
|
const patterns = [
|
|
@@ -20069,7 +20069,7 @@ function extractInstagramMp4UrlsFromText(text) {
|
|
|
20069
20069
|
];
|
|
20070
20070
|
for (const pattern of patterns) {
|
|
20071
20071
|
let match;
|
|
20072
|
-
while ((match = pattern.exec(
|
|
20072
|
+
while ((match = pattern.exec(text2)) !== null) {
|
|
20073
20073
|
const normalized = normalizeInstagramMediaUrl(match[0].replace(/%5C/gi, "\\"));
|
|
20074
20074
|
if (!normalized || seen.has(normalized)) continue;
|
|
20075
20075
|
seen.add(normalized);
|
|
@@ -20099,9 +20099,9 @@ function selectInstagramTracks(tracks) {
|
|
|
20099
20099
|
const selectedAudioTrack = tracks.filter((track) => track.streamType === "audio").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0] ?? null;
|
|
20100
20100
|
return { selectedVideoTrack, selectedAudioTrack };
|
|
20101
20101
|
}
|
|
20102
|
-
function parseCount(
|
|
20103
|
-
if (!
|
|
20104
|
-
const match =
|
|
20102
|
+
function parseCount(text2) {
|
|
20103
|
+
if (!text2) return null;
|
|
20104
|
+
const match = text2.match(/([\d,.]+)\s*posts?/i);
|
|
20105
20105
|
if (!match) return null;
|
|
20106
20106
|
const value = Number(match[1].replace(/[,.]/g, ""));
|
|
20107
20107
|
return Number.isFinite(value) ? value : null;
|
|
@@ -20160,8 +20160,8 @@ async function collectInstagramProfileContentFromPage(page2, opts) {
|
|
|
20160
20160
|
return items.length - before;
|
|
20161
20161
|
};
|
|
20162
20162
|
const readBodyText = async () => page2.evaluate(() => document.body?.innerText ?? "").catch(() => "");
|
|
20163
|
-
const profileLines = (
|
|
20164
|
-
const profilePostCountText = (
|
|
20163
|
+
const profileLines = (text2) => text2.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
20164
|
+
const profilePostCountText = (text2) => profileLines(text2).find((line) => /\bposts?\b/i.test(line) && /[\d,.]/.test(line)) ?? null;
|
|
20165
20165
|
const scrollMetrics = async () => page2.evaluate(() => ({
|
|
20166
20166
|
scrollY: typeof window.scrollY === "number" ? window.scrollY : null,
|
|
20167
20167
|
scrollHeight: Math.max(
|
|
@@ -20404,8 +20404,8 @@ function blobKeyPrefix(shortcode, sourceUrl) {
|
|
|
20404
20404
|
const slug4 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
20405
20405
|
return `instagram/${stamp}-${slug4}`;
|
|
20406
20406
|
}
|
|
20407
|
-
async function fetchMediaBytes(
|
|
20408
|
-
const res = await fetch(
|
|
20407
|
+
async function fetchMediaBytes(url2, referer, runtimeUserAgent) {
|
|
20408
|
+
const res = await fetch(url2, {
|
|
20409
20409
|
headers: {
|
|
20410
20410
|
...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
|
|
20411
20411
|
"Referer": referer
|
|
@@ -20535,11 +20535,11 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
20535
20535
|
const runtimeUserAgent = await detectPageUserAgent(page2);
|
|
20536
20536
|
const capturedMediaUrls = [];
|
|
20537
20537
|
page2.on("response", (response) => {
|
|
20538
|
-
const
|
|
20539
|
-
if (!
|
|
20538
|
+
const url2 = normalizeInstagramMediaUrl(response.url());
|
|
20539
|
+
if (!url2) return;
|
|
20540
20540
|
const headers = response.headers();
|
|
20541
20541
|
const contentType = headers["content-type"] ?? "";
|
|
20542
|
-
if (contentType.includes("video/mp4") ||
|
|
20542
|
+
if (contentType.includes("video/mp4") || url2.includes(".mp4")) capturedMediaUrls.push(url2);
|
|
20543
20543
|
});
|
|
20544
20544
|
await driver.navigateTo(sourceUrl.href);
|
|
20545
20545
|
const extraction = await extractInstagramMediaFromPage(page2, sourceUrl.href, capturedMediaUrls);
|
|
@@ -20715,8 +20715,8 @@ function buildSerpQuery(topic, subreddit) {
|
|
|
20715
20715
|
const scope = subreddit ? `site:reddit.com/r/${subreddit}` : "site:reddit.com";
|
|
20716
20716
|
return `${scope} ${topic}`.trim();
|
|
20717
20717
|
}
|
|
20718
|
-
function subredditFromUrl(
|
|
20719
|
-
const m =
|
|
20718
|
+
function subredditFromUrl(url2) {
|
|
20719
|
+
const m = url2.match(/\/r\/([^/]+)\//i);
|
|
20720
20720
|
return m ? m[1] : "";
|
|
20721
20721
|
}
|
|
20722
20722
|
function extractRedditThreadUrls(organicResults, limit) {
|
|
@@ -20726,13 +20726,13 @@ function extractRedditThreadUrls(organicResults, limit) {
|
|
|
20726
20726
|
for (const raw of organicResults) {
|
|
20727
20727
|
if (!raw || typeof raw !== "object") continue;
|
|
20728
20728
|
const r = raw;
|
|
20729
|
-
const
|
|
20730
|
-
if (!
|
|
20731
|
-
seen.add(
|
|
20729
|
+
const url2 = canonicalThreadUrl(r.url);
|
|
20730
|
+
if (!url2 || seen.has(url2)) continue;
|
|
20731
|
+
seen.add(url2);
|
|
20732
20732
|
out.push({
|
|
20733
20733
|
title: typeof r.title === "string" ? r.title.trim() : "",
|
|
20734
|
-
url,
|
|
20735
|
-
subreddit: subredditFromUrl(
|
|
20734
|
+
url: url2,
|
|
20735
|
+
subreddit: subredditFromUrl(url2)
|
|
20736
20736
|
});
|
|
20737
20737
|
if (out.length >= limit) break;
|
|
20738
20738
|
}
|
|
@@ -20750,9 +20750,9 @@ async function mapWithConcurrency2(items, concurrency, fn) {
|
|
|
20750
20750
|
await Promise.all(workers);
|
|
20751
20751
|
return results;
|
|
20752
20752
|
}
|
|
20753
|
-
function parseCountText(
|
|
20754
|
-
if (typeof
|
|
20755
|
-
const m =
|
|
20753
|
+
function parseCountText(text2) {
|
|
20754
|
+
if (typeof text2 !== "string") return 0;
|
|
20755
|
+
const m = text2.replace(/,/g, "").match(/\d+/);
|
|
20756
20756
|
return m ? Number(m[0]) : 0;
|
|
20757
20757
|
}
|
|
20758
20758
|
function canonicalThreadUrl(href) {
|
|
@@ -20780,15 +20780,15 @@ function normalizeSearchRows(rows) {
|
|
|
20780
20780
|
for (const raw of rows) {
|
|
20781
20781
|
if (!raw || typeof raw !== "object") continue;
|
|
20782
20782
|
const r = raw;
|
|
20783
|
-
const
|
|
20783
|
+
const url2 = canonicalThreadUrl(r.url);
|
|
20784
20784
|
const title = typeof r.title === "string" ? r.title.trim() : "";
|
|
20785
|
-
if (!
|
|
20786
|
-
seen.add(
|
|
20785
|
+
if (!url2 || !title || seen.has(url2)) continue;
|
|
20786
|
+
seen.add(url2);
|
|
20787
20787
|
const score = parseCountText(r.scoreText);
|
|
20788
20788
|
const commentCount = parseCountText(r.commentsText);
|
|
20789
20789
|
out.push({
|
|
20790
20790
|
title,
|
|
20791
|
-
url,
|
|
20791
|
+
url: url2,
|
|
20792
20792
|
subreddit: typeof r.subreddit === "string" ? r.subreddit.trim() : "",
|
|
20793
20793
|
score,
|
|
20794
20794
|
commentCount,
|
|
@@ -20843,10 +20843,10 @@ function isQuestionSentence(s) {
|
|
|
20843
20843
|
function extractQuestions(texts, max = 5) {
|
|
20844
20844
|
const out = [];
|
|
20845
20845
|
const seen = /* @__PURE__ */ new Set();
|
|
20846
|
-
for (const
|
|
20846
|
+
for (const text2 of texts) {
|
|
20847
20847
|
if (out.length >= max) break;
|
|
20848
|
-
if (!
|
|
20849
|
-
const sentences =
|
|
20848
|
+
if (!text2) continue;
|
|
20849
|
+
const sentences = text2.split(/(?<=[.!?])\s+|\n+/);
|
|
20850
20850
|
for (const raw of sentences) {
|
|
20851
20851
|
const s = raw.trim();
|
|
20852
20852
|
if (s.length < 8 || s.length > 300) continue;
|
|
@@ -20899,9 +20899,9 @@ var RedditTrendingBodySchema = z20.object({
|
|
|
20899
20899
|
function invalidRequest4(message) {
|
|
20900
20900
|
return { error_code: "invalid_request", message };
|
|
20901
20901
|
}
|
|
20902
|
-
function normalizeRedditUrl(
|
|
20902
|
+
function normalizeRedditUrl(url2, host2 = "www") {
|
|
20903
20903
|
try {
|
|
20904
|
-
const u = new URL(
|
|
20904
|
+
const u = new URL(url2.trim());
|
|
20905
20905
|
if (!/(^|\.)reddit\.com$/.test(u.hostname.toLowerCase()) && u.hostname.toLowerCase() !== "redd.it") return null;
|
|
20906
20906
|
u.hostname = `${host2}.reddit.com`;
|
|
20907
20907
|
u.protocol = "https:";
|
|
@@ -20999,14 +20999,14 @@ function defaultRedditKernelConfig() {
|
|
|
20999
20999
|
locale: "en-US"
|
|
21000
21000
|
};
|
|
21001
21001
|
}
|
|
21002
|
-
async function scrapeRedditPage(
|
|
21002
|
+
async function scrapeRedditPage(url2, script, accept, salvage) {
|
|
21003
21003
|
const driver = new BrowserDriver();
|
|
21004
21004
|
let navigationError = null;
|
|
21005
21005
|
try {
|
|
21006
21006
|
await driver.launch(defaultRedditKernelConfig());
|
|
21007
21007
|
const page2 = driver.getPage();
|
|
21008
21008
|
try {
|
|
21009
|
-
await page2.goto(
|
|
21009
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
21010
21010
|
} catch (err) {
|
|
21011
21011
|
navigationError = err instanceof Error ? err.message : String(err);
|
|
21012
21012
|
}
|
|
@@ -21014,7 +21014,7 @@ async function scrapeRedditPage(url, script, accept, salvage) {
|
|
|
21014
21014
|
} catch (err) {
|
|
21015
21015
|
console.warn(JSON.stringify({
|
|
21016
21016
|
event: "reddit_page_attempt_failed",
|
|
21017
|
-
host: new URL(
|
|
21017
|
+
host: new URL(url2).hostname,
|
|
21018
21018
|
navigationError,
|
|
21019
21019
|
error: err instanceof Error ? err.message : String(err)
|
|
21020
21020
|
}));
|
|
@@ -21023,8 +21023,8 @@ async function scrapeRedditPage(url, script, accept, salvage) {
|
|
|
21023
21023
|
await driver.close();
|
|
21024
21024
|
}
|
|
21025
21025
|
}
|
|
21026
|
-
async function scrapeRedditThread(
|
|
21027
|
-
const normalUrl = normalizeRedditUrl(
|
|
21026
|
+
async function scrapeRedditThread(url2) {
|
|
21027
|
+
const normalUrl = normalizeRedditUrl(url2, "www");
|
|
21028
21028
|
if (!normalUrl) return null;
|
|
21029
21029
|
const usableThread = (data) => !data.blocked && (Boolean(data.postBody) || data.comments.length > 0);
|
|
21030
21030
|
const normal = await scrapeRedditPage(
|
|
@@ -22426,22 +22426,22 @@ function policy2() {
|
|
|
22426
22426
|
}
|
|
22427
22427
|
function allowedMapsImageUrl(raw) {
|
|
22428
22428
|
try {
|
|
22429
|
-
const
|
|
22430
|
-
if (
|
|
22431
|
-
const host2 =
|
|
22429
|
+
const url2 = new URL(raw);
|
|
22430
|
+
if (url2.protocol !== "https:") return null;
|
|
22431
|
+
const host2 = url2.hostname.toLowerCase().replace(/\.$/, "");
|
|
22432
22432
|
if (host2 !== "googleusercontent.com" && !host2.endsWith(".googleusercontent.com") && host2 !== "streetviewpixels-pa.googleapis.com") return null;
|
|
22433
|
-
return
|
|
22433
|
+
return url2;
|
|
22434
22434
|
} catch {
|
|
22435
22435
|
return null;
|
|
22436
22436
|
}
|
|
22437
22437
|
}
|
|
22438
22438
|
function preferredMapsImageRendition(rawUrl) {
|
|
22439
|
-
const
|
|
22440
|
-
if (!
|
|
22441
|
-
if (
|
|
22442
|
-
|
|
22439
|
+
const url2 = allowedMapsImageUrl(rawUrl);
|
|
22440
|
+
if (!url2) return rawUrl;
|
|
22441
|
+
if (url2.hostname.toLowerCase().endsWith("googleusercontent.com") && /=[^/]+$/.test(url2.pathname)) {
|
|
22442
|
+
url2.pathname = url2.pathname.replace(/=[^/]+$/, "=s2048");
|
|
22443
22443
|
}
|
|
22444
|
-
return
|
|
22444
|
+
return url2.href;
|
|
22445
22445
|
}
|
|
22446
22446
|
function imageMime(bytes) {
|
|
22447
22447
|
if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
|
|
@@ -22482,10 +22482,10 @@ async function readBounded(response) {
|
|
|
22482
22482
|
return bytes;
|
|
22483
22483
|
}
|
|
22484
22484
|
async function fetchMapsImage(rawUrl) {
|
|
22485
|
-
let
|
|
22486
|
-
if (!
|
|
22485
|
+
let url2 = allowedMapsImageUrl(preferredMapsImageRendition(rawUrl));
|
|
22486
|
+
if (!url2) throw new Error("image_host_not_allowed");
|
|
22487
22487
|
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
|
|
22488
|
-
const checked = await validatePublicHttpUrl(
|
|
22488
|
+
const checked = await validatePublicHttpUrl(url2.href, { field: "Maps image URL" });
|
|
22489
22489
|
if (checked.error || !checked.parsed) throw new Error("image_url_rejected");
|
|
22490
22490
|
const response = await fetch(checked.parsed.href, {
|
|
22491
22491
|
redirect: "manual",
|
|
@@ -22495,15 +22495,15 @@ async function fetchMapsImage(rawUrl) {
|
|
|
22495
22495
|
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
22496
22496
|
const location2 = response.headers.get("location");
|
|
22497
22497
|
if (!location2 || redirect === MAX_REDIRECTS) throw new Error("image_redirect_rejected");
|
|
22498
|
-
|
|
22499
|
-
if (!
|
|
22498
|
+
url2 = allowedMapsImageUrl(new URL(location2, url2).href);
|
|
22499
|
+
if (!url2) throw new Error("image_redirect_host_not_allowed");
|
|
22500
22500
|
continue;
|
|
22501
22501
|
}
|
|
22502
22502
|
if (!response.ok) throw new Error(`image_fetch_${response.status}`);
|
|
22503
22503
|
const body = await readBounded(response);
|
|
22504
22504
|
const detected = imageMime(body);
|
|
22505
22505
|
if (!detected) throw new Error("image_content_invalid");
|
|
22506
|
-
return { bytes: Buffer.from(body), ...detected, finalUrl:
|
|
22506
|
+
return { bytes: Buffer.from(body), ...detected, finalUrl: url2.href };
|
|
22507
22507
|
}
|
|
22508
22508
|
throw new Error("image_redirect_rejected");
|
|
22509
22509
|
}
|
|
@@ -22881,8 +22881,8 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
22881
22881
|
});
|
|
22882
22882
|
const page2 = driver.getPage();
|
|
22883
22883
|
for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
|
|
22884
|
-
const
|
|
22885
|
-
await page2.goto(
|
|
22884
|
+
const url2 = reviewPageUrl(domain, pageNum);
|
|
22885
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
22886
22886
|
await page2.waitForTimeout(2e3);
|
|
22887
22887
|
const cards = await extractReviewCards(page2);
|
|
22888
22888
|
pagesFetched = pageNum;
|
|
@@ -23089,8 +23089,8 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
23089
23089
|
});
|
|
23090
23090
|
const page2 = driver.getPage();
|
|
23091
23091
|
for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
|
|
23092
|
-
const
|
|
23093
|
-
await page2.goto(
|
|
23092
|
+
const url2 = reviewPageUrl2(product, pageNum);
|
|
23093
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
23094
23094
|
await page2.waitForTimeout(5e3);
|
|
23095
23095
|
await page2.evaluate(() => window.scrollTo(0, 1200)).catch(() => {
|
|
23096
23096
|
});
|
|
@@ -23915,70 +23915,70 @@ async function createLeadListUploadSession(ownerId3, filenameHint, idempotencyKe
|
|
|
23915
23915
|
replayed: !persisted.created
|
|
23916
23916
|
};
|
|
23917
23917
|
}
|
|
23918
|
-
async function hostedHead(
|
|
23918
|
+
async function hostedHead(record2) {
|
|
23919
23919
|
const token7 = inputArtifactToken();
|
|
23920
23920
|
if (!token7) return null;
|
|
23921
23921
|
try {
|
|
23922
23922
|
const { head } = await import("@vercel/blob");
|
|
23923
|
-
const result2 = await head(
|
|
23924
|
-
if (result2.pathname !==
|
|
23923
|
+
const result2 = await head(record2.blobPathname, { token: token7 });
|
|
23924
|
+
if (result2.pathname !== record2.blobPathname) return null;
|
|
23925
23925
|
return { size: result2.size, contentType: result2.contentType.split(";", 1)[0].trim().toLowerCase() };
|
|
23926
23926
|
} catch {
|
|
23927
23927
|
return null;
|
|
23928
23928
|
}
|
|
23929
23929
|
}
|
|
23930
23930
|
async function inspectLeadListUpload(uploadId, ownerId3) {
|
|
23931
|
-
const
|
|
23932
|
-
if (!
|
|
23933
|
-
if (Date.parse(
|
|
23931
|
+
const record2 = await getLeadListUpload(uploadId, safeOwner(ownerId3));
|
|
23932
|
+
if (!record2) return null;
|
|
23933
|
+
if (Date.parse(record2.expiresAt) <= Date.now() || record2.status === "expired") {
|
|
23934
23934
|
return {
|
|
23935
|
-
uploadId:
|
|
23935
|
+
uploadId: record2.id,
|
|
23936
23936
|
state: "expired",
|
|
23937
|
-
observedMimeType:
|
|
23938
|
-
observedBytes:
|
|
23939
|
-
sha256:
|
|
23940
|
-
scanState:
|
|
23941
|
-
expiresAt:
|
|
23937
|
+
observedMimeType: record2.observedMimeType,
|
|
23938
|
+
observedBytes: record2.observedBytes,
|
|
23939
|
+
sha256: record2.sha256,
|
|
23940
|
+
scanState: record2.scanState,
|
|
23941
|
+
expiresAt: record2.expiresAt
|
|
23942
23942
|
};
|
|
23943
23943
|
}
|
|
23944
|
-
if (
|
|
23944
|
+
if (record2.status === "invalid") {
|
|
23945
23945
|
return {
|
|
23946
|
-
uploadId:
|
|
23946
|
+
uploadId: record2.id,
|
|
23947
23947
|
state: "invalid",
|
|
23948
|
-
observedMimeType:
|
|
23949
|
-
observedBytes:
|
|
23950
|
-
sha256:
|
|
23951
|
-
scanState:
|
|
23952
|
-
expiresAt:
|
|
23948
|
+
observedMimeType: record2.observedMimeType,
|
|
23949
|
+
observedBytes: record2.observedBytes,
|
|
23950
|
+
sha256: record2.sha256,
|
|
23951
|
+
scanState: record2.scanState,
|
|
23952
|
+
expiresAt: record2.expiresAt
|
|
23953
23953
|
};
|
|
23954
23954
|
}
|
|
23955
|
-
if (
|
|
23955
|
+
if (record2.observedBytes !== null && record2.sha256) {
|
|
23956
23956
|
return {
|
|
23957
|
-
uploadId:
|
|
23957
|
+
uploadId: record2.id,
|
|
23958
23958
|
state: "ready",
|
|
23959
|
-
observedMimeType:
|
|
23960
|
-
observedBytes:
|
|
23961
|
-
sha256:
|
|
23962
|
-
scanState:
|
|
23963
|
-
expiresAt:
|
|
23959
|
+
observedMimeType: record2.observedMimeType,
|
|
23960
|
+
observedBytes: record2.observedBytes,
|
|
23961
|
+
sha256: record2.sha256,
|
|
23962
|
+
scanState: record2.scanState,
|
|
23963
|
+
expiresAt: record2.expiresAt
|
|
23964
23964
|
};
|
|
23965
23965
|
}
|
|
23966
|
-
const observed = await hostedHead(
|
|
23966
|
+
const observed = await hostedHead(record2);
|
|
23967
23967
|
if (!observed) {
|
|
23968
23968
|
return {
|
|
23969
|
-
uploadId:
|
|
23970
|
-
state: Date.parse(
|
|
23969
|
+
uploadId: record2.id,
|
|
23970
|
+
state: Date.parse(record2.uploadExpiresAt) <= Date.now() ? "expired" : "pending",
|
|
23971
23971
|
observedMimeType: null,
|
|
23972
23972
|
observedBytes: null,
|
|
23973
23973
|
sha256: null,
|
|
23974
23974
|
scanState: "pending",
|
|
23975
|
-
expiresAt:
|
|
23975
|
+
expiresAt: record2.expiresAt
|
|
23976
23976
|
};
|
|
23977
23977
|
}
|
|
23978
|
-
if (observed.size < 1 || observed.size > LEAD_LIST_UPLOAD_MAX_BYTES || !
|
|
23978
|
+
if (observed.size < 1 || observed.size > LEAD_LIST_UPLOAD_MAX_BYTES || !record2.allowedMimeTypes.includes(observed.contentType)) {
|
|
23979
23979
|
await completeLeadListUpload({
|
|
23980
|
-
id:
|
|
23981
|
-
ownerId:
|
|
23980
|
+
id: record2.id,
|
|
23981
|
+
ownerId: record2.ownerId,
|
|
23982
23982
|
observedMimeType: observed.contentType,
|
|
23983
23983
|
observedBytes: observed.size,
|
|
23984
23984
|
sha256: "",
|
|
@@ -23986,25 +23986,25 @@ async function inspectLeadListUpload(uploadId, ownerId3) {
|
|
|
23986
23986
|
status: "invalid"
|
|
23987
23987
|
});
|
|
23988
23988
|
return {
|
|
23989
|
-
uploadId:
|
|
23989
|
+
uploadId: record2.id,
|
|
23990
23990
|
state: "invalid",
|
|
23991
23991
|
observedMimeType: observed.contentType,
|
|
23992
23992
|
observedBytes: observed.size,
|
|
23993
23993
|
sha256: null,
|
|
23994
23994
|
scanState: "rejected",
|
|
23995
|
-
expiresAt:
|
|
23995
|
+
expiresAt: record2.expiresAt
|
|
23996
23996
|
};
|
|
23997
23997
|
}
|
|
23998
23998
|
const buffer = await readPrivateArtifactBuffer({
|
|
23999
23999
|
policy: uploadPolicy(),
|
|
24000
|
-
artifactId:
|
|
24000
|
+
artifactId: record2.blobPathname,
|
|
24001
24001
|
maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
|
|
24002
24002
|
});
|
|
24003
24003
|
if (!buffer) return null;
|
|
24004
24004
|
const sha2564 = createHash8("sha256").update(buffer).digest("hex");
|
|
24005
24005
|
const completed = await completeLeadListUpload({
|
|
24006
|
-
id:
|
|
24007
|
-
ownerId:
|
|
24006
|
+
id: record2.id,
|
|
24007
|
+
ownerId: record2.ownerId,
|
|
24008
24008
|
observedMimeType: observed.contentType,
|
|
24009
24009
|
observedBytes: observed.size,
|
|
24010
24010
|
sha256: sha2564,
|
|
@@ -24025,11 +24025,11 @@ async function inspectLeadListUpload(uploadId, ownerId3) {
|
|
|
24025
24025
|
async function readLeadListUpload(uploadId, ownerId3) {
|
|
24026
24026
|
const inspection = await inspectLeadListUpload(uploadId, ownerId3);
|
|
24027
24027
|
if (!inspection || inspection.state !== "ready") return null;
|
|
24028
|
-
const
|
|
24029
|
-
if (!
|
|
24028
|
+
const record2 = await getLeadListUpload(uploadId, safeOwner(ownerId3));
|
|
24029
|
+
if (!record2) return null;
|
|
24030
24030
|
return readPrivateArtifactBuffer({
|
|
24031
24031
|
policy: uploadPolicy(),
|
|
24032
|
-
artifactId:
|
|
24032
|
+
artifactId: record2.blobPathname,
|
|
24033
24033
|
maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
|
|
24034
24034
|
});
|
|
24035
24035
|
}
|
|
@@ -24079,18 +24079,18 @@ async function readNormalizedLeadListArtifact(artifactId, ownerId3) {
|
|
|
24079
24079
|
}
|
|
24080
24080
|
}
|
|
24081
24081
|
async function deleteLeadListUploadSource(uploadId, ownerId3) {
|
|
24082
|
-
const
|
|
24083
|
-
if (!
|
|
24082
|
+
const record2 = await getLeadListUpload(uploadId, safeOwner(ownerId3));
|
|
24083
|
+
if (!record2) return false;
|
|
24084
24084
|
const token7 = inputArtifactToken();
|
|
24085
24085
|
try {
|
|
24086
24086
|
if (token7) {
|
|
24087
24087
|
const { del } = await import("@vercel/blob");
|
|
24088
|
-
await del(
|
|
24088
|
+
await del(record2.blobPathname, { token: token7 });
|
|
24089
24089
|
} else if (!hostedByEnvironment2()) {
|
|
24090
24090
|
const base = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir3(), "Downloads", "mcp-scraper");
|
|
24091
|
-
await rm(join7(base, "blobs",
|
|
24091
|
+
await rm(join7(base, "blobs", record2.blobPathname), { force: true });
|
|
24092
24092
|
}
|
|
24093
|
-
await markLeadListUploadImported(
|
|
24093
|
+
await markLeadListUploadImported(record2.id, record2.ownerId);
|
|
24094
24094
|
return true;
|
|
24095
24095
|
} catch {
|
|
24096
24096
|
return false;
|
|
@@ -24175,8 +24175,8 @@ function normalizeScalar(value, warnings, formulaCell = false) {
|
|
|
24175
24175
|
if (Array.isArray(candidate.richText)) {
|
|
24176
24176
|
return candidate.richText.map((part) => {
|
|
24177
24177
|
if (!part || typeof part !== "object") return "";
|
|
24178
|
-
const
|
|
24179
|
-
return typeof
|
|
24178
|
+
const text2 = part.text;
|
|
24179
|
+
return typeof text2 === "string" ? text2 : "";
|
|
24180
24180
|
}).join("");
|
|
24181
24181
|
}
|
|
24182
24182
|
if (typeof candidate.text === "string") return candidate.text;
|
|
@@ -24257,16 +24257,16 @@ function normalizeLeadTable(input) {
|
|
|
24257
24257
|
}
|
|
24258
24258
|
return { headers, rows: records, rowCount: records.length, columnCount: headers.length };
|
|
24259
24259
|
}
|
|
24260
|
-
function parseLeadListCsv(
|
|
24261
|
-
if (typeof
|
|
24260
|
+
function parseLeadListCsv(text2, delimiter = ",", options = {}) {
|
|
24261
|
+
if (typeof text2 !== "string") fail2("invalid_text_encoding", "CSV/TSV input must be UTF-8 text.");
|
|
24262
24262
|
const maxBytes = options.maxBytes ?? 25 * 1024 * 1024;
|
|
24263
24263
|
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) fail2("invalid_bounds", "CSV/TSV byte limit is invalid.");
|
|
24264
|
-
if (Buffer.byteLength(
|
|
24264
|
+
if (Buffer.byteLength(text2, "utf8") > maxBytes) {
|
|
24265
24265
|
fail2("lead_list_bounds_exceeded", `CSV/TSV text exceeds ${maxBytes} bytes.`);
|
|
24266
24266
|
}
|
|
24267
24267
|
if (delimiter !== "," && delimiter !== " ") fail2("invalid_delimiter", "Delimiter must be comma or tab.");
|
|
24268
|
-
if (
|
|
24269
|
-
const parsed = Papa3.parse(
|
|
24268
|
+
if (text2.includes("\0")) fail2("invalid_text_encoding", "CSV/TSV input is not valid UTF-8 text.");
|
|
24269
|
+
const parsed = Papa3.parse(text2.replace(/^\uFEFF/, ""), {
|
|
24270
24270
|
delimiter,
|
|
24271
24271
|
dynamicTyping: false,
|
|
24272
24272
|
skipEmptyLines: "greedy"
|
|
@@ -24457,8 +24457,8 @@ function suggestLeadColumnMap(headers, sampleRows = []) {
|
|
|
24457
24457
|
}
|
|
24458
24458
|
return { mapping, fields };
|
|
24459
24459
|
}
|
|
24460
|
-
function detectLeadListTextDelimiter(
|
|
24461
|
-
const firstLogicalLine =
|
|
24460
|
+
function detectLeadListTextDelimiter(text2) {
|
|
24461
|
+
const firstLogicalLine = text2.split(/\r?\n/, 1)[0] ?? "";
|
|
24462
24462
|
const commaCount = (firstLogicalLine.match(/,/g) ?? []).length;
|
|
24463
24463
|
const tabCount = (firstLogicalLine.match(/\t/g) ?? []).length;
|
|
24464
24464
|
return tabCount > commaCount ? " " : ",";
|
|
@@ -24520,23 +24520,23 @@ function publicError(error) {
|
|
|
24520
24520
|
body: { error: code, message: code === "lead_list_input_failed" ? "Lead-list input processing failed." : code.replace(/_/g, " ") }
|
|
24521
24521
|
};
|
|
24522
24522
|
}
|
|
24523
|
-
async function importEnvelope(
|
|
24524
|
-
const normalized = table ?? await readNormalizedLeadListArtifact(
|
|
24523
|
+
async function importEnvelope(record2, table) {
|
|
24524
|
+
const normalized = table ?? await readNormalizedLeadListArtifact(record2.normalizedArtifactId, record2.ownerId);
|
|
24525
24525
|
return {
|
|
24526
24526
|
status: "ready",
|
|
24527
|
-
leadListId:
|
|
24528
|
-
format:
|
|
24529
|
-
sheets:
|
|
24530
|
-
selectedSheet:
|
|
24531
|
-
headers:
|
|
24532
|
-
rowCount:
|
|
24533
|
-
columnCount:
|
|
24527
|
+
leadListId: record2.id,
|
|
24528
|
+
format: record2.sourceFormat,
|
|
24529
|
+
sheets: record2.sheets,
|
|
24530
|
+
selectedSheet: record2.selectedSheet,
|
|
24531
|
+
headers: record2.headers,
|
|
24532
|
+
rowCount: record2.rowCount,
|
|
24533
|
+
columnCount: record2.columnCount,
|
|
24534
24534
|
sampleRows: normalized?.rows.slice(0, 10) ?? [],
|
|
24535
|
-
suggestedColumnMap:
|
|
24536
|
-
mappingSuggestions:
|
|
24537
|
-
scanState:
|
|
24538
|
-
warnings:
|
|
24539
|
-
expiresAt:
|
|
24535
|
+
suggestedColumnMap: record2.suggestedColumnMap.mapping,
|
|
24536
|
+
mappingSuggestions: record2.suggestedColumnMap.fields,
|
|
24537
|
+
scanState: record2.scanState,
|
|
24538
|
+
warnings: record2.warnings,
|
|
24539
|
+
expiresAt: record2.expiresAt,
|
|
24540
24540
|
billing: { mapsMc: 0, pagesMc: 0, totalMc: 0 }
|
|
24541
24541
|
};
|
|
24542
24542
|
}
|
|
@@ -24578,12 +24578,12 @@ function decodeUtf8(buffer) {
|
|
|
24578
24578
|
throw new LeadListImportError("invalid_text_encoding", "Uploaded text must be valid UTF-8.");
|
|
24579
24579
|
}
|
|
24580
24580
|
}
|
|
24581
|
-
function textDelimiter(raw,
|
|
24581
|
+
function textDelimiter(raw, text2, observedMime) {
|
|
24582
24582
|
if (raw === "," || raw === "comma") return ",";
|
|
24583
24583
|
if (raw === " " || raw === "tab") return " ";
|
|
24584
24584
|
if (raw !== void 0 && raw !== "auto") throw new LeadListImportError("invalid_delimiter", "delimiter must be comma, tab, or auto.");
|
|
24585
24585
|
if (observedMime === "text/tab-separated-values") return " ";
|
|
24586
|
-
return detectLeadListTextDelimiter(
|
|
24586
|
+
return detectLeadListTextDelimiter(text2);
|
|
24587
24587
|
}
|
|
24588
24588
|
leadListInputApp.post("/imports", async (c) => {
|
|
24589
24589
|
try {
|
|
@@ -24693,9 +24693,9 @@ leadListInputApp.post("/imports", async (c) => {
|
|
|
24693
24693
|
throw new LeadListImportError("invalid_xlsx", "Uploaded workbook is not a valid XLSX ZIP archive.");
|
|
24694
24694
|
}
|
|
24695
24695
|
if (buffer.length > LEAD_LIST_UPLOAD_MAX_BYTES) throw new LeadListImportError("lead_list_bounds_exceeded", "Uploaded source exceeds 25 MiB.");
|
|
24696
|
-
const
|
|
24697
|
-
const delimiter = textDelimiter(body.delimiter,
|
|
24698
|
-
table = parseLeadListCsv(
|
|
24696
|
+
const text2 = decodeUtf8(buffer);
|
|
24697
|
+
const delimiter = textDelimiter(body.delimiter, text2, inspection.observedMimeType);
|
|
24698
|
+
table = parseLeadListCsv(text2, delimiter);
|
|
24699
24699
|
format = delimiter === " " ? "tsv" : "csv";
|
|
24700
24700
|
}
|
|
24701
24701
|
await markLeadListUploadStructurallyValidated({
|
|
@@ -25965,9 +25965,9 @@ function durableRecordFromFlat(value) {
|
|
|
25965
25965
|
parentQuestion: clean2(row.parent_question) || null
|
|
25966
25966
|
};
|
|
25967
25967
|
}
|
|
25968
|
-
function completenessRank(
|
|
25969
|
-
const answer = Boolean(
|
|
25970
|
-
const source = Boolean(
|
|
25968
|
+
function completenessRank(record2) {
|
|
25969
|
+
const answer = Boolean(record2.answer.trim());
|
|
25970
|
+
const source = Boolean(record2.sourceCite.trim());
|
|
25971
25971
|
return answer && source ? 3 : answer ? 2 : source ? 1 : 0;
|
|
25972
25972
|
}
|
|
25973
25973
|
function sameRecord(left, right) {
|
|
@@ -26000,17 +26000,17 @@ function mergePaaProgressRecord(existing, incoming, attempt, observedAt) {
|
|
|
26000
26000
|
meaningful
|
|
26001
26001
|
};
|
|
26002
26002
|
}
|
|
26003
|
-
function flatFromRecord(
|
|
26003
|
+
function flatFromRecord(record2, seedQuery) {
|
|
26004
26004
|
return {
|
|
26005
26005
|
seed_query: seedQuery,
|
|
26006
|
-
question:
|
|
26007
|
-
answer:
|
|
26008
|
-
source_title:
|
|
26009
|
-
source_site:
|
|
26010
|
-
source_cite:
|
|
26011
|
-
depth:
|
|
26012
|
-
parent_question:
|
|
26013
|
-
extracted_at:
|
|
26006
|
+
question: record2.question,
|
|
26007
|
+
answer: record2.answer,
|
|
26008
|
+
source_title: record2.sourceTitle,
|
|
26009
|
+
source_site: record2.sourceSite,
|
|
26010
|
+
source_cite: record2.sourceCite,
|
|
26011
|
+
depth: record2.depth,
|
|
26012
|
+
parent_question: record2.parentQuestion ?? "",
|
|
26013
|
+
extracted_at: record2.observedAt
|
|
26014
26014
|
};
|
|
26015
26015
|
}
|
|
26016
26016
|
var ProgressivePaaCheckpoint = class {
|
|
@@ -26107,7 +26107,7 @@ var ProgressivePaaCheckpoint = class {
|
|
|
26107
26107
|
}
|
|
26108
26108
|
};
|
|
26109
26109
|
function partialTerminalResult(job, options, maxQuestions, progress) {
|
|
26110
|
-
const flat = progress.records.map((
|
|
26110
|
+
const flat = progress.records.map((record2) => flatFromRecord(record2, options.query ?? job.query));
|
|
26111
26111
|
const withoutAnswer = flat.filter((row) => !row.answer).length;
|
|
26112
26112
|
const withoutSource = flat.filter((row) => !row.source_cite).length;
|
|
26113
26113
|
const latestInterruption = progress.interruptions.at(-1);
|
|
@@ -26234,7 +26234,7 @@ async function runPaaHarvestJob(jobId2) {
|
|
|
26234
26234
|
progress.mergeTerminalRows(harvestResult.flat, lastAttempt, (/* @__PURE__ */ new Date()).toISOString());
|
|
26235
26235
|
await progress.flush("captured", { force: true });
|
|
26236
26236
|
const terminalProgress = progress.checkpoint("captured");
|
|
26237
|
-
const mergedFlat = terminalProgress.records.map((
|
|
26237
|
+
const mergedFlat = terminalProgress.records.map((record2) => flatFromRecord(record2, options.query ?? job.query));
|
|
26238
26238
|
const rawDiagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : {};
|
|
26239
26239
|
const rawCompleteness = rawDiagnostics.completeness && typeof rawDiagnostics.completeness === "object" ? rawDiagnostics.completeness : {};
|
|
26240
26240
|
const missingAnswers = mergedFlat.filter((row) => !row.answer).length;
|
|
@@ -26348,9 +26348,9 @@ function parseProviderError(value) {
|
|
|
26348
26348
|
return message2 ? { code: null, message: message2 } : null;
|
|
26349
26349
|
}
|
|
26350
26350
|
if (typeof value !== "object" || Array.isArray(value)) return null;
|
|
26351
|
-
const
|
|
26352
|
-
const code = boundedText(
|
|
26353
|
-
const message = boundedText(
|
|
26351
|
+
const record2 = value;
|
|
26352
|
+
const code = boundedText(record2.code ?? record2.type ?? record2.name);
|
|
26353
|
+
const message = boundedText(record2.message ?? record2.error);
|
|
26354
26354
|
return code || message ? { code, message } : null;
|
|
26355
26355
|
}
|
|
26356
26356
|
function parseSessionPayload(payload, expectedSessionId, checkedAt) {
|
|
@@ -26493,16 +26493,16 @@ function recoverableRecords(result2) {
|
|
|
26493
26493
|
const questions = /* @__PURE__ */ new Map();
|
|
26494
26494
|
for (const raw of records) {
|
|
26495
26495
|
if (!raw || typeof raw !== "object") continue;
|
|
26496
|
-
const
|
|
26497
|
-
const question = normalizeQuestion2(
|
|
26496
|
+
const record2 = raw;
|
|
26497
|
+
const question = normalizeQuestion2(record2.question);
|
|
26498
26498
|
if (!question) continue;
|
|
26499
26499
|
const existing = questions.get(question);
|
|
26500
26500
|
if (!existing) {
|
|
26501
|
-
questions.set(question,
|
|
26501
|
+
questions.set(question, record2);
|
|
26502
26502
|
continue;
|
|
26503
26503
|
}
|
|
26504
26504
|
const completeness = (value) => Number(Boolean(String(value.answer ?? "").trim())) * 2 + Number(Boolean(String(value.sourceCite ?? value.source_cite ?? "").trim()));
|
|
26505
|
-
if (completeness(
|
|
26505
|
+
if (completeness(record2) >= completeness(existing)) questions.set(question, record2);
|
|
26506
26506
|
}
|
|
26507
26507
|
return [...questions.values()];
|
|
26508
26508
|
}
|
|
@@ -26510,16 +26510,16 @@ function stalePartialResult(result2, records, seedQuery) {
|
|
|
26510
26510
|
const current = result2 && typeof result2 === "object" ? result2 : {};
|
|
26511
26511
|
const diagnostics = current.diagnostics && typeof current.diagnostics === "object" ? current.diagnostics : {};
|
|
26512
26512
|
const completeness = diagnostics.completeness && typeof diagnostics.completeness === "object" ? diagnostics.completeness : {};
|
|
26513
|
-
const flat = records.map((
|
|
26514
|
-
seed_query: typeof
|
|
26515
|
-
question: typeof
|
|
26516
|
-
answer: typeof
|
|
26517
|
-
source_title: typeof
|
|
26518
|
-
source_site: typeof
|
|
26519
|
-
source_cite: typeof
|
|
26520
|
-
depth: typeof
|
|
26521
|
-
parent_question: typeof
|
|
26522
|
-
extracted_at: typeof
|
|
26513
|
+
const flat = records.map((record2) => ({
|
|
26514
|
+
seed_query: typeof record2.seed_query === "string" ? record2.seed_query : seedQuery,
|
|
26515
|
+
question: typeof record2.question === "string" ? record2.question : "",
|
|
26516
|
+
answer: typeof record2.answer === "string" ? record2.answer : "",
|
|
26517
|
+
source_title: typeof record2.source_title === "string" ? record2.source_title : typeof record2.sourceTitle === "string" ? record2.sourceTitle : "",
|
|
26518
|
+
source_site: typeof record2.source_site === "string" ? record2.source_site : typeof record2.sourceSite === "string" ? record2.sourceSite : "",
|
|
26519
|
+
source_cite: typeof record2.source_cite === "string" ? record2.source_cite : typeof record2.sourceCite === "string" ? record2.sourceCite : "",
|
|
26520
|
+
depth: typeof record2.depth === "number" ? record2.depth : 1,
|
|
26521
|
+
parent_question: typeof record2.parent_question === "string" ? record2.parent_question : typeof record2.parentQuestion === "string" ? record2.parentQuestion : "",
|
|
26522
|
+
extracted_at: typeof record2.extracted_at === "string" ? record2.extracted_at : typeof record2.observedAt === "string" ? record2.observedAt : (/* @__PURE__ */ new Date()).toISOString()
|
|
26523
26523
|
}));
|
|
26524
26524
|
const uniqueCount = flat.length;
|
|
26525
26525
|
return {
|
|
@@ -26791,8 +26791,8 @@ var SchedulePatchSchema = z28.object({
|
|
|
26791
26791
|
function hostedWorkflowOutputDir() {
|
|
26792
26792
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
26793
26793
|
}
|
|
26794
|
-
function originFromUrl(
|
|
26795
|
-
const parsed = new URL(
|
|
26794
|
+
function originFromUrl(url2) {
|
|
26795
|
+
const parsed = new URL(url2);
|
|
26796
26796
|
return `${parsed.protocol}//${parsed.host}`;
|
|
26797
26797
|
}
|
|
26798
26798
|
function addCadence(fromIso, cadence) {
|
|
@@ -26810,11 +26810,11 @@ function validateWorkflowInput(workflowId, input) {
|
|
|
26810
26810
|
const definition = workflowDefinition(workflowId);
|
|
26811
26811
|
return definition.inputSchema.parse(input);
|
|
26812
26812
|
}
|
|
26813
|
-
async function validateWebhookUrl(
|
|
26814
|
-
if (!
|
|
26815
|
-
const result2 = await validatePublicHttpUrl(
|
|
26813
|
+
async function validateWebhookUrl(url2) {
|
|
26814
|
+
if (!url2) return null;
|
|
26815
|
+
const result2 = await validatePublicHttpUrl(url2, { field: "webhookUrl", requireHttps: true });
|
|
26816
26816
|
if (result2.error) throw new Error(result2.error);
|
|
26817
|
-
return
|
|
26817
|
+
return url2;
|
|
26818
26818
|
}
|
|
26819
26819
|
function runDownloadUrl(c, runId, artifactId) {
|
|
26820
26820
|
return `${originFromUrl(c.req.url)}/workflows/runs/${runId}/artifacts/${artifactId}`;
|
|
@@ -27474,10 +27474,10 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
27474
27474
|
const completedAtDate = /* @__PURE__ */ new Date();
|
|
27475
27475
|
const completedAt = completedAtDate.toISOString();
|
|
27476
27476
|
const durationMs = completedAtDate.getTime() - startedAtDate.getTime();
|
|
27477
|
-
const
|
|
27477
|
+
const text2 = stripMarkdown(extraction.bodyMarkdown);
|
|
27478
27478
|
const htmlContent = truncateContent(extraction.bodyHtml, maxContentChars);
|
|
27479
27479
|
const markdownContent = truncateContent(extraction.bodyMarkdown, maxContentChars);
|
|
27480
|
-
const textContent = truncateContent(
|
|
27480
|
+
const textContent = truncateContent(text2, maxContentChars);
|
|
27481
27481
|
const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
|
|
27482
27482
|
const metrics = countLinks(extraction.bodyHtml, extraction.url);
|
|
27483
27483
|
const wordCount2 = countWords(extraction.bodyMarkdown);
|
|
@@ -27542,7 +27542,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
27542
27542
|
hashes: {
|
|
27543
27543
|
htmlSha256: sha256(extraction.bodyHtml),
|
|
27544
27544
|
markdownSha256: contentSha256,
|
|
27545
|
-
textSha256: sha256(
|
|
27545
|
+
textSha256: sha256(text2)
|
|
27546
27546
|
},
|
|
27547
27547
|
diagnostics,
|
|
27548
27548
|
...contentIncluded ? {
|
|
@@ -27913,10 +27913,10 @@ function normalizePageSnapshotLimit(input) {
|
|
|
27913
27913
|
return Math.max(0, Math.min(10, input.pageSnapshotLimit));
|
|
27914
27914
|
}
|
|
27915
27915
|
function addUniqueTarget(targets, seenUrls, target) {
|
|
27916
|
-
const
|
|
27917
|
-
if (!
|
|
27918
|
-
seenUrls.add(
|
|
27919
|
-
targets.push({ ...target, url });
|
|
27916
|
+
const url2 = target.url.trim();
|
|
27917
|
+
if (!url2 || seenUrls.has(url2)) return;
|
|
27918
|
+
seenUrls.add(url2);
|
|
27919
|
+
targets.push({ ...target, url: url2 });
|
|
27920
27920
|
}
|
|
27921
27921
|
function collectPageSnapshotTargets(harvestResult, limit) {
|
|
27922
27922
|
if (limit <= 0) return [];
|
|
@@ -28063,8 +28063,8 @@ async function enforceSerpIntelligenceRateLimit(c, userId) {
|
|
|
28063
28063
|
);
|
|
28064
28064
|
}
|
|
28065
28065
|
function pageSnapshotTargetsFromBody(body) {
|
|
28066
|
-
return body.targets?.length ? body.targets : (body.urls ?? []).map((
|
|
28067
|
-
url,
|
|
28066
|
+
return body.targets?.length ? body.targets : (body.urls ?? []).map((url2) => ({
|
|
28067
|
+
url: url2,
|
|
28068
28068
|
sourceKind: "configured_target",
|
|
28069
28069
|
sourcePosition: null
|
|
28070
28070
|
}));
|
|
@@ -29584,11 +29584,11 @@ var FanoutCdpCapture = class {
|
|
|
29584
29584
|
client2.on("Network.responseReceived", (e) => {
|
|
29585
29585
|
const id2 = e.requestId;
|
|
29586
29586
|
const res = e.response;
|
|
29587
|
-
const
|
|
29587
|
+
const url2 = res?.url || this.reqUrl.get(id2) || "";
|
|
29588
29588
|
const headers = res?.headers || {};
|
|
29589
29589
|
const ct = String(headers["content-type"] || headers["Content-Type"] || "");
|
|
29590
29590
|
if (id2) this.ctById.set(id2, ct);
|
|
29591
|
-
if (id2 && (this.isAnswerUrl(
|
|
29591
|
+
if (id2 && (this.isAnswerUrl(url2) || /event-stream/i.test(ct))) this.answerIds.add(id2);
|
|
29592
29592
|
});
|
|
29593
29593
|
client2.on("Network.loadingFinished", (e) => this.grab(e.requestId));
|
|
29594
29594
|
}
|
|
@@ -29621,25 +29621,25 @@ var FanoutCdpCapture = class {
|
|
|
29621
29621
|
}
|
|
29622
29622
|
this.client = null;
|
|
29623
29623
|
}
|
|
29624
|
-
isAnswerUrl(
|
|
29624
|
+
isAnswerUrl(url2) {
|
|
29625
29625
|
let path5 = "";
|
|
29626
29626
|
try {
|
|
29627
|
-
path5 = new URL(
|
|
29627
|
+
path5 = new URL(url2).pathname;
|
|
29628
29628
|
} catch {
|
|
29629
|
-
path5 = String(
|
|
29629
|
+
path5 = String(url2 || "").split("?")[0];
|
|
29630
29630
|
}
|
|
29631
29631
|
return ANSWER_RE.test(path5);
|
|
29632
29632
|
}
|
|
29633
29633
|
grab(requestId) {
|
|
29634
29634
|
if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
|
|
29635
|
-
const
|
|
29635
|
+
const url2 = this.reqUrl.get(requestId) || "";
|
|
29636
29636
|
const ct = this.ctById.get(requestId) || "";
|
|
29637
29637
|
const client2 = this.client;
|
|
29638
29638
|
const p = client2.send("Network.getResponseBody", { requestId }).then((r) => {
|
|
29639
|
-
const
|
|
29640
|
-
if (
|
|
29641
|
-
this.bodies.push({ url, ct, text });
|
|
29642
|
-
if (this.isAnswerUrl(
|
|
29639
|
+
const text2 = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
|
|
29640
|
+
if (text2) {
|
|
29641
|
+
this.bodies.push({ url: url2, ct, text: text2 });
|
|
29642
|
+
if (this.isAnswerUrl(url2)) this.answerStreams++;
|
|
29643
29643
|
}
|
|
29644
29644
|
}).catch(() => void 0);
|
|
29645
29645
|
this.pending.push(p);
|
|
@@ -29682,14 +29682,14 @@ function unesc(s) {
|
|
|
29682
29682
|
function stripCitationMarkers(s) {
|
|
29683
29683
|
return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
|
|
29684
29684
|
}
|
|
29685
|
-
function sliceBalanced(
|
|
29686
|
-
const open =
|
|
29685
|
+
function sliceBalanced(text2, openIdx) {
|
|
29686
|
+
const open = text2[openIdx];
|
|
29687
29687
|
const close = open === "[" ? "]" : "}";
|
|
29688
29688
|
let depth = 0;
|
|
29689
29689
|
let inStr = false;
|
|
29690
29690
|
let esc = false;
|
|
29691
|
-
for (let i = openIdx; i <
|
|
29692
|
-
const ch =
|
|
29691
|
+
for (let i = openIdx; i < text2.length; i++) {
|
|
29692
|
+
const ch = text2[i];
|
|
29693
29693
|
if (inStr) {
|
|
29694
29694
|
if (esc) esc = false;
|
|
29695
29695
|
else if (ch === "\\") esc = true;
|
|
@@ -29703,18 +29703,18 @@ function sliceBalanced(text, openIdx) {
|
|
|
29703
29703
|
if (ch === open) depth++;
|
|
29704
29704
|
else if (ch === close) {
|
|
29705
29705
|
depth--;
|
|
29706
|
-
if (depth === 0) return
|
|
29706
|
+
if (depth === 0) return text2.slice(openIdx, i + 1);
|
|
29707
29707
|
}
|
|
29708
29708
|
}
|
|
29709
29709
|
return null;
|
|
29710
29710
|
}
|
|
29711
|
-
function findJsonValues(
|
|
29711
|
+
function findJsonValues(text2, field) {
|
|
29712
29712
|
const out = [];
|
|
29713
29713
|
const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
|
|
29714
29714
|
let m;
|
|
29715
|
-
while (m = re.exec(
|
|
29715
|
+
while (m = re.exec(text2)) {
|
|
29716
29716
|
const openIdx = m.index + m[0].length - 1;
|
|
29717
|
-
const slice = sliceBalanced(
|
|
29717
|
+
const slice = sliceBalanced(text2, openIdx);
|
|
29718
29718
|
if (slice) {
|
|
29719
29719
|
try {
|
|
29720
29720
|
out.push(JSON.parse(slice));
|
|
@@ -29818,7 +29818,7 @@ function reconstructFromSsePatches(raw) {
|
|
|
29818
29818
|
ops.forEach(handleOp);
|
|
29819
29819
|
}
|
|
29820
29820
|
if (!state.found) return raw;
|
|
29821
|
-
const citedRefs = [...citedUrls].map((
|
|
29821
|
+
const citedRefs = [...citedUrls].map((url2) => ({ type: "grouped_webpages", items: [{ url: url2 }] }));
|
|
29822
29822
|
const synthetic = JSON.stringify({
|
|
29823
29823
|
content_references: [...contentReferences, ...citedRefs],
|
|
29824
29824
|
search_result_groups: searchResultGroups,
|
|
@@ -29838,20 +29838,20 @@ var chatgptAdapter = {
|
|
|
29838
29838
|
const queries = /* @__PURE__ */ new Set();
|
|
29839
29839
|
const unmapped = /* @__PURE__ */ new Set();
|
|
29840
29840
|
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
29841
|
-
const ensure = (
|
|
29842
|
-
if (!isExternal(
|
|
29843
|
-
const k = normUrl(
|
|
29841
|
+
const ensure = (url2) => {
|
|
29842
|
+
if (!isExternal(url2)) return null;
|
|
29843
|
+
const k = normUrl(url2);
|
|
29844
29844
|
let s = sources.get(k);
|
|
29845
29845
|
if (!s) {
|
|
29846
|
-
s = { url, domain: domainOf(
|
|
29846
|
+
s = { url: url2, domain: domainOf(url2), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
|
|
29847
29847
|
sources.set(k, s);
|
|
29848
29848
|
}
|
|
29849
29849
|
return s;
|
|
29850
29850
|
};
|
|
29851
|
-
const addSnippet = (
|
|
29852
|
-
const clean3 = stripCitationMarkers(
|
|
29853
|
-
if (!isExternal(
|
|
29854
|
-
snippets.set(normUrl(
|
|
29851
|
+
const addSnippet = (url2, title, text2) => {
|
|
29852
|
+
const clean3 = stripCitationMarkers(text2);
|
|
29853
|
+
if (!isExternal(url2) || clean3.length < 2) return;
|
|
29854
|
+
snippets.set(normUrl(url2), { url: url2, domain: domainOf(url2), title: title || domainOf(url2), text: clean3.slice(0, 400) });
|
|
29855
29855
|
};
|
|
29856
29856
|
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
29857
29857
|
for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
|
|
@@ -29871,13 +29871,13 @@ var chatgptAdapter = {
|
|
|
29871
29871
|
for (const g of v) {
|
|
29872
29872
|
const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
|
|
29873
29873
|
for (const e of entries) {
|
|
29874
|
-
const
|
|
29875
|
-
const s = ensure(
|
|
29874
|
+
const url2 = typeof e.url === "string" ? e.url : "";
|
|
29875
|
+
const s = ensure(url2);
|
|
29876
29876
|
if (!s) continue;
|
|
29877
29877
|
if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
|
|
29878
29878
|
const ref = normRef(e.ref_id);
|
|
29879
29879
|
if (ref && s.round == null) s.round = roundOf(ref);
|
|
29880
|
-
if (typeof e.snippet === "string" && e.snippet) addSnippet(
|
|
29880
|
+
if (typeof e.snippet === "string" && e.snippet) addSnippet(url2, s.title, e.snippet);
|
|
29881
29881
|
}
|
|
29882
29882
|
}
|
|
29883
29883
|
}
|
|
@@ -29929,11 +29929,11 @@ var chatgptAdapter = {
|
|
|
29929
29929
|
});
|
|
29930
29930
|
}
|
|
29931
29931
|
for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
|
|
29932
|
-
const
|
|
29932
|
+
const url2 = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
|
|
29933
29933
|
const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
|
|
29934
|
-
if (
|
|
29935
|
-
ensure(
|
|
29936
|
-
addSnippet(
|
|
29934
|
+
if (url2) {
|
|
29935
|
+
ensure(url2);
|
|
29936
|
+
addSnippet(url2, title ? unesc(title) : "", unesc(m[2]));
|
|
29937
29937
|
}
|
|
29938
29938
|
}
|
|
29939
29939
|
const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
|
|
@@ -29979,12 +29979,12 @@ var claudeAdapter = {
|
|
|
29979
29979
|
const queries = /* @__PURE__ */ new Set();
|
|
29980
29980
|
const unmapped = /* @__PURE__ */ new Set();
|
|
29981
29981
|
const meta = { model: "", finishType: "", title: "", rounds: 0 };
|
|
29982
|
-
const ensure = (
|
|
29983
|
-
if (!
|
|
29984
|
-
const k = normUrl(
|
|
29982
|
+
const ensure = (url2) => {
|
|
29983
|
+
if (!url2 || !isExternal(url2) || isFavicon(url2)) return null;
|
|
29984
|
+
const k = normUrl(url2);
|
|
29985
29985
|
let s = sources.get(k);
|
|
29986
29986
|
if (!s) {
|
|
29987
|
-
s = { url, domain: domainOf(
|
|
29987
|
+
s = { url: url2, domain: domainOf(url2), title: "", cited: false, timesCited: 0, snippet: "", round: null };
|
|
29988
29988
|
sources.set(k, s);
|
|
29989
29989
|
}
|
|
29990
29990
|
return s;
|
|
@@ -30199,7 +30199,7 @@ var COMPOSER_SELECTORS = [
|
|
|
30199
30199
|
'div[contenteditable="true"]',
|
|
30200
30200
|
"textarea"
|
|
30201
30201
|
];
|
|
30202
|
-
async function sendPrompt(page2,
|
|
30202
|
+
async function sendPrompt(page2, text2) {
|
|
30203
30203
|
for (const selector of COMPOSER_SELECTORS) {
|
|
30204
30204
|
const locator = page2.locator(selector).first();
|
|
30205
30205
|
try {
|
|
@@ -30209,7 +30209,7 @@ async function sendPrompt(page2, text) {
|
|
|
30209
30209
|
}
|
|
30210
30210
|
await locator.click({ timeout: 4e3 }).catch(() => void 0);
|
|
30211
30211
|
await locator.focus().catch(() => void 0);
|
|
30212
|
-
await page2.keyboard.type(
|
|
30212
|
+
await page2.keyboard.type(text2, { delay: 12 });
|
|
30213
30213
|
await page2.waitForTimeout(300).catch(() => void 0);
|
|
30214
30214
|
await page2.keyboard.press("Enter");
|
|
30215
30215
|
return true;
|
|
@@ -30243,9 +30243,9 @@ async function runFanoutCapture(page2, input) {
|
|
|
30243
30243
|
if (!adapter) {
|
|
30244
30244
|
throw new Error(`query_fanout_workflow supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
|
|
30245
30245
|
}
|
|
30246
|
-
const
|
|
30247
|
-
const parsed = adapter.parse(
|
|
30248
|
-
const result2 = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes:
|
|
30246
|
+
const text2 = cap.bestAnswerText();
|
|
30247
|
+
const parsed = adapter.parse(text2, input.prompt || "");
|
|
30248
|
+
const result2 = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text2.length });
|
|
30249
30249
|
const enriched = enrichFanout(result2, { firstPartyDomain: input.first_party_domain });
|
|
30250
30250
|
const exports = input.export ? exportFanout(enriched) : null;
|
|
30251
30251
|
return { result: enriched, exports };
|
|
@@ -30369,10 +30369,10 @@ async function click(runtimeSessionId, x, y, opts = {}) {
|
|
|
30369
30369
|
...opts.numClicks ? { num_clicks: opts.numClicks } : {}
|
|
30370
30370
|
});
|
|
30371
30371
|
}
|
|
30372
|
-
async function typeText(runtimeSessionId,
|
|
30372
|
+
async function typeText(runtimeSessionId, text2, delayMs) {
|
|
30373
30373
|
const k = client();
|
|
30374
30374
|
await k.browsers.computer.typeText(runtimeSessionId, {
|
|
30375
|
-
text,
|
|
30375
|
+
text: text2,
|
|
30376
30376
|
...typeof delayMs === "number" ? { delay: delayMs } : {}
|
|
30377
30377
|
});
|
|
30378
30378
|
}
|
|
@@ -30384,12 +30384,12 @@ async function pressKeys(runtimeSessionId, keys) {
|
|
|
30384
30384
|
const k = client();
|
|
30385
30385
|
await k.browsers.computer.pressKey(runtimeSessionId, { keys });
|
|
30386
30386
|
}
|
|
30387
|
-
async function goto(cdpWsUrl,
|
|
30387
|
+
async function goto(cdpWsUrl, url2) {
|
|
30388
30388
|
const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
|
|
30389
30389
|
try {
|
|
30390
30390
|
const context = await currentOrRuntimeContext(browser);
|
|
30391
30391
|
const page2 = context.pages()[0] ?? await context.newPage();
|
|
30392
|
-
await page2.goto(
|
|
30392
|
+
await page2.goto(url2, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
30393
30393
|
return { url: page2.url(), title: await page2.title() };
|
|
30394
30394
|
} finally {
|
|
30395
30395
|
keepHostedBrowserAlive(browser);
|
|
@@ -30410,7 +30410,7 @@ async function readPage(cdpWsUrl) {
|
|
|
30410
30410
|
try {
|
|
30411
30411
|
const context = await currentOrRuntimeContext(browser);
|
|
30412
30412
|
const page2 = context.pages()[0] ?? await context.newPage();
|
|
30413
|
-
const
|
|
30413
|
+
const url2 = page2.url();
|
|
30414
30414
|
const title = await page2.title().catch(() => "");
|
|
30415
30415
|
const data = await page2.evaluate(() => {
|
|
30416
30416
|
const SEL = 'a[href], button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [onclick]';
|
|
@@ -30435,11 +30435,11 @@ async function readPage(cdpWsUrl) {
|
|
|
30435
30435
|
bottom: Math.round(r.bottom)
|
|
30436
30436
|
});
|
|
30437
30437
|
}
|
|
30438
|
-
const
|
|
30439
|
-
return { text, els: out };
|
|
30438
|
+
const text2 = (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").trim().slice(0, 6e3);
|
|
30439
|
+
return { text: text2, els: out };
|
|
30440
30440
|
});
|
|
30441
30441
|
return {
|
|
30442
|
-
url,
|
|
30442
|
+
url: url2,
|
|
30443
30443
|
title,
|
|
30444
30444
|
text: data.text,
|
|
30445
30445
|
elements: data.els.map((e, i) => ({
|
|
@@ -30465,7 +30465,7 @@ async function locatePageTargets(cdpWsUrl, targets) {
|
|
|
30465
30465
|
try {
|
|
30466
30466
|
const context = await currentOrRuntimeContext(browser);
|
|
30467
30467
|
const page2 = context.pages()[0] ?? await context.newPage();
|
|
30468
|
-
const
|
|
30468
|
+
const url2 = page2.url();
|
|
30469
30469
|
const title = await page2.title().catch(() => "");
|
|
30470
30470
|
await page2.evaluate(() => {
|
|
30471
30471
|
;
|
|
@@ -30569,13 +30569,13 @@ async function locatePageTargets(cdpWsUrl, targets) {
|
|
|
30569
30569
|
const match = target.match === "exact" ? "exact" : "contains";
|
|
30570
30570
|
try {
|
|
30571
30571
|
const selector = target.selector?.trim();
|
|
30572
|
-
const
|
|
30573
|
-
const candidates = selector ? selectorMatches(selector) :
|
|
30572
|
+
const text2 = target.text?.trim();
|
|
30573
|
+
const candidates = selector ? selectorMatches(selector) : text2 ? textMatches(text2, match) : [];
|
|
30574
30574
|
const selected = candidates[index];
|
|
30575
30575
|
return {
|
|
30576
30576
|
name: target.name ?? null,
|
|
30577
30577
|
selector: selector || null,
|
|
30578
|
-
text:
|
|
30578
|
+
text: text2 || null,
|
|
30579
30579
|
match,
|
|
30580
30580
|
index,
|
|
30581
30581
|
found: Boolean(selected),
|
|
@@ -30597,7 +30597,7 @@ async function locatePageTargets(cdpWsUrl, targets) {
|
|
|
30597
30597
|
});
|
|
30598
30598
|
return { viewport, targets: results };
|
|
30599
30599
|
}, targets);
|
|
30600
|
-
return { url, title, viewport: data.viewport, targets: data.targets };
|
|
30600
|
+
return { url: url2, title, viewport: data.viewport, targets: data.targets };
|
|
30601
30601
|
} finally {
|
|
30602
30602
|
keepHostedBrowserAlive(browser);
|
|
30603
30603
|
}
|
|
@@ -31211,17 +31211,17 @@ function buildBrowserAgentRoutes() {
|
|
|
31211
31211
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
31212
31212
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
31213
31213
|
const body = await c.req.json().catch(() => ({}));
|
|
31214
|
-
const
|
|
31215
|
-
if (!
|
|
31214
|
+
const url2 = typeof body.url === "string" ? body.url : "";
|
|
31215
|
+
if (!url2) return c.json({ error: "url is required" }, 400);
|
|
31216
31216
|
const t0 = Date.now();
|
|
31217
31217
|
try {
|
|
31218
|
-
const result2 = await goto(row.cdp_ws_url,
|
|
31218
|
+
const result2 = await goto(row.cdp_ws_url, url2);
|
|
31219
31219
|
await charge(row.id, user.id, t0);
|
|
31220
|
-
await recordAction({ sessionId: row.id, type: "goto", params: { url }, ok: true });
|
|
31220
|
+
await recordAction({ sessionId: row.id, type: "goto", params: { url: url2 }, ok: true });
|
|
31221
31221
|
return c.json(result2);
|
|
31222
31222
|
} catch (err) {
|
|
31223
31223
|
await charge(row.id, user.id, t0);
|
|
31224
|
-
await recordAction({ sessionId: row.id, type: "goto", params: { url }, ok: false, error: String(err) });
|
|
31224
|
+
await recordAction({ sessionId: row.id, type: "goto", params: { url: url2 }, ok: false, error: String(err) });
|
|
31225
31225
|
return c.json(failure(err), 502);
|
|
31226
31226
|
}
|
|
31227
31227
|
});
|
|
@@ -31345,13 +31345,13 @@ function buildBrowserAgentRoutes() {
|
|
|
31345
31345
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
31346
31346
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
31347
31347
|
const body = await c.req.json().catch(() => ({}));
|
|
31348
|
-
const
|
|
31349
|
-
if (!
|
|
31348
|
+
const text2 = typeof body.text === "string" ? body.text : "";
|
|
31349
|
+
if (!text2) return c.json({ error: "text is required" }, 400);
|
|
31350
31350
|
const t0 = Date.now();
|
|
31351
31351
|
try {
|
|
31352
|
-
await typeText(row.runtime_session_id,
|
|
31352
|
+
await typeText(row.runtime_session_id, text2, typeof body.delay === "number" ? body.delay : void 0);
|
|
31353
31353
|
await charge(row.id, user.id, t0);
|
|
31354
|
-
await recordAction({ sessionId: row.id, type: "type", params: { length:
|
|
31354
|
+
await recordAction({ sessionId: row.id, type: "type", params: { length: text2.length }, ok: true });
|
|
31355
31355
|
return c.json({ ok: true });
|
|
31356
31356
|
} catch (err) {
|
|
31357
31357
|
await charge(row.id, user.id, t0);
|
|
@@ -33791,18 +33791,18 @@ function normalizePipelines(value) {
|
|
|
33791
33791
|
});
|
|
33792
33792
|
}
|
|
33793
33793
|
function normalizeProviderSchema(provider, raw) {
|
|
33794
|
-
const
|
|
33794
|
+
const record2 = isRecord(raw) ? raw : {};
|
|
33795
33795
|
const fields = [];
|
|
33796
33796
|
const objects = [];
|
|
33797
|
-
let pipelines = normalizePipelines(
|
|
33797
|
+
let pipelines = normalizePipelines(record2.pipelines);
|
|
33798
33798
|
const push = (field) => {
|
|
33799
33799
|
if (field) fields.push(field);
|
|
33800
33800
|
};
|
|
33801
33801
|
if (provider === "hubspot") {
|
|
33802
33802
|
for (const [key2, object2] of [["contacts", "contacts"], ["deals", "deals"]]) {
|
|
33803
|
-
if (!Array.isArray(
|
|
33803
|
+
if (!Array.isArray(record2[key2])) continue;
|
|
33804
33804
|
objects.push(object2);
|
|
33805
|
-
for (const item of
|
|
33805
|
+
for (const item of record2[key2]) if (isRecord(item)) push(normalizedField({
|
|
33806
33806
|
object: object2,
|
|
33807
33807
|
internalName: item.name,
|
|
33808
33808
|
label: item.label,
|
|
@@ -33811,7 +33811,7 @@ function normalizeProviderSchema(provider, raw) {
|
|
|
33811
33811
|
}));
|
|
33812
33812
|
}
|
|
33813
33813
|
} else if (provider === "salesforce") {
|
|
33814
|
-
if (Array.isArray(
|
|
33814
|
+
if (Array.isArray(record2.objects)) for (const entry of record2.objects) {
|
|
33815
33815
|
if (!isRecord(entry)) continue;
|
|
33816
33816
|
const object2 = safeString(entry.object, 100);
|
|
33817
33817
|
if (!object2) continue;
|
|
@@ -33829,15 +33829,15 @@ function normalizeProviderSchema(provider, raw) {
|
|
|
33829
33829
|
}
|
|
33830
33830
|
}
|
|
33831
33831
|
} else if (provider === "highlevel") {
|
|
33832
|
-
if (Array.isArray(
|
|
33833
|
-
for (const item of
|
|
33832
|
+
if (Array.isArray(record2.customFields)) {
|
|
33833
|
+
for (const item of record2.customFields) if (isRecord(item)) {
|
|
33834
33834
|
const object2 = item.model === "opportunity" ? "opportunities" : "contacts";
|
|
33835
33835
|
objects.push(object2);
|
|
33836
33836
|
push(normalizedField({ object: object2, internalName: item.fieldKey ?? item.id, label: item.name, type: item.dataType, writable: true }));
|
|
33837
33837
|
}
|
|
33838
33838
|
}
|
|
33839
33839
|
} else if (provider === "zoho") {
|
|
33840
|
-
const modules = Array.isArray(
|
|
33840
|
+
const modules = Array.isArray(record2.moduleResults) ? record2.moduleResults : [record2];
|
|
33841
33841
|
for (const moduleResult of modules) {
|
|
33842
33842
|
if (!isRecord(moduleResult)) continue;
|
|
33843
33843
|
const object2 = safeString(moduleResult.module, 100);
|
|
@@ -33855,22 +33855,22 @@ function normalizeProviderSchema(provider, raw) {
|
|
|
33855
33855
|
}
|
|
33856
33856
|
}
|
|
33857
33857
|
} else if (provider === "pipedrive") {
|
|
33858
|
-
if (Array.isArray(
|
|
33859
|
-
for (const item of
|
|
33858
|
+
if (Array.isArray(record2.fields)) {
|
|
33859
|
+
for (const item of record2.fields) if (isRecord(item)) {
|
|
33860
33860
|
const object2 = item.objectType === "deal" ? "deals" : "persons";
|
|
33861
33861
|
objects.push(object2);
|
|
33862
33862
|
push(normalizedField({ object: object2, internalName: item.code, label: item.name, type: item.fieldType, writable: item.writable === true }));
|
|
33863
33863
|
}
|
|
33864
33864
|
}
|
|
33865
|
-
const stages = Array.isArray(
|
|
33866
|
-
pipelines = normalizePipelines(Array.isArray(
|
|
33865
|
+
const stages = Array.isArray(record2.stages) ? record2.stages : [];
|
|
33866
|
+
pipelines = normalizePipelines(Array.isArray(record2.pipelines) ? record2.pipelines.map((pipeline) => {
|
|
33867
33867
|
if (!isRecord(pipeline)) return pipeline;
|
|
33868
33868
|
return { ...pipeline, id: String(pipeline.id ?? ""), stages: stages.filter((stage) => isRecord(stage) && String(stage.pipelineId) === String(pipeline.id)).map((stage) => ({ ...stage, id: String(stage.id ?? "") })) };
|
|
33869
33869
|
}) : []);
|
|
33870
33870
|
} else if (provider === "keap") {
|
|
33871
33871
|
objects.push("contacts");
|
|
33872
|
-
if (Array.isArray(
|
|
33873
|
-
for (const item of
|
|
33872
|
+
if (Array.isArray(record2.contactFields)) {
|
|
33873
|
+
for (const item of record2.contactFields) if (isRecord(item)) push(normalizedField({
|
|
33874
33874
|
object: "contacts",
|
|
33875
33875
|
internalName: item.fieldName ?? item.id,
|
|
33876
33876
|
label: item.label,
|
|
@@ -33878,14 +33878,14 @@ function normalizeProviderSchema(provider, raw) {
|
|
|
33878
33878
|
writable: true
|
|
33879
33879
|
}));
|
|
33880
33880
|
}
|
|
33881
|
-
const stages = Array.isArray(
|
|
33881
|
+
const stages = Array.isArray(record2.opportunityStages) ? record2.opportunityStages : [];
|
|
33882
33882
|
if (stages.length) pipelines = [{ id: "opportunities", name: "Opportunities", stages: normalizePipelines([{ id: "opportunities", name: "Opportunities", stages }])[0]?.stages ?? [] }];
|
|
33883
33883
|
}
|
|
33884
33884
|
return {
|
|
33885
33885
|
fields,
|
|
33886
33886
|
objects: [...new Set(objects)],
|
|
33887
33887
|
pipelines,
|
|
33888
|
-
...Array.isArray(
|
|
33888
|
+
...Array.isArray(record2.allowedOperations) ? { allowedOperations: record2.allowedOperations.filter((value) => typeof value === "string") } : {}
|
|
33889
33889
|
};
|
|
33890
33890
|
}
|
|
33891
33891
|
function normalizeCreatedFields(value) {
|
|
@@ -34543,9 +34543,9 @@ function safeTags(value) {
|
|
|
34543
34543
|
}
|
|
34544
34544
|
function safeOccurredAt(...values) {
|
|
34545
34545
|
for (const value of values) {
|
|
34546
|
-
const
|
|
34547
|
-
if (!
|
|
34548
|
-
const parsed = new Date(
|
|
34546
|
+
const text2 = safeString2(value, 100);
|
|
34547
|
+
if (!text2) continue;
|
|
34548
|
+
const parsed = new Date(text2);
|
|
34549
34549
|
if (Number.isFinite(parsed.getTime())) return parsed.toISOString();
|
|
34550
34550
|
}
|
|
34551
34551
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -34562,10 +34562,10 @@ function safeSignatureEqual(actual, expected) {
|
|
|
34562
34562
|
}
|
|
34563
34563
|
}
|
|
34564
34564
|
function webhookTimestampFresh(value, now = Date.now(), toleranceMs = 5 * 60 * 1e3) {
|
|
34565
|
-
const
|
|
34566
|
-
if (!
|
|
34567
|
-
const numeric = Number(
|
|
34568
|
-
const timestamp3 = Number.isFinite(numeric) ? numeric > 1e10 ? numeric : numeric * 1e3 : Date.parse(
|
|
34565
|
+
const text2 = safeString2(value, 100);
|
|
34566
|
+
if (!text2) return false;
|
|
34567
|
+
const numeric = Number(text2);
|
|
34568
|
+
const timestamp3 = Number.isFinite(numeric) ? numeric > 1e10 ? numeric : numeric * 1e3 : Date.parse(text2);
|
|
34569
34569
|
return Number.isFinite(timestamp3) && Math.abs(now - timestamp3) <= toleranceMs;
|
|
34570
34570
|
}
|
|
34571
34571
|
|
|
@@ -34784,8 +34784,8 @@ function boundedLimit(value) {
|
|
|
34784
34784
|
function safeReferrer(value) {
|
|
34785
34785
|
if (typeof value !== "string" || !value) return null;
|
|
34786
34786
|
try {
|
|
34787
|
-
const
|
|
34788
|
-
return `${
|
|
34787
|
+
const url2 = new URL(value);
|
|
34788
|
+
return `${url2.origin}${url2.pathname}`.slice(0, 2e3);
|
|
34789
34789
|
} catch {
|
|
34790
34790
|
return null;
|
|
34791
34791
|
}
|
|
@@ -36466,10 +36466,10 @@ async function recomputeConfirmedLeadScores(input) {
|
|
|
36466
36466
|
};
|
|
36467
36467
|
const evaluationIds = [];
|
|
36468
36468
|
for (const stored of definitions.rows) {
|
|
36469
|
-
const
|
|
36470
|
-
const groups = Array.isArray(stored.rules) ? stored.rules :
|
|
36469
|
+
const record2 = stored.rules && typeof stored.rules === "object" && !Array.isArray(stored.rules) ? stored.rules : {};
|
|
36470
|
+
const groups = Array.isArray(stored.rules) ? stored.rules : record2.groups;
|
|
36471
36471
|
if (!Array.isArray(groups) || groups.length === 0) continue;
|
|
36472
|
-
const permittedProperties = Array.isArray(
|
|
36472
|
+
const permittedProperties = Array.isArray(record2.permittedProperties) ? record2.permittedProperties.filter((value) => typeof value === "string") : [];
|
|
36473
36473
|
try {
|
|
36474
36474
|
const tables = createAnalyticsLeadScoringTables();
|
|
36475
36475
|
createAnalyticsLeadScore({
|
|
@@ -36843,8 +36843,8 @@ async function getAnalyticsJourney(siteId, userId, personId) {
|
|
|
36843
36843
|
const safeReferrer2 = (value) => {
|
|
36844
36844
|
if (typeof value !== "string" || !value) return null;
|
|
36845
36845
|
try {
|
|
36846
|
-
const
|
|
36847
|
-
return `${
|
|
36846
|
+
const url2 = new URL(value);
|
|
36847
|
+
return `${url2.origin}${url2.pathname}`.slice(0, 2e3);
|
|
36848
36848
|
} catch {
|
|
36849
36849
|
return null;
|
|
36850
36850
|
}
|
|
@@ -37700,57 +37700,57 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
37700
37700
|
const clientId = get("client_id") ?? "";
|
|
37701
37701
|
const redirectUri = get("redirect_uri") ?? "";
|
|
37702
37702
|
const verifier = get("code_verifier") ?? "";
|
|
37703
|
-
const
|
|
37704
|
-
if (!
|
|
37705
|
-
if (new Date(
|
|
37706
|
-
if (
|
|
37707
|
-
if (
|
|
37708
|
-
if (!pkceMatches(verifier,
|
|
37709
|
-
const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(
|
|
37703
|
+
const record2 = await withOAuthDatabaseDeadline("consume-authorization-code", consumeCode(code));
|
|
37704
|
+
if (!record2) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
|
|
37705
|
+
if (new Date(record2.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
|
|
37706
|
+
if (record2.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
37707
|
+
if (record2.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
|
|
37708
|
+
if (!pkceMatches(verifier, record2.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
|
|
37709
|
+
const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(record2.identity));
|
|
37710
37710
|
const plan = user ? resolvePlan(user) : "free";
|
|
37711
|
-
const audience =
|
|
37712
|
-
const accessToken = await mintAccessToken(
|
|
37711
|
+
const audience = record2.resource ?? RESOURCE();
|
|
37712
|
+
const accessToken = await mintAccessToken(record2.identity, record2.scope, plan, audience);
|
|
37713
37713
|
const refreshToken = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
37714
37714
|
await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
|
|
37715
37715
|
refresh_token: refreshToken,
|
|
37716
37716
|
client_id: clientId,
|
|
37717
|
-
identity:
|
|
37718
|
-
scope:
|
|
37717
|
+
identity: record2.identity,
|
|
37718
|
+
scope: record2.scope,
|
|
37719
37719
|
resource: audience,
|
|
37720
37720
|
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
37721
37721
|
revoked: 0
|
|
37722
37722
|
}));
|
|
37723
|
-
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope:
|
|
37723
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record2.scope });
|
|
37724
37724
|
}
|
|
37725
37725
|
if (grantType === "refresh_token") {
|
|
37726
37726
|
const refreshToken = get("refresh_token") ?? "";
|
|
37727
37727
|
const clientId = get("client_id") ?? "";
|
|
37728
|
-
const
|
|
37729
|
-
if (
|
|
37730
|
-
const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(
|
|
37728
|
+
const record2 = await withOAuthDatabaseDeadline("get-refresh-token", getRefresh(refreshToken));
|
|
37729
|
+
if (record2 && record2.revoked && record2.replaced_by && record2.rotated_at && Date.now() - new Date(record2.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record2.client_id === clientId)) {
|
|
37730
|
+
const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(record2.identity));
|
|
37731
37731
|
const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
|
|
37732
|
-
const graceAudience =
|
|
37733
|
-
const graceAccess = await mintAccessToken(
|
|
37734
|
-
return c.json({ access_token: graceAccess, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token:
|
|
37735
|
-
}
|
|
37736
|
-
if (!
|
|
37737
|
-
if (new Date(
|
|
37738
|
-
if (clientId &&
|
|
37739
|
-
const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(
|
|
37732
|
+
const graceAudience = record2.resource ?? RESOURCE();
|
|
37733
|
+
const graceAccess = await mintAccessToken(record2.identity, record2.scope, gracePlan, graceAudience);
|
|
37734
|
+
return c.json({ access_token: graceAccess, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: record2.replaced_by, scope: record2.scope });
|
|
37735
|
+
}
|
|
37736
|
+
if (!record2 || record2.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
|
|
37737
|
+
if (new Date(record2.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
|
|
37738
|
+
if (clientId && record2.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
|
|
37739
|
+
const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(record2.identity));
|
|
37740
37740
|
const plan = user ? resolvePlan(user) : "free";
|
|
37741
|
-
const audience =
|
|
37742
|
-
const accessToken = await mintAccessToken(
|
|
37741
|
+
const audience = record2.resource ?? RESOURCE();
|
|
37742
|
+
const accessToken = await mintAccessToken(record2.identity, record2.scope, plan, audience);
|
|
37743
37743
|
const nextRefresh = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
37744
37744
|
await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
|
|
37745
37745
|
refresh_token: nextRefresh,
|
|
37746
|
-
client_id:
|
|
37747
|
-
identity:
|
|
37748
|
-
scope:
|
|
37746
|
+
client_id: record2.client_id,
|
|
37747
|
+
identity: record2.identity,
|
|
37748
|
+
scope: record2.scope,
|
|
37749
37749
|
resource: audience,
|
|
37750
37750
|
expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
|
|
37751
37751
|
revoked: 0
|
|
37752
37752
|
}));
|
|
37753
|
-
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope:
|
|
37753
|
+
return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record2.scope });
|
|
37754
37754
|
}
|
|
37755
37755
|
return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
|
|
37756
37756
|
});
|
|
@@ -39214,9 +39214,9 @@ function isAutomatedMailbox(email) {
|
|
|
39214
39214
|
return /^(?:no-?reply|do-?not-?reply|mailer-daemon|postmaster|dmarc|bounce|notifications?)$/.test(local);
|
|
39215
39215
|
}
|
|
39216
39216
|
function extractReplyText(value) {
|
|
39217
|
-
const
|
|
39218
|
-
if (!
|
|
39219
|
-
const lines =
|
|
39217
|
+
const text2 = String(value ?? "").replace(/\0/g, "").trim();
|
|
39218
|
+
if (!text2) return "";
|
|
39219
|
+
const lines = text2.split(/\r?\n/);
|
|
39220
39220
|
const kept = [];
|
|
39221
39221
|
for (const line of lines) {
|
|
39222
39222
|
if (/^On .+ wrote:\s*$/i.test(line.trim())) break;
|
|
@@ -40122,11 +40122,11 @@ function renderWorkshopFieldNotesBody() {
|
|
|
40122
40122
|
...workshopFieldNotes.flatMap((note, index) => {
|
|
40123
40123
|
const category = note.category === activeCategory ? [] : [`<h2 style="margin:${index === 0 ? 32 : 40}px 0 12px;color:#2563eb;font-size:13px;line-height:1.3;letter-spacing:.08em;text-transform:uppercase">${escapeHtml3(note.category)}</h2>`];
|
|
40124
40124
|
activeCategory = note.category;
|
|
40125
|
-
const
|
|
40125
|
+
const url2 = escapeHtml3(workshopFieldNoteUrl(note.slug));
|
|
40126
40126
|
const sentences = note.snippet.map(escapeHtml3).join(" ");
|
|
40127
40127
|
return [
|
|
40128
40128
|
...category,
|
|
40129
|
-
`<div style="padding:20px 0;border-top:1px solid #e5e7eb"><div style="color:#6b7280;font-size:12px;font-weight:700;letter-spacing:.06em">${String(index + 1).padStart(2, "0")}</div><h3 style="margin:6px 0 8px;font-size:19px;line-height:1.3"><a href="${
|
|
40129
|
+
`<div style="padding:20px 0;border-top:1px solid #e5e7eb"><div style="color:#6b7280;font-size:12px;font-weight:700;letter-spacing:.06em">${String(index + 1).padStart(2, "0")}</div><h3 style="margin:6px 0 8px;font-size:19px;line-height:1.3"><a href="${url2}" style="color:#101114;text-decoration:none">${escapeHtml3(note.title)}</a></h3><p style="margin:0;color:#454952;font-size:15px;line-height:1.65">${sentences}</p><p style="margin:10px 0 0"><a href="${url2}" style="color:#2563eb;font-size:14px;font-weight:700;text-decoration:none">Read this note \u2192</a></p></div>`
|
|
40130
40130
|
];
|
|
40131
40131
|
})
|
|
40132
40132
|
].join("");
|
|
@@ -40298,8 +40298,8 @@ function eventConfig() {
|
|
|
40298
40298
|
}
|
|
40299
40299
|
function isZoomJoinUrl(value) {
|
|
40300
40300
|
try {
|
|
40301
|
-
const
|
|
40302
|
-
return
|
|
40301
|
+
const url2 = new URL(value);
|
|
40302
|
+
return url2.protocol === "https:" && (url2.hostname === "zoom.us" || url2.hostname.endsWith(".zoom.us"));
|
|
40303
40303
|
} catch {
|
|
40304
40304
|
return false;
|
|
40305
40305
|
}
|
|
@@ -40715,7 +40715,7 @@ function safeUrl(value, allowImage = false) {
|
|
|
40715
40715
|
}
|
|
40716
40716
|
function markdownRenderer() {
|
|
40717
40717
|
const renderer = new Renderer();
|
|
40718
|
-
renderer.html = ({ text }) => `<pre><code>${escapeHtml5(
|
|
40718
|
+
renderer.html = ({ text: text2 }) => `<pre><code>${escapeHtml5(text2)}</code></pre>`;
|
|
40719
40719
|
renderer.link = function({ href, title, tokens: tokens2 }) {
|
|
40720
40720
|
const label = this.parser.parseInline(tokens2);
|
|
40721
40721
|
const safe = safeUrl(href);
|
|
@@ -40724,11 +40724,11 @@ function markdownRenderer() {
|
|
|
40724
40724
|
const external = safe.startsWith("http://") || safe.startsWith("https://") ? ' target="_blank" rel="noopener noreferrer"' : "";
|
|
40725
40725
|
return `<a href="${escapeHtml5(safe)}"${titleAttribute}${external}>${label}</a>`;
|
|
40726
40726
|
};
|
|
40727
|
-
renderer.image = ({ href, title, text }) => {
|
|
40727
|
+
renderer.image = ({ href, title, text: text2 }) => {
|
|
40728
40728
|
const safe = safeUrl(href, true);
|
|
40729
|
-
if (!safe) return `<em>[Image omitted: ${escapeHtml5(
|
|
40729
|
+
if (!safe) return `<em>[Image omitted: ${escapeHtml5(text2)}]</em>`;
|
|
40730
40730
|
const titleAttribute = title ? ` title="${escapeHtml5(title)}"` : "";
|
|
40731
|
-
return `<img src="${escapeHtml5(safe)}" alt="${escapeHtml5(
|
|
40731
|
+
return `<img src="${escapeHtml5(safe)}" alt="${escapeHtml5(text2)}"${titleAttribute} loading="lazy">`;
|
|
40732
40732
|
};
|
|
40733
40733
|
return new Marked({ gfm: true, breaks: false, renderer });
|
|
40734
40734
|
}
|
|
@@ -41112,19 +41112,19 @@ function isBlockedAddress(address2) {
|
|
|
41112
41112
|
return false;
|
|
41113
41113
|
}
|
|
41114
41114
|
async function assertPublicHttpsUrl(rawUrl) {
|
|
41115
|
-
let
|
|
41115
|
+
let url2;
|
|
41116
41116
|
try {
|
|
41117
|
-
|
|
41117
|
+
url2 = new URL(rawUrl);
|
|
41118
41118
|
} catch {
|
|
41119
41119
|
throw new CommonsImageError("image_url_invalid", "The image source is not a valid URL.");
|
|
41120
41120
|
}
|
|
41121
|
-
if (
|
|
41121
|
+
if (url2.protocol !== "https:") {
|
|
41122
41122
|
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
41123
41123
|
}
|
|
41124
|
-
if (
|
|
41124
|
+
if (url2.username || url2.password) {
|
|
41125
41125
|
throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
|
|
41126
41126
|
}
|
|
41127
|
-
const hostname =
|
|
41127
|
+
const hostname = url2.hostname.replace(/^\[|\]$/g, "");
|
|
41128
41128
|
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
41129
41129
|
if (!addresses.length) {
|
|
41130
41130
|
throw new CommonsImageError("image_host_unresolvable", "The image source host could not be resolved.");
|
|
@@ -41132,7 +41132,7 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
41132
41132
|
if (addresses.some((entry) => isBlockedAddress(entry.address))) {
|
|
41133
41133
|
throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
|
|
41134
41134
|
}
|
|
41135
|
-
return
|
|
41135
|
+
return url2;
|
|
41136
41136
|
}
|
|
41137
41137
|
async function readResponseBytes(response, maxBytes) {
|
|
41138
41138
|
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
@@ -41158,10 +41158,10 @@ async function readResponseBytes(response, maxBytes) {
|
|
|
41158
41158
|
async function downloadPublicSource(sourceUrl) {
|
|
41159
41159
|
let current = sourceUrl;
|
|
41160
41160
|
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
41161
|
-
const
|
|
41161
|
+
const url2 = await assertPublicHttpsUrl(current);
|
|
41162
41162
|
let response;
|
|
41163
41163
|
try {
|
|
41164
|
-
response = await fetch(
|
|
41164
|
+
response = await fetch(url2, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
|
|
41165
41165
|
} catch (error) {
|
|
41166
41166
|
if (error instanceof CommonsImageError) throw error;
|
|
41167
41167
|
throw new CommonsImageError(
|
|
@@ -41175,7 +41175,7 @@ async function downloadPublicSource(sourceUrl) {
|
|
|
41175
41175
|
const location2 = response.headers.get("location");
|
|
41176
41176
|
if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
|
|
41177
41177
|
try {
|
|
41178
|
-
current = new URL(location2,
|
|
41178
|
+
current = new URL(location2, url2).toString();
|
|
41179
41179
|
} catch {
|
|
41180
41180
|
throw new CommonsImageError("image_redirect_invalid", "Image source redirected to an invalid destination.");
|
|
41181
41181
|
}
|
|
@@ -41192,7 +41192,7 @@ async function downloadPublicSource(sourceUrl) {
|
|
|
41192
41192
|
}
|
|
41193
41193
|
return {
|
|
41194
41194
|
bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
|
|
41195
|
-
finalUrl:
|
|
41195
|
+
finalUrl: url2.toString(),
|
|
41196
41196
|
contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
|
|
41197
41197
|
};
|
|
41198
41198
|
}
|
|
@@ -41209,12 +41209,12 @@ function decodeBase64Image(value) {
|
|
|
41209
41209
|
}
|
|
41210
41210
|
function safeProvenanceUrl(rawUrl) {
|
|
41211
41211
|
try {
|
|
41212
|
-
const
|
|
41213
|
-
|
|
41214
|
-
|
|
41215
|
-
|
|
41216
|
-
|
|
41217
|
-
return
|
|
41212
|
+
const url2 = new URL(rawUrl);
|
|
41213
|
+
url2.username = "";
|
|
41214
|
+
url2.password = "";
|
|
41215
|
+
url2.search = "";
|
|
41216
|
+
url2.hash = "";
|
|
41217
|
+
return url2.toString();
|
|
41218
41218
|
} catch {
|
|
41219
41219
|
return null;
|
|
41220
41220
|
}
|
|
@@ -41321,21 +41321,21 @@ function commonsImageBlobStore() {
|
|
|
41321
41321
|
}
|
|
41322
41322
|
function normalizeCommonsImageUrl(rawUrl) {
|
|
41323
41323
|
try {
|
|
41324
|
-
const
|
|
41325
|
-
if (
|
|
41326
|
-
|
|
41327
|
-
return
|
|
41324
|
+
const url2 = new URL(rawUrl.trim());
|
|
41325
|
+
if (url2.protocol !== "https:") return null;
|
|
41326
|
+
url2.hash = "";
|
|
41327
|
+
return url2.toString();
|
|
41328
41328
|
} catch {
|
|
41329
41329
|
return null;
|
|
41330
41330
|
}
|
|
41331
41331
|
}
|
|
41332
41332
|
async function findCommonsHostedImageByUrl(rawUrl) {
|
|
41333
|
-
const
|
|
41334
|
-
if (!
|
|
41333
|
+
const url2 = normalizeCommonsImageUrl(rawUrl);
|
|
41334
|
+
if (!url2) return null;
|
|
41335
41335
|
await ensureCommonsImageSchema();
|
|
41336
41336
|
const result2 = await getDb().execute({
|
|
41337
41337
|
sql: "SELECT id, url, content_type, bytes, digest, source_url FROM commons_images WHERE url = ? LIMIT 1",
|
|
41338
|
-
args: [
|
|
41338
|
+
args: [url2]
|
|
41339
41339
|
});
|
|
41340
41340
|
const row = result2.rows[0];
|
|
41341
41341
|
if (!row) return null;
|
|
@@ -41613,8 +41613,8 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
41613
41613
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
41614
41614
|
}
|
|
41615
41615
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
41616
|
-
const
|
|
41617
|
-
return `<${
|
|
41616
|
+
const url2 = commonsLinksetUrl(idOrSlug);
|
|
41617
|
+
return `<${url2}>; rel="linkset"; type="${COMMONS_LINKSET_MEDIA_TYPE}"; profile="${COMMONS_RELATIONSHIP_PROFILE}"`;
|
|
41618
41618
|
}
|
|
41619
41619
|
function commonsLinksetUrl(idOrSlug) {
|
|
41620
41620
|
const base = (process.env.APP_URL?.trim() || "https://mcpscraper.dev").replace(/\/+$/, "");
|
|
@@ -41629,32 +41629,32 @@ function commonsPublicWikiBaseUrl() {
|
|
|
41629
41629
|
function isSupportedPredicate(value) {
|
|
41630
41630
|
if (REGISTERED_RELATIONS.has(value)) return true;
|
|
41631
41631
|
try {
|
|
41632
|
-
const
|
|
41633
|
-
return
|
|
41632
|
+
const url2 = new URL(value);
|
|
41633
|
+
return url2.protocol === "https:" && Boolean(url2.hostname);
|
|
41634
41634
|
} catch {
|
|
41635
41635
|
return false;
|
|
41636
41636
|
}
|
|
41637
41637
|
}
|
|
41638
41638
|
function optionalText(value, max) {
|
|
41639
|
-
const
|
|
41640
|
-
return
|
|
41639
|
+
const text2 = String(value ?? "").trim();
|
|
41640
|
+
return text2 ? text2.slice(0, max) : void 0;
|
|
41641
41641
|
}
|
|
41642
41642
|
function optionalHttpsUrl(value, field) {
|
|
41643
|
-
const
|
|
41644
|
-
if (!
|
|
41643
|
+
const text2 = optionalText(value, 2e3);
|
|
41644
|
+
if (!text2) return void 0;
|
|
41645
41645
|
try {
|
|
41646
|
-
const
|
|
41647
|
-
if (
|
|
41648
|
-
|
|
41649
|
-
return
|
|
41646
|
+
const url2 = new URL(text2);
|
|
41647
|
+
if (url2.protocol !== "https:") throw new Error();
|
|
41648
|
+
url2.hash = "";
|
|
41649
|
+
return url2.href;
|
|
41650
41650
|
} catch {
|
|
41651
41651
|
throw new Error(`${field} must be an absolute HTTPS URL.`);
|
|
41652
41652
|
}
|
|
41653
41653
|
}
|
|
41654
41654
|
function optionalDate(value, field) {
|
|
41655
|
-
const
|
|
41656
|
-
if (!
|
|
41657
|
-
const timestamp3 = Date.parse(
|
|
41655
|
+
const text2 = optionalText(value, 80);
|
|
41656
|
+
if (!text2) return void 0;
|
|
41657
|
+
const timestamp3 = Date.parse(text2);
|
|
41658
41658
|
if (!Number.isFinite(timestamp3)) throw new Error(`${field} must be an ISO-compatible date.`);
|
|
41659
41659
|
return new Date(timestamp3).toISOString();
|
|
41660
41660
|
}
|
|
@@ -42197,18 +42197,18 @@ async function searchCommonsEntities(filters2 = {}, userId) {
|
|
|
42197
42197
|
});
|
|
42198
42198
|
const mode = searchMode;
|
|
42199
42199
|
const ranked = candidates.rows.map((row) => {
|
|
42200
|
-
const
|
|
42200
|
+
const record2 = row;
|
|
42201
42201
|
const score = relevanceScore(terms, {
|
|
42202
|
-
title: String(
|
|
42203
|
-
description: String(
|
|
42204
|
-
tags: parseJsonArray(
|
|
42205
|
-
keywords: parseJsonArray(
|
|
42206
|
-
searchText: String(
|
|
42207
|
-
claimText: String(
|
|
42208
|
-
});
|
|
42209
|
-
const semantic = semanticScores.get(String(
|
|
42202
|
+
title: String(record2.title ?? ""),
|
|
42203
|
+
description: String(record2.description ?? ""),
|
|
42204
|
+
tags: parseJsonArray(record2.tags_json),
|
|
42205
|
+
keywords: parseJsonArray(record2.keywords_json),
|
|
42206
|
+
searchText: String(record2.search_text ?? ""),
|
|
42207
|
+
claimText: String(record2.claim_text ?? "")
|
|
42208
|
+
});
|
|
42209
|
+
const semantic = semanticScores.get(String(record2.id)) ?? 0;
|
|
42210
42210
|
const fused = mode === "semantic" ? semantic * 100 : score + semantic * 60;
|
|
42211
|
-
return { id: String(
|
|
42211
|
+
return { id: String(record2.id), updatedAt: String(record2.updated_at ?? ""), title: String(record2.title ?? ""), score: fused, lexical: score, semantic };
|
|
42212
42212
|
}).filter((candidate) => mode === "semantic" ? candidate.semantic > 0 : candidate.lexical > 0 || candidate.semantic >= 0.62).sort((left, right) => right.score - left.score || right.updatedAt.localeCompare(left.updatedAt) || left.title.localeCompare(right.title));
|
|
42213
42213
|
const pageIds = ranked.slice(offset, offset + limit).map((candidate) => candidate.id);
|
|
42214
42214
|
if (!pageIds.length) return { results: [], filters: mergedFilters, limit, offset };
|
|
@@ -42921,7 +42921,7 @@ async function potentialDuplicateCandidates(input) {
|
|
|
42921
42921
|
candidates.set(entity.id, entity);
|
|
42922
42922
|
continue;
|
|
42923
42923
|
}
|
|
42924
|
-
if ([...sourceUrlSet(entity.source)].some((
|
|
42924
|
+
if ([...sourceUrlSet(entity.source)].some((url2) => sourceUrls.has(url2))) candidates.set(entity.id, entity);
|
|
42925
42925
|
}
|
|
42926
42926
|
}
|
|
42927
42927
|
return [...candidates.values()].slice(0, input.maxCandidates).map((entity) => ({
|
|
@@ -42952,10 +42952,10 @@ async function commonsTagVocabulary() {
|
|
|
42952
42952
|
});
|
|
42953
42953
|
const counts = /* @__PURE__ */ new Map();
|
|
42954
42954
|
for (const row of result2.rows) {
|
|
42955
|
-
const
|
|
42955
|
+
const record2 = row;
|
|
42956
42956
|
for (const tag of [
|
|
42957
|
-
...parseJson4(
|
|
42958
|
-
...parseJson4(
|
|
42957
|
+
...parseJson4(record2.tags_json, []),
|
|
42958
|
+
...parseJson4(record2.keywords_json, [])
|
|
42959
42959
|
]) {
|
|
42960
42960
|
const cleaned = optionalText2(tag, 120);
|
|
42961
42961
|
if (!cleaned) continue;
|
|
@@ -42988,11 +42988,11 @@ function resolveCommonsTags(candidates, vocabulary) {
|
|
|
42988
42988
|
}
|
|
42989
42989
|
function extractMarkdownHeadings(markdown) {
|
|
42990
42990
|
return String(markdown || "").split(/\r?\n/).map((line) => line.match(/^\s{0,3}(#{2,5})\s+(.+?)\s*#*\s*$/)).filter((match) => Boolean(match)).map((match) => {
|
|
42991
|
-
const
|
|
42991
|
+
const text2 = stripInlineMarkdown(match[2]).slice(0, 180);
|
|
42992
42992
|
return {
|
|
42993
42993
|
level: match[1].length,
|
|
42994
|
-
text,
|
|
42995
|
-
id: normalizeSlug(
|
|
42994
|
+
text: text2,
|
|
42995
|
+
id: normalizeSlug(text2)
|
|
42996
42996
|
};
|
|
42997
42997
|
});
|
|
42998
42998
|
}
|
|
@@ -43044,10 +43044,10 @@ function sourceUrlSet(source) {
|
|
|
43044
43044
|
}
|
|
43045
43045
|
function canonicalizeSourceUrl(value) {
|
|
43046
43046
|
try {
|
|
43047
|
-
const
|
|
43048
|
-
|
|
43049
|
-
if (
|
|
43050
|
-
return
|
|
43047
|
+
const url2 = new URL(value);
|
|
43048
|
+
url2.hash = "";
|
|
43049
|
+
if (url2.pathname !== "/") url2.pathname = url2.pathname.replace(/\/+$/, "");
|
|
43050
|
+
return url2.href.replace(/\/+$/, "");
|
|
43051
43051
|
} catch {
|
|
43052
43052
|
return String(value || "").trim().replace(/\/+$/, "");
|
|
43053
43053
|
}
|
|
@@ -43613,13 +43613,13 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
43613
43613
|
if (!Array.isArray(relatedLinks)) return [];
|
|
43614
43614
|
return relatedLinks.map((link) => {
|
|
43615
43615
|
const title = requiredText(link.title, "relatedLinks.title", 1, 180);
|
|
43616
|
-
const
|
|
43616
|
+
const url2 = optionalText2(link.url, 2e3);
|
|
43617
43617
|
const slug4 = link.slug ? normalizeSlug(link.slug) : void 0;
|
|
43618
43618
|
const entityId = normalizeEntityId(link.entityId);
|
|
43619
|
-
const needsLink = link.needsLink === true || !
|
|
43619
|
+
const needsLink = link.needsLink === true || !url2 && !slug4 && !entityId;
|
|
43620
43620
|
return {
|
|
43621
43621
|
title,
|
|
43622
|
-
url,
|
|
43622
|
+
url: url2,
|
|
43623
43623
|
slug: slug4,
|
|
43624
43624
|
entityId,
|
|
43625
43625
|
relationship: optionalText2(link.relationship, 120) ?? void 0,
|
|
@@ -43774,8 +43774,8 @@ async function publishedEntityIndexFor(titles) {
|
|
|
43774
43774
|
args: [...slugKeys, ...titleKeys]
|
|
43775
43775
|
});
|
|
43776
43776
|
for (const row of result2.rows) {
|
|
43777
|
-
const
|
|
43778
|
-
const entry = { id: String(
|
|
43777
|
+
const record2 = row;
|
|
43778
|
+
const entry = { id: String(record2.id), slug: String(record2.slug), title: String(record2.title) };
|
|
43779
43779
|
index.set(entry.slug, entry);
|
|
43780
43780
|
index.set(normalizeNeedsLinkTitle(entry.title), entry);
|
|
43781
43781
|
}
|
|
@@ -43900,9 +43900,9 @@ function allocateClaimId() {
|
|
|
43900
43900
|
return `TPW-S${Date.now()}${Math.floor(Math.random() * 1e6).toString().padStart(6, "0")}`;
|
|
43901
43901
|
}
|
|
43902
43902
|
function normalizeEntityId(value) {
|
|
43903
|
-
const
|
|
43904
|
-
if (!
|
|
43905
|
-
return
|
|
43903
|
+
const text2 = optionalText2(value, 80);
|
|
43904
|
+
if (!text2) return void 0;
|
|
43905
|
+
return text2.replace(/\s+/g, "").toUpperCase();
|
|
43906
43906
|
}
|
|
43907
43907
|
function normalizeEntityType(value) {
|
|
43908
43908
|
const raw = optionalText2(value, 120) || DEFAULT_ENTITY_TYPE;
|
|
@@ -43929,13 +43929,13 @@ function normalizeSlug(value) {
|
|
|
43929
43929
|
return String(value || "").trim().toLowerCase().replace(/['"]/g, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 140);
|
|
43930
43930
|
}
|
|
43931
43931
|
function requiredText(value, field, min, max) {
|
|
43932
|
-
const
|
|
43933
|
-
if (
|
|
43934
|
-
return
|
|
43932
|
+
const text2 = String(value ?? "").trim();
|
|
43933
|
+
if (text2.length < min) throw new CommonsRepositoryError(`${field.replaceAll(".", "_")}_required`, `${field} must be at least ${min} characters.`);
|
|
43934
|
+
return text2.slice(0, max);
|
|
43935
43935
|
}
|
|
43936
43936
|
function optionalText2(value, max) {
|
|
43937
|
-
const
|
|
43938
|
-
return
|
|
43937
|
+
const text2 = String(value ?? "").trim();
|
|
43938
|
+
return text2 ? text2.slice(0, max) : void 0;
|
|
43939
43939
|
}
|
|
43940
43940
|
function normalizeRecord(value) {
|
|
43941
43941
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
@@ -43953,12 +43953,12 @@ function uniqueStrings(values) {
|
|
|
43953
43953
|
const seen = /* @__PURE__ */ new Set();
|
|
43954
43954
|
const out = [];
|
|
43955
43955
|
for (const value of values) {
|
|
43956
|
-
const
|
|
43957
|
-
if (!
|
|
43958
|
-
const key2 =
|
|
43956
|
+
const text2 = optionalText2(value, 120);
|
|
43957
|
+
if (!text2) continue;
|
|
43958
|
+
const key2 = text2.toLowerCase();
|
|
43959
43959
|
if (seen.has(key2)) continue;
|
|
43960
43960
|
seen.add(key2);
|
|
43961
|
-
out.push(
|
|
43961
|
+
out.push(text2);
|
|
43962
43962
|
}
|
|
43963
43963
|
return out;
|
|
43964
43964
|
}
|
|
@@ -44447,8 +44447,8 @@ async function getCommonsPublicationEditionHtml(subdomainInput, editionSlugInput
|
|
|
44447
44447
|
args: [publication.latestEditionId || "", publication.id]
|
|
44448
44448
|
});
|
|
44449
44449
|
if (!result2.rows[0]) return null;
|
|
44450
|
-
const
|
|
44451
|
-
return { publication, edition: rowToEdition(
|
|
44450
|
+
const record2 = result2.rows[0];
|
|
44451
|
+
return { publication, edition: rowToEdition(record2, publication.subdomain), html: String(record2.html || "") };
|
|
44452
44452
|
}
|
|
44453
44453
|
async function getPublicationById(id2) {
|
|
44454
44454
|
const result2 = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE id = ? LIMIT 1", args: [id2] });
|
|
@@ -44469,8 +44469,8 @@ async function getEditionByIdempotency(userId, idempotencyKey4) {
|
|
|
44469
44469
|
args: [userId, idempotencyKey4]
|
|
44470
44470
|
});
|
|
44471
44471
|
if (!result2.rows[0]) return null;
|
|
44472
|
-
const
|
|
44473
|
-
return rowToEdition(
|
|
44472
|
+
const record2 = result2.rows[0];
|
|
44473
|
+
return rowToEdition(record2, String(record2.subdomain));
|
|
44474
44474
|
}
|
|
44475
44475
|
async function getPublicationEditionById(id2) {
|
|
44476
44476
|
const result2 = await getDb().execute({
|
|
@@ -44478,8 +44478,8 @@ async function getPublicationEditionById(id2) {
|
|
|
44478
44478
|
args: [id2]
|
|
44479
44479
|
});
|
|
44480
44480
|
if (!result2.rows[0]) return null;
|
|
44481
|
-
const
|
|
44482
|
-
return rowToEdition(
|
|
44481
|
+
const record2 = result2.rows[0];
|
|
44482
|
+
return rowToEdition(record2, String(record2.subdomain));
|
|
44483
44483
|
}
|
|
44484
44484
|
function rowToPublication(row) {
|
|
44485
44485
|
const subdomain = String(row.subdomain);
|
|
@@ -45168,9 +45168,9 @@ function canonical(value) {
|
|
|
45168
45168
|
return JSON.stringify(value);
|
|
45169
45169
|
}
|
|
45170
45170
|
function safeCsvCell(value) {
|
|
45171
|
-
let
|
|
45172
|
-
if (/^[=+\-@\t\r]/.test(
|
|
45173
|
-
return /[\n\r,"]/.test(
|
|
45171
|
+
let text2 = value == null ? "" : String(value);
|
|
45172
|
+
if (/^[=+\-@\t\r]/.test(text2)) text2 = `'${text2}`;
|
|
45173
|
+
return /[\n\r,"]/.test(text2) ? `"${text2.replaceAll('"', '""')}"` : text2;
|
|
45174
45174
|
}
|
|
45175
45175
|
function analyticsManualImportErrorCsv(errors) {
|
|
45176
45176
|
return ["row,code,field,message", ...errors.map((error) => [error.row, error.code, error.field, error.message].map(safeCsvCell).join(","))].join("\n");
|
|
@@ -45292,9 +45292,9 @@ var AnalyticsManualExportRequestSchema = z38.object({
|
|
|
45292
45292
|
acknowledgedPurpose: z38.enum(["business_operations", "advertising_measurement"]).optional()
|
|
45293
45293
|
}).strict();
|
|
45294
45294
|
function csvCell(value) {
|
|
45295
|
-
let
|
|
45296
|
-
if (/^[=+\-@\t\r]/.test(
|
|
45297
|
-
return /[\n\r,"]/.test(
|
|
45295
|
+
let text2 = value == null ? "" : String(value);
|
|
45296
|
+
if (/^[=+\-@\t\r]/.test(text2)) text2 = `'${text2}`;
|
|
45297
|
+
return /[\n\r,"]/.test(text2) ? `"${text2.replaceAll('"', '""')}"` : text2;
|
|
45298
45298
|
}
|
|
45299
45299
|
function csv(headers, rows) {
|
|
45300
45300
|
return [headers.join(","), ...rows.map((row) => headers.map((header) => csvCell(row[header])).join(","))].join("\n");
|
|
@@ -45563,7 +45563,7 @@ function resolveConnectedDataExportScope(dataset, scope) {
|
|
|
45563
45563
|
}
|
|
45564
45564
|
function previewRecord(value) {
|
|
45565
45565
|
if (!value || typeof value !== "object" || Array.isArray(value)) return value;
|
|
45566
|
-
const
|
|
45566
|
+
const record2 = value;
|
|
45567
45567
|
const preview2 = {};
|
|
45568
45568
|
for (const key2 of [
|
|
45569
45569
|
"recordType",
|
|
@@ -45596,11 +45596,11 @@ function previewRecord(value) {
|
|
|
45596
45596
|
"text",
|
|
45597
45597
|
"replyCount"
|
|
45598
45598
|
]) {
|
|
45599
|
-
if (
|
|
45600
|
-
const item =
|
|
45599
|
+
if (record2[key2] === void 0) continue;
|
|
45600
|
+
const item = record2[key2];
|
|
45601
45601
|
preview2[key2] = typeof item === "string" && item.length > 240 ? `${item.slice(0, 240)}\u2026` : item;
|
|
45602
45602
|
}
|
|
45603
|
-
return Object.keys(preview2).length > 0 ? preview2 : { recordType:
|
|
45603
|
+
return Object.keys(preview2).length > 0 ? preview2 : { recordType: record2.recordType ?? "connected_data_record" };
|
|
45604
45604
|
}
|
|
45605
45605
|
function finitePositive(value, fallback) {
|
|
45606
45606
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
@@ -45647,7 +45647,7 @@ async function collectConnectedDataExport(args) {
|
|
|
45647
45647
|
listed += page2.counts?.listed ?? page2.records.length;
|
|
45648
45648
|
failed += page2.counts?.failed ?? 0;
|
|
45649
45649
|
for (const warning of page2.warnings ?? []) warnings.add(warning);
|
|
45650
|
-
const pageLines = page2.records.map((
|
|
45650
|
+
const pageLines = page2.records.map((record2) => JSON.stringify({ type: "record", record: record2 }));
|
|
45651
45651
|
const pageBytes = pageLines.reduce((sum, line) => sum + Buffer.byteLength(line) + 1, 0);
|
|
45652
45652
|
if (recordBytes + pageBytes > maxBytes) {
|
|
45653
45653
|
continuationCursor = beforePageCursor ?? null;
|
|
@@ -45752,21 +45752,21 @@ function deriveSlackMessageIndexFields(args) {
|
|
|
45752
45752
|
return name ? [name] : [];
|
|
45753
45753
|
});
|
|
45754
45754
|
const urls = /* @__PURE__ */ new Set();
|
|
45755
|
-
const
|
|
45756
|
-
for (const match of
|
|
45757
|
-
const
|
|
45758
|
-
if (
|
|
45755
|
+
const text2 = args.text ?? "";
|
|
45756
|
+
for (const match of text2.matchAll(/<(https?:\/\/[^>|]+)(?:\|[^>]*)?>/gi)) {
|
|
45757
|
+
const url2 = normalizedUrl(match[1]);
|
|
45758
|
+
if (url2) urls.add(url2);
|
|
45759
45759
|
}
|
|
45760
|
-
for (const match of
|
|
45761
|
-
const
|
|
45762
|
-
if (
|
|
45760
|
+
for (const match of text2.matchAll(/https?:\/\/[^\s<>()]+/gi)) {
|
|
45761
|
+
const url2 = normalizedUrl(match[0]);
|
|
45762
|
+
if (url2) urls.add(url2);
|
|
45763
45763
|
}
|
|
45764
45764
|
const attachments = Array.isArray(args.providerData.attachments) ? args.providerData.attachments : [];
|
|
45765
45765
|
for (const attachment of attachments) {
|
|
45766
45766
|
if (!isRecord2(attachment)) continue;
|
|
45767
45767
|
for (const key2 of ["url", "title_link", "from_url", "service_url"]) {
|
|
45768
|
-
const
|
|
45769
|
-
if (
|
|
45768
|
+
const url2 = normalizedUrl(attachment[key2]);
|
|
45769
|
+
if (url2) urls.add(url2);
|
|
45770
45770
|
}
|
|
45771
45771
|
}
|
|
45772
45772
|
const profile = isRecord2(args.providerData.user_profile) ? args.providerData.user_profile : isRecord2(args.providerData.profile) ? args.providerData.profile : null;
|
|
@@ -45848,7 +45848,7 @@ function normalizeSlackMessage(row, channelId, parentMessageTs) {
|
|
|
45848
45848
|
const threadTs = inferredParentMessageTs ?? rowThreadTs ?? (replyCount > 0 ? timestamp3 : null);
|
|
45849
45849
|
const occurredAt = slackTimestampToIso(timestamp3);
|
|
45850
45850
|
const files = Array.isArray(row.files) ? row.files : [];
|
|
45851
|
-
const
|
|
45851
|
+
const text2 = typeof row.text === "string" ? row.text : null;
|
|
45852
45852
|
return {
|
|
45853
45853
|
recordType: inferredParentMessageTs ? "slack_thread_reply" : "slack_message",
|
|
45854
45854
|
dataset: "slack_channel_messages",
|
|
@@ -45860,7 +45860,7 @@ function normalizeSlackMessage(row, channelId, parentMessageTs) {
|
|
|
45860
45860
|
isThreadReply: inferredParentMessageTs !== null,
|
|
45861
45861
|
occurredAt,
|
|
45862
45862
|
userId: cleanString2(row.user, 100) ?? cleanString2(row.bot_id, 100),
|
|
45863
|
-
text,
|
|
45863
|
+
text: text2,
|
|
45864
45864
|
subtype: cleanString2(row.subtype, 100),
|
|
45865
45865
|
replyCount,
|
|
45866
45866
|
files,
|
|
@@ -45868,7 +45868,7 @@ function normalizeSlackMessage(row, channelId, parentMessageTs) {
|
|
|
45868
45868
|
providerData: row,
|
|
45869
45869
|
...deriveSlackMessageIndexFields({
|
|
45870
45870
|
occurredAt,
|
|
45871
|
-
text,
|
|
45871
|
+
text: text2,
|
|
45872
45872
|
files,
|
|
45873
45873
|
providerData: row
|
|
45874
45874
|
})
|
|
@@ -45951,12 +45951,12 @@ function errorText(error) {
|
|
|
45951
45951
|
}
|
|
45952
45952
|
}
|
|
45953
45953
|
function retryAfterMilliseconds(error, attempt) {
|
|
45954
|
-
const
|
|
45955
|
-
if (!/429|rate.?limit|resource_exhausted/i.test(
|
|
45954
|
+
const text2 = errorText(error);
|
|
45955
|
+
if (!/429|rate.?limit|resource_exhausted/i.test(text2)) return null;
|
|
45956
45956
|
if (isRecord3(error) && typeof error.retryAfterMs === "number" && Number.isFinite(error.retryAfterMs)) {
|
|
45957
45957
|
return Math.min(Math.max(error.retryAfterMs, 1e3), 6e4);
|
|
45958
45958
|
}
|
|
45959
|
-
const explicit =
|
|
45959
|
+
const explicit = text2.match(/retry[_ -]?after[^0-9]{0,20}(\d{1,4})/i);
|
|
45960
45960
|
const seconds = explicit ? Number(explicit[1]) : attempt;
|
|
45961
45961
|
return Math.min(Math.max(seconds, 1) * 1e3, 6e4);
|
|
45962
45962
|
}
|
|
@@ -46249,9 +46249,9 @@ function cleanString3(value, max = 300) {
|
|
|
46249
46249
|
const trimmed = value.trim();
|
|
46250
46250
|
return trimmed ? trimmed.slice(0, max) : null;
|
|
46251
46251
|
}
|
|
46252
|
-
function firstString(
|
|
46252
|
+
function firstString(record2, keys, max = 300) {
|
|
46253
46253
|
for (const key2 of keys) {
|
|
46254
|
-
const value = cleanString3(
|
|
46254
|
+
const value = cleanString3(record2[key2], max);
|
|
46255
46255
|
if (value) return value;
|
|
46256
46256
|
}
|
|
46257
46257
|
return null;
|
|
@@ -46270,8 +46270,8 @@ function cleanHttpsUrl(value) {
|
|
|
46270
46270
|
const candidate = cleanString3(value, 2e3);
|
|
46271
46271
|
if (!candidate) return null;
|
|
46272
46272
|
try {
|
|
46273
|
-
const
|
|
46274
|
-
return
|
|
46273
|
+
const url2 = new URL(candidate);
|
|
46274
|
+
return url2.protocol === "https:" ? url2.toString() : null;
|
|
46275
46275
|
} catch {
|
|
46276
46276
|
return null;
|
|
46277
46277
|
}
|
|
@@ -46360,14 +46360,14 @@ function defaultControlErrorMessage(status) {
|
|
|
46360
46360
|
function safeControlErrorPayload(body, responseStatus) {
|
|
46361
46361
|
const status = controlErrorStatus(responseStatus);
|
|
46362
46362
|
const unwrapped = unwrapData(body);
|
|
46363
|
-
const
|
|
46364
|
-
const candidateCode =
|
|
46363
|
+
const record2 = isRecord4(unwrapped) ? unwrapped : isRecord4(body) ? body : null;
|
|
46364
|
+
const candidateCode = record2 ? firstString(record2, ["code", "errorCode", "error_code"], 100) : null;
|
|
46365
46365
|
const normalizedCandidateCode = candidateCode ? CONTROL_ERROR_CODE_ALIASES.get(candidateCode) ?? candidateCode : null;
|
|
46366
46366
|
const code = normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) ? normalizedCandidateCode : defaultControlErrorCode(status);
|
|
46367
|
-
const candidateMessage =
|
|
46367
|
+
const candidateMessage = record2 ? firstString(record2, ["error", "message"], 500) : null;
|
|
46368
46368
|
const fixedMessage = FIXED_CONTROL_ERROR_MESSAGES.get(code);
|
|
46369
46369
|
const message = fixedMessage ?? (normalizedCandidateCode && SAFE_CONTROL_ERROR_CODES.has(normalizedCandidateCode) && candidateMessage ? candidateMessage.replace(/[\u0000-\u001f\u007f]/g, " ") : defaultControlErrorMessage(status));
|
|
46370
|
-
const retryable =
|
|
46370
|
+
const retryable = record2 && typeof record2.retryable === "boolean" ? record2.retryable : status === 429 || status >= 500;
|
|
46371
46371
|
return { status, code, message, retryable };
|
|
46372
46372
|
}
|
|
46373
46373
|
async function controlRequest(path5, init, timeoutMs = 2e4) {
|
|
@@ -46389,10 +46389,10 @@ async function controlRequest(path5, init, timeoutMs = 2e4) {
|
|
|
46389
46389
|
true
|
|
46390
46390
|
);
|
|
46391
46391
|
});
|
|
46392
|
-
const
|
|
46392
|
+
const text2 = await response.text();
|
|
46393
46393
|
let body = {};
|
|
46394
46394
|
try {
|
|
46395
|
-
body =
|
|
46395
|
+
body = text2 ? JSON.parse(text2) : {};
|
|
46396
46396
|
} catch {
|
|
46397
46397
|
body = {};
|
|
46398
46398
|
}
|
|
@@ -47293,13 +47293,13 @@ function operationItems(tool, value) {
|
|
|
47293
47293
|
}
|
|
47294
47294
|
function normalizeAnalyticsActivationAssets(tool, value, accountName) {
|
|
47295
47295
|
return operationItems(tool, value).flatMap((item) => {
|
|
47296
|
-
const
|
|
47296
|
+
const record2 = recordValue(item);
|
|
47297
47297
|
const id2 = boundedString2(
|
|
47298
|
-
|
|
47298
|
+
record2.id ?? record2.datasetId ?? record2.dataset_id ?? record2.conversionActionId ?? record2.conversion_action_id ?? record2.pixelId ?? record2.pixel_id ?? record2.pixelCode ?? record2.pixel_code,
|
|
47299
47299
|
500
|
|
47300
47300
|
);
|
|
47301
47301
|
if (!id2) return [];
|
|
47302
|
-
const name = boundedString2(
|
|
47302
|
+
const name = boundedString2(record2.name ?? record2.label ?? record2.displayName ?? record2.display_name, 240) ?? id2;
|
|
47303
47303
|
const safeAccountName = boundedString2(accountName, 240);
|
|
47304
47304
|
return [{ id: id2, name, ...safeAccountName ? { accountName: safeAccountName } : {} }];
|
|
47305
47305
|
}).slice(0, 500);
|
|
@@ -47308,10 +47308,10 @@ function normalizeMetaAccounts(value) {
|
|
|
47308
47308
|
const root = recordValue(value);
|
|
47309
47309
|
if (!Array.isArray(root.adAccounts)) return [];
|
|
47310
47310
|
return root.adAccounts.flatMap((item) => {
|
|
47311
|
-
const
|
|
47312
|
-
const id2 = boundedString2(
|
|
47311
|
+
const record2 = recordValue(item);
|
|
47312
|
+
const id2 = boundedString2(record2.id ?? record2.accountId ?? record2.account_id, 240);
|
|
47313
47313
|
if (!id2) return [];
|
|
47314
|
-
const name = boundedString2(
|
|
47314
|
+
const name = boundedString2(record2.name, 240);
|
|
47315
47315
|
return [{ id: id2, ...name ? { name } : {} }];
|
|
47316
47316
|
}).slice(0, 25);
|
|
47317
47317
|
}
|
|
@@ -47352,9 +47352,9 @@ function terminalResult(discovery) {
|
|
|
47352
47352
|
};
|
|
47353
47353
|
}
|
|
47354
47354
|
function analyticsActivationAssetFailure(error) {
|
|
47355
|
-
const
|
|
47356
|
-
const status = typeof
|
|
47357
|
-
const code = boundedString2(
|
|
47355
|
+
const record2 = recordValue(error);
|
|
47356
|
+
const status = typeof record2.status === "number" ? record2.status : void 0;
|
|
47357
|
+
const code = boundedString2(record2.code, 120);
|
|
47358
47358
|
const permissionLimited = status === 401 || status === 403 || code === "missing_provider_permission" || code === "missing_permission" || code === "permission_verification_unavailable";
|
|
47359
47359
|
return terminalResult(permissionLimited ? "permission_limited" : "provider_error");
|
|
47360
47360
|
}
|
|
@@ -47621,10 +47621,10 @@ async function createThorbitConnectUrl(userId) {
|
|
|
47621
47621
|
ON CONFLICT(state_hash) DO NOTHING`,
|
|
47622
47622
|
[stateHash, userId]
|
|
47623
47623
|
);
|
|
47624
|
-
const
|
|
47625
|
-
|
|
47626
|
-
|
|
47627
|
-
return
|
|
47624
|
+
const url2 = new URL(`${thorbitBaseUrl()}/api/v1/x-ray/entitlement`);
|
|
47625
|
+
url2.searchParams.set("redirect_uri", dashboardCallbackUrl());
|
|
47626
|
+
url2.searchParams.set("state", state);
|
|
47627
|
+
return url2.toString();
|
|
47628
47628
|
}
|
|
47629
47629
|
function verifyThorbitAssertion(token7) {
|
|
47630
47630
|
const [encoded, signature] = token7.split(".");
|
|
@@ -48072,21 +48072,21 @@ function normalizeDefinitionInput(input) {
|
|
|
48072
48072
|
isDefault: input.isDefault ?? false
|
|
48073
48073
|
};
|
|
48074
48074
|
}
|
|
48075
|
-
function recordMatchesStage(
|
|
48075
|
+
function recordMatchesStage(record2, stage) {
|
|
48076
48076
|
const rule = stage.rule;
|
|
48077
|
-
if (
|
|
48078
|
-
if (
|
|
48079
|
-
if (
|
|
48080
|
-
const provider =
|
|
48081
|
-
return !!provider && !!rule.providerAliases?.[provider]?.includes(
|
|
48077
|
+
if (record2.kind === "event" && (rule.eventNames?.includes(record2.eventName) || !!record2.eventDefinitionId && rule.eventDefinitionIds?.includes(record2.eventDefinitionId))) return true;
|
|
48078
|
+
if (record2.kind === "conversion" && rule.conversionKinds?.includes(record2.eventName)) return true;
|
|
48079
|
+
if (record2.kind === "external" && (rule.externalEventNames?.includes(record2.eventName) || !!record2.externalEventKind && rule.externalEventKinds?.includes(record2.externalEventKind))) return true;
|
|
48080
|
+
const provider = record2.provider?.toLowerCase();
|
|
48081
|
+
return !!provider && !!rule.providerAliases?.[provider]?.includes(record2.eventName);
|
|
48082
48082
|
}
|
|
48083
48083
|
function progressPeople(stages, records) {
|
|
48084
48084
|
const byPerson = /* @__PURE__ */ new Map();
|
|
48085
|
-
for (const
|
|
48086
|
-
if (!
|
|
48087
|
-
const existing = byPerson.get(
|
|
48088
|
-
existing.push(
|
|
48089
|
-
byPerson.set(
|
|
48085
|
+
for (const record2 of records) {
|
|
48086
|
+
if (!record2.personId || !Number.isFinite(Date.parse(record2.occurredAt))) continue;
|
|
48087
|
+
const existing = byPerson.get(record2.personId) ?? [];
|
|
48088
|
+
existing.push(record2);
|
|
48089
|
+
byPerson.set(record2.personId, existing);
|
|
48090
48090
|
}
|
|
48091
48091
|
const progress = [];
|
|
48092
48092
|
for (const [personId, personRecords] of byPerson) {
|
|
@@ -48095,7 +48095,7 @@ function progressPeople(stages, records) {
|
|
|
48095
48095
|
let priorTime = Number.NEGATIVE_INFINITY;
|
|
48096
48096
|
let priorIndex = -1;
|
|
48097
48097
|
for (const stage of stages) {
|
|
48098
|
-
const nextIndex = personRecords.findIndex((
|
|
48098
|
+
const nextIndex = personRecords.findIndex((record2, index) => index > priorIndex && Date.parse(record2.occurredAt) >= priorTime && recordMatchesStage(record2, stage));
|
|
48099
48099
|
const match = nextIndex >= 0 ? personRecords[nextIndex] : null;
|
|
48100
48100
|
matches.push(match);
|
|
48101
48101
|
if (!match) break;
|
|
@@ -48106,7 +48106,7 @@ function progressPeople(stages, records) {
|
|
|
48106
48106
|
if (matches[0]) {
|
|
48107
48107
|
progress.push({
|
|
48108
48108
|
personId,
|
|
48109
|
-
publicPersonRef: personRecords.find((
|
|
48109
|
+
publicPersonRef: personRecords.find((record2) => record2.publicPersonRef)?.publicPersonRef ?? null,
|
|
48110
48110
|
matches
|
|
48111
48111
|
});
|
|
48112
48112
|
}
|
|
@@ -48223,7 +48223,7 @@ function paginateAnalyticsFunnelProgress(input) {
|
|
|
48223
48223
|
source: entry.source || "(direct)",
|
|
48224
48224
|
campaign: entry.campaign || "(none)",
|
|
48225
48225
|
adId: entry.adId || "(none)",
|
|
48226
|
-
revenueMinor: person.matches.reduce((sum,
|
|
48226
|
+
revenueMinor: person.matches.reduce((sum, record2) => sum + (record2?.valueMinor ?? 0), 0)
|
|
48227
48227
|
};
|
|
48228
48228
|
});
|
|
48229
48229
|
return {
|
|
@@ -48601,6 +48601,8 @@ var ANALYTICS_REST_ROUTES = [
|
|
|
48601
48601
|
["POST", "/v1/linker/consume"],
|
|
48602
48602
|
["POST", "/v1/linker/issue"],
|
|
48603
48603
|
["GET", "/v1/pixel.js"],
|
|
48604
|
+
["GET", "/v1/pixels/:publicPixelId/consent-policy"],
|
|
48605
|
+
["POST", "/v1/pixels/:publicPixelId/consent-receipts"],
|
|
48604
48606
|
["GET", "/v1/pixels/:publicPixelId/event-definitions"],
|
|
48605
48607
|
["POST", "/v1/webhooks/:provider/:connectionId"]
|
|
48606
48608
|
];
|
|
@@ -48859,6 +48861,31 @@ var AnalyticsConsentWithdrawalResponseSchema = okWith({
|
|
|
48859
48861
|
downstream: z40.object({ candidateAssociationsExpired: z40.literal(true), activationEligible: z40.literal(false) }).strict(),
|
|
48860
48862
|
replayed: z40.boolean().optional()
|
|
48861
48863
|
});
|
|
48864
|
+
var AnalyticsPublicConsentGatewayResponseSchema = okWith({
|
|
48865
|
+
config: z40.object({
|
|
48866
|
+
schemaVersion: z40.literal(1),
|
|
48867
|
+
policyVersion: z40.number().int().positive(),
|
|
48868
|
+
jurisdictionProfile: z40.string().min(1),
|
|
48869
|
+
presentation: z40.object({
|
|
48870
|
+
title: z40.string().min(1),
|
|
48871
|
+
explanation: z40.string().min(1),
|
|
48872
|
+
privacyUrl: z40.string().min(1),
|
|
48873
|
+
policyUrl: z40.string().min(1),
|
|
48874
|
+
accent: z40.string().regex(/^#[0-9a-f]{6}$/),
|
|
48875
|
+
radius: z40.number().int().min(0).max(24),
|
|
48876
|
+
fontFamily: z40.literal("inherit"),
|
|
48877
|
+
interactionVersion: z40.literal("two_stage_equal_effort_v1")
|
|
48878
|
+
}).strict(),
|
|
48879
|
+
purposes: z40.array(z40.object({
|
|
48880
|
+
key: z40.enum(["site_analytics", "enhanced_matching", "advertising_measurement"]),
|
|
48881
|
+
label: z40.string().min(1),
|
|
48882
|
+
description: z40.string().min(1)
|
|
48883
|
+
}).strict()).length(3)
|
|
48884
|
+
}).strict()
|
|
48885
|
+
});
|
|
48886
|
+
var AnalyticsPublicConsentReceiptResponseSchema = okWith({
|
|
48887
|
+
receipt: z40.object({ receiptId: z40.string().uuid(), status: z40.literal("recorded") }).strict()
|
|
48888
|
+
});
|
|
48862
48889
|
var AnalyticsLeadScoreSchema = z40.object({
|
|
48863
48890
|
id: z40.string().min(1),
|
|
48864
48891
|
namespaceId: z40.string().min(1),
|
|
@@ -49207,6 +49234,8 @@ var RESPONSE_SCHEMAS = {
|
|
|
49207
49234
|
"analytics.consent_policies.v1": okWith({ policies: z40.array(AnalyticsConsentPolicySchema) }),
|
|
49208
49235
|
"analytics.consent_policy.v1": okWith({ policy: AnalyticsConsentPolicySchema }),
|
|
49209
49236
|
"analytics.consent_withdrawal.v1": AnalyticsConsentWithdrawalResponseSchema,
|
|
49237
|
+
"analytics.public_consent_gateway.v1": AnalyticsPublicConsentGatewayResponseSchema,
|
|
49238
|
+
"analytics.public_consent_receipt.v1": AnalyticsPublicConsentReceiptResponseSchema,
|
|
49210
49239
|
"analytics.campaign_links.v1": namedDataPage("links"),
|
|
49211
49240
|
"analytics.link.v1": namedObject("link"),
|
|
49212
49241
|
"analytics.forms.v1": namedDataPage("forms"),
|
|
@@ -49253,6 +49282,8 @@ consume("analytics.pixel.v1", [["POST", "/sites/:siteId/pixels"]]);
|
|
|
49253
49282
|
consume("analytics.groups.v1", [["GET", "/sites/:siteId/pixels/:pixelId/host-groups"]]);
|
|
49254
49283
|
consume("analytics.group.v1", [["POST", "/sites/:siteId/pixels/:pixelId/host-groups"]]);
|
|
49255
49284
|
consume("analytics.events.accepted.v1", [["POST", "/v1/events"]]);
|
|
49285
|
+
consume("analytics.public_consent_gateway.v1", [["GET", "/v1/pixels/:publicPixelId/consent-policy"]]);
|
|
49286
|
+
consume("analytics.public_consent_receipt.v1", [["POST", "/v1/pixels/:publicPixelId/consent-receipts"]]);
|
|
49256
49287
|
consume("analytics.definitions.v1", [["GET", "/sites/:siteId/event-definitions"]]);
|
|
49257
49288
|
consume("analytics.definition.v1", [["POST", "/sites/:siteId/event-definitions"], ["PATCH", "/sites/:siteId/event-definitions/:definitionId"], ["DELETE", "/sites/:siteId/event-definitions/:definitionId"]]);
|
|
49258
49289
|
consume("analytics.definition_test.v1", [["POST", "/sites/:siteId/event-definitions/:definitionId/test"]]);
|
|
@@ -49855,6 +49886,111 @@ async function getAnalyticsConsentCoverage(siteId, options = {}) {
|
|
|
49855
49886
|
};
|
|
49856
49887
|
}
|
|
49857
49888
|
|
|
49889
|
+
// src/api/analytics-consent-gateway.ts
|
|
49890
|
+
var ANALYTICS_GATEWAY_PURPOSES = [
|
|
49891
|
+
"site_analytics",
|
|
49892
|
+
"enhanced_matching",
|
|
49893
|
+
"advertising_measurement"
|
|
49894
|
+
];
|
|
49895
|
+
var DEFAULT_COPY = {
|
|
49896
|
+
site_analytics: {
|
|
49897
|
+
label: "Site analytics",
|
|
49898
|
+
description: "Measure visits, pages, engagement and sessions so this business can improve its website."
|
|
49899
|
+
},
|
|
49900
|
+
enhanced_matching: {
|
|
49901
|
+
label: "Enhanced matching",
|
|
49902
|
+
description: "Connect consented visits when a person later provides identifying information to this business."
|
|
49903
|
+
},
|
|
49904
|
+
advertising_measurement: {
|
|
49905
|
+
label: "Advertising measurement",
|
|
49906
|
+
description: "Measure which ads contributed to leads and send permitted conversion signals to connected ad accounts."
|
|
49907
|
+
}
|
|
49908
|
+
};
|
|
49909
|
+
function record(value) {
|
|
49910
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
49911
|
+
}
|
|
49912
|
+
function text(value, fallback, max) {
|
|
49913
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, max) : fallback;
|
|
49914
|
+
}
|
|
49915
|
+
function url(value, fallback) {
|
|
49916
|
+
const candidate = text(value, fallback, 2e3);
|
|
49917
|
+
if (candidate.startsWith("/") && !candidate.startsWith("//")) return candidate;
|
|
49918
|
+
try {
|
|
49919
|
+
const parsed = new URL(candidate);
|
|
49920
|
+
return parsed.protocol === "https:" ? parsed.toString() : fallback;
|
|
49921
|
+
} catch {
|
|
49922
|
+
return fallback;
|
|
49923
|
+
}
|
|
49924
|
+
}
|
|
49925
|
+
function accent(value) {
|
|
49926
|
+
const candidate = typeof value === "string" ? value.trim() : "";
|
|
49927
|
+
return /^#[0-9a-f]{6}$/i.test(candidate) ? candidate.toLowerCase() : "#087f8c";
|
|
49928
|
+
}
|
|
49929
|
+
function radius(value) {
|
|
49930
|
+
const candidate = typeof value === "number" && Number.isFinite(value) ? Math.round(value) : 12;
|
|
49931
|
+
return Math.max(0, Math.min(24, candidate));
|
|
49932
|
+
}
|
|
49933
|
+
function buildAnalyticsPublicConsentGatewayConfig(rows, pixelId) {
|
|
49934
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
49935
|
+
for (const row of rows) {
|
|
49936
|
+
if (!row.enabled || row.retired_at) continue;
|
|
49937
|
+
if (!ANALYTICS_GATEWAY_PURPOSES.includes(row.purpose)) continue;
|
|
49938
|
+
const policy7 = record(row.policy);
|
|
49939
|
+
const scopedPixel = policy7.pixelId;
|
|
49940
|
+
if (typeof scopedPixel === "string" && scopedPixel && scopedPixel !== pixelId) continue;
|
|
49941
|
+
const key2 = `${row.policy_version}:${row.jurisdiction_profile}`;
|
|
49942
|
+
const group2 = candidates.get(key2) ?? /* @__PURE__ */ new Map();
|
|
49943
|
+
group2.set(row.purpose, row);
|
|
49944
|
+
candidates.set(key2, group2);
|
|
49945
|
+
}
|
|
49946
|
+
const complete2 = [...candidates.entries()].filter(([, group2]) => ANALYTICS_GATEWAY_PURPOSES.every((purpose) => group2.has(purpose))).sort((a, b) => {
|
|
49947
|
+
const [aVersion] = a[0].split(":");
|
|
49948
|
+
const [bVersion] = b[0].split(":");
|
|
49949
|
+
return Number(bVersion) - Number(aVersion);
|
|
49950
|
+
})[0];
|
|
49951
|
+
if (!complete2) return null;
|
|
49952
|
+
const [, group] = complete2;
|
|
49953
|
+
const first = group.get("site_analytics");
|
|
49954
|
+
const firstPolicy = record(first.policy);
|
|
49955
|
+
const presentation = record(firstPolicy.presentation);
|
|
49956
|
+
const purposes = ANALYTICS_GATEWAY_PURPOSES.map((key2) => {
|
|
49957
|
+
const policy7 = record(group.get(key2).policy);
|
|
49958
|
+
return {
|
|
49959
|
+
key: key2,
|
|
49960
|
+
label: text(policy7.label, DEFAULT_COPY[key2].label, 100),
|
|
49961
|
+
description: text(policy7.description, DEFAULT_COPY[key2].description, 400)
|
|
49962
|
+
};
|
|
49963
|
+
});
|
|
49964
|
+
return {
|
|
49965
|
+
schemaVersion: 1,
|
|
49966
|
+
policyVersion: first.policy_version,
|
|
49967
|
+
jurisdictionProfile: text(first.jurisdiction_profile, "global", 80),
|
|
49968
|
+
presentation: {
|
|
49969
|
+
title: text(presentation.title, "Your privacy choices", 120),
|
|
49970
|
+
explanation: text(
|
|
49971
|
+
presentation.explanation,
|
|
49972
|
+
"We use first-party data to understand visits, improve this site and measure advertising. This business does not sell your personal data.",
|
|
49973
|
+
600
|
|
49974
|
+
),
|
|
49975
|
+
privacyUrl: url(presentation.privacyUrl, "/privacy"),
|
|
49976
|
+
policyUrl: url(presentation.policyUrl, "/privacy#analytics"),
|
|
49977
|
+
accent: accent(presentation.accent),
|
|
49978
|
+
radius: radius(presentation.radius),
|
|
49979
|
+
fontFamily: "inherit",
|
|
49980
|
+
interactionVersion: "two_stage_equal_effort_v1"
|
|
49981
|
+
},
|
|
49982
|
+
purposes
|
|
49983
|
+
};
|
|
49984
|
+
}
|
|
49985
|
+
function analyticsConsentGatewaySdk() {
|
|
49986
|
+
return `;(()=>{const s=document.currentScript||document.querySelector('script[src*="/analytics/v1/pixel.js"][data-pixel]'),api=window.mcpAnalytics;if(!s||!api||s.dataset.consent!=='required')return;const pixel=String(s.dataset.pixel||'');if(!pixel)return;const suffix=pixel.replace(/[^a-z0-9]/gi,'_'),consentKey='mcp_xray_consent_'+suffix,subjectKey='mcp_xray_consent_subject_'+suffix,configUrl=new URL('/analytics/v1/pixels/'+encodeURIComponent(pixel)+'/consent-policy',s.src).toString(),receiptUrl=new URL('/analytics/v1/pixels/'+encodeURIComponent(pixel)+'/consent-receipts',s.src).toString(),read=k=>{try{return localStorage.getItem(k)}catch{return null}},write=(k,v)=>{try{localStorage.setItem(k,v)}catch{}},parse=v=>{try{return JSON.parse(v||'')}catch{return null}},id=()=>crypto.randomUUID?crypto.randomUUID():'choice_'+Date.now().toString(36)+Math.random().toString(36).slice(2),subject=()=>{let value=read(subjectKey);if(!value){value=id();write(subjectKey,value)}return value},gpc=()=>Boolean(navigator.globalPrivacyControl),choice=v=>v===true||v==='granted'?'granted':'denied';let config=null,host=null,root=null,panel=null,launcher=null,presentedAt=new Date().toISOString();const el=(tag,attrs={},value='')=>{const node=document.createElement(tag);for(const [key,val] of Object.entries(attrs)){if(key==='class')node.className=String(val);else if(key.startsWith('aria-'))node.setAttribute(key,String(val));else node[key]=val}if(value)node.textContent=value;return node},saved=()=>parse(read(consentKey)),currentChoices=()=>{const state=saved(),out={};for(const purpose of config.purposes)out[purpose.key]=state&&state.purposes&&state.purposes[purpose.key]==='granted';return out},postReceipt=(action,purposes,choiceAt)=>{let subjectRef=subject();try{const debug=api.debug();if(debug&&debug.visitorId)subjectRef=debug.visitorId}catch{}fetch(receiptUrl,{method:'POST',headers:{'content-type':'text/plain;charset=UTF-8'},credentials:'omit',keepalive:true,body:JSON.stringify({schemaVersion:1,subjectRef,policyVersion:config.policyVersion,jurisdictionProfile:config.jurisdictionProfile,purposes,globalPrivacyControl:gpc(),action,presentedAt,choiceAt})}).catch(()=>{})},closePanel=()=>{if(panel){panel.remove();panel=null}showLauncher()},showLauncher=()=>{if(launcher)return;launcher=el('button',{class:'xray-launcher',type:'button','aria-label':'Open privacy choices'},'Privacy choices');launcher.addEventListener('click',()=>{launcher.remove();launcher=null;render(true)});root.append(launcher)},apply=(action,values)=>{const purposes={};for(const purpose of config.purposes)purposes[purpose.key]=gpc()?'denied':choice(values[purpose.key]);const choiceAt=new Date().toISOString();api.consent({schemaVersion:1,policyVersion:config.policyVersion,jurisdictionProfile:config.jurisdictionProfile,presentedAt,choiceAt,globalPrivacyControl:gpc(),action,purposes});postReceipt(gpc()?'gpc':action,purposes,choiceAt);closePanel();return purposes},button=(label,handler)=>{const node=el('button',{class:'xray-action',type:'button'},label);node.addEventListener('click',handler);return node};function render(editing=false){if(panel)panel.remove();panel=el('section',{class:'xray-panel',role:'dialog','aria-modal':'false','aria-labelledby':'xray-consent-title'});const head=el('div',{class:'xray-head'}),title=el('h2',{id:'xray-consent-title'},config.presentation.title),close=el('button',{class:'xray-close',type:'button','aria-label':'Close privacy choices'},'Close');close.addEventListener('click',closePanel);head.append(title,close);panel.append(head,el('p',{class:'xray-explanation'},config.presentation.explanation));if(!editing){panel.append(button('Review privacy choices',()=>render(true)))}else if(gpc()){panel.append(el('p',{class:'xray-gpc',role:'status'},'Your browser Global Privacy Control signal keeps optional measurement off.'))}else{const values=currentChoices(),list=el('div',{class:'xray-choices'});for(const purpose of config.purposes){const label=el('label',{class:'xray-choice'}),input=el('input',{type:'checkbox',checked:Boolean(values[purpose.key])}),copy=el('span'),strong=el('strong',{},purpose.label),description=el('span',{class:'xray-description'},purpose.description);input.addEventListener('change',()=>{values[purpose.key]=input.checked});copy.append(strong,description);label.append(input,copy);list.append(label)}panel.append(list);const actions=el('div',{class:'xray-actions'});actions.append(button('Accept all',()=>apply('accept_all',Object.fromEntries(config.purposes.map(p=>[p.key,true])))),button('Reject non-essential',()=>apply('reject_all',Object.fromEntries(config.purposes.map(p=>[p.key,false])))),button('Save choices',()=>apply('save_choices',values)));panel.append(actions)}const links=el('div',{class:'xray-links'}),privacy=el('a',{href:config.presentation.privacyUrl},'Privacy'),policy=el('a',{href:config.presentation.policyUrl},'Analytics policy');links.append(privacy,policy);panel.append(links);root.append(panel);(panel.querySelector('button'))?.focus()}function mount(){host=el('div',{id:'mcp-xray-consent-gateway'});host.style.fontFamily='inherit';document.body.append(host);root=host.attachShadow({mode:'open'});const style=el('style');style.textContent=':host{font-family:inherit;color:#0a0a0a}.xray-panel,.xray-launcher{font-family:inherit;box-sizing:border-box}.xray-panel{position:fixed;z-index:2147483000;right:max(16px,env(safe-area-inset-right));bottom:max(16px,env(safe-area-inset-bottom));width:min(420px,calc(100vw - 32px));display:grid;gap:14px;padding:20px;background:#fff;color:#0a0a0a;border:1px solid color-mix(in srgb,'+config.presentation.accent+' 28%,#d5dde5);border-radius:'+config.presentation.radius+'px;box-shadow:0 18px 56px rgba(20,37,48,.18);line-height:1.45}.xray-head{display:flex;align-items:start;justify-content:space-between;gap:16px}.xray-head h2{margin:0;font:800 20px/1.2 inherit;letter-spacing:-.02em}.xray-explanation,.xray-gpc{margin:0;font-size:14px;color:#4b5563}.xray-gpc{padding:12px;background:#f4f7f8;border-radius:max(4px,calc('+config.presentation.radius+'px - 4px))}.xray-close{border:0;background:transparent;color:#4b5563;text-decoration:underline;cursor:pointer;font:600 13px/1 inherit}.xray-action,.xray-launcher{min-height:44px;border:1px solid '+config.presentation.accent+';border-radius:max(4px,calc('+config.presentation.radius+'px - 4px));background:'+config.presentation.accent+';color:#fff;padding:10px 14px;font:750 14px/1.2 inherit;cursor:pointer}.xray-choices{display:grid;gap:10px}.xray-choice{display:grid;grid-template-columns:20px 1fr;gap:10px;align-items:start;padding:10px;border:1px solid #d5dde5;border-radius:max(4px,calc('+config.presentation.radius+'px - 4px));font-size:13px}.xray-choice input{width:18px;height:18px;margin:1px 0;accent-color:'+config.presentation.accent+'}.xray-choice strong,.xray-description{display:block}.xray-description{margin-top:3px;color:#5f6b76;line-height:1.4}.xray-actions{display:grid;grid-template-columns:1fr;gap:8px}.xray-actions .xray-action{width:100%}.xray-links{display:flex;gap:16px;flex-wrap:wrap;font-size:12px}.xray-links a{color:#334155;text-decoration:underline}.xray-launcher{position:fixed;z-index:2147482999;right:max(16px,env(safe-area-inset-right));bottom:max(16px,env(safe-area-inset-bottom));box-shadow:0 8px 28px rgba(20,37,48,.16)}@media(max-width:520px){.xray-panel{left:max(12px,env(safe-area-inset-left));right:max(12px,env(safe-area-inset-right));bottom:max(12px,env(safe-area-inset-bottom));width:auto;max-height:calc(100dvh - 24px);overflow:auto;padding:16px}.xray-actions{grid-template-columns:1fr}.xray-launcher{right:max(12px,env(safe-area-inset-right));bottom:max(12px,env(safe-area-inset-bottom))}}@media(prefers-reduced-motion:no-preference){.xray-panel{animation:xray-in .18s ease-out}@keyframes xray-in{from{opacity:0;transform:translateY(8px)}to{opacity:1;transform:none}}}';root.append(style);window.xrayConsentGateway={open:()=>{if(launcher){launcher.remove();launcher=null}render(true)},acceptAll:()=>apply('accept_all',Object.fromEntries(config.purposes.map(p=>[p.key,true]))),rejectNonEssential:()=>apply('reject_all',Object.fromEntries(config.purposes.map(p=>[p.key,false]))),saveChoices:values=>apply('save_choices',values||{}),withdraw:()=>apply('withdraw',Object.fromEntries(config.purposes.map(p=>[p.key,false])))};if(gpc()){apply('gpc',{});showLauncher();return}const prior=saved();if(prior&&prior.policyVersion===config.policyVersion)showLauncher();else render(false)}fetch(configUrl,{credentials:'omit'}).then(response=>response.ok?response.json():null).then(body=>{if(!body||body.ok!==true||!body.config)return;config=body.config;mount()}).catch(()=>{})})();`;
|
|
49987
|
+
}
|
|
49988
|
+
|
|
49989
|
+
// src/api/analytics-visual-picker-ux.ts
|
|
49990
|
+
function analyticsVisualPickerUxSdk() {
|
|
49991
|
+
return `;(()=>{const toolbar=document.querySelector('#mcp-xray-picker-toolbar'),script=document.currentScript||document.querySelector('script[src*="/analytics/v1/pixel.js"][data-pixel]'),opener=window.opener;if(!toolbar||!script||!opener)return;const dashboardOrigin=new URL(script.src,location.href).origin,pixelId=String(script.dataset.pixel||''),status=toolbar.querySelector('span'),outline=document.querySelector('#mcp-xray-picker-outline'),buttons=[...toolbar.querySelectorAll('button')],pageButton=buttons.find(button=>button.textContent==='Track this page'),addButton=toolbar.querySelector('button[data-primary]'),cancelButton=buttons.find(button=>button.textContent==='Cancel');if(!status||!addButton||!cancelButton)return;toolbar.setAttribute('role','region');toolbar.setAttribute('aria-label','X-Ray visual event picker');status.setAttribute('role','status');status.setAttribute('aria-live','polite');if(pageButton)pageButton.dataset.pickerAction='page';addButton.dataset.pickerAction='add';cancelButton.dataset.pickerAction='cancel';const minimizeButton=document.createElement('button');minimizeButton.type='button';minimizeButton.dataset.pickerMinimize='true';minimizeButton.textContent='Minimize';minimizeButton.setAttribute('aria-expanded','true');minimizeButton.setAttribute('aria-label','Minimize X-Ray picker');toolbar.append(minimizeButton);const uxStyle=document.createElement('style');uxStyle.id='mcp-xray-picker-ux-style';uxStyle.textContent='#mcp-xray-picker-toolbar{box-sizing:border-box!important}#mcp-xray-picker-toolbar button[data-picker-minimize]{white-space:nowrap!important}#mcp-xray-picker-toolbar button[data-picker-action="add"][data-state="saved"]{border-color:#087f8c!important;background:#e8f7f5!important;color:#075c64!important}@media(max-width:600px){#mcp-xray-picker-toolbar:not(.is-minimized){display:grid!important;grid-template-columns:auto minmax(0,1fr) auto auto!important}#mcp-xray-picker-toolbar:not(.is-minimized)>strong{grid-column:1;grid-row:1}#mcp-xray-picker-toolbar:not(.is-minimized)>span{grid-column:2/4;grid-row:1}#mcp-xray-picker-toolbar:not(.is-minimized)>button[data-picker-minimize]{grid-column:4;grid-row:1}#mcp-xray-picker-toolbar:not(.is-minimized)>button[data-picker-action="page"]{grid-column:1;grid-row:2}#mcp-xray-picker-toolbar:not(.is-minimized)>button[data-picker-action="add"]{grid-column:2/4;grid-row:2}#mcp-xray-picker-toolbar:not(.is-minimized)>button[data-picker-action="cancel"]{grid-column:4;grid-row:2}}#mcp-xray-picker-toolbar.is-minimized{top:auto!important;right:12px!important;bottom:max(12px,env(safe-area-inset-bottom))!important;left:auto!important;transform:none!important;width:auto!important;display:flex!important;padding:8px 10px!important}#mcp-xray-picker-toolbar.is-minimized>:not(strong):not(button[data-picker-minimize]){display:none!important}@media(max-width:600px){#mcp-xray-picker-toolbar.is-minimized{bottom:calc(84px + env(safe-area-inset-bottom))!important}}';document.documentElement.append(uxStyle);let selectedElement=null,ackTimer=0;const interactive=target=>target instanceof Element?target.closest('a,button,form,input[type="button"],input[type="submit"],[role="button"],[data-xray-event]'):null,placeSelected=()=>{if(!selectedElement||!outline)return;const rect=selectedElement.getBoundingClientRect();outline.hidden=false;outline.style.left=rect.left+'px';outline.style.top=rect.top+'px';outline.style.width=rect.width+'px';outline.style.height=rect.height+'px'},setMinimized=minimized=>{toolbar.classList.toggle('is-minimized',minimized);minimizeButton.textContent=minimized?'Expand picker':'Minimize';minimizeButton.setAttribute('aria-expanded',String(!minimized));minimizeButton.setAttribute('aria-label',minimized?'Expand X-Ray picker':'Minimize X-Ray picker');if(!minimized)setTimeout(placeSelected,0)};minimizeButton.addEventListener('click',()=>setMinimized(!toolbar.classList.contains('is-minimized')));toolbar.addEventListener('pointerover',placeSelected,{passive:true});addEventListener('click',event=>{if(toolbar.contains(event.target))return;const element=interactive(event.target);if(!element)return;selectedElement=element;queueMicrotask(()=>{if(!status.textContent.startsWith('Selected'))status.textContent='Selected \u2713 \xB7 '+status.textContent;addButton.textContent='Add to X-Ray';addButton.dataset.state='selected'});},{capture:true});if(pageButton)pageButton.addEventListener('click',()=>{selectedElement=null;queueMicrotask(()=>{status.textContent='Selected \u2713 \xB7 Current page \xB7 '+location.pathname;addButton.dataset.state='selected'})});addButton.addEventListener('click',()=>{if(addButton.disabled)return;addButton.disabled=true;addButton.textContent='Adding\u2026';addButton.dataset.state='pending';status.textContent='Sending the editable draft to X-Ray\u2026';clearTimeout(ackTimer);ackTimer=setTimeout(()=>{if(addButton.dataset.state!=='pending')return;addButton.disabled=false;addButton.textContent='Try Add again';addButton.dataset.state='retry';status.textContent='No confirmation received. Keep this page open and try again.'},5000)});addEventListener('message',event=>{if(event.source!==opener||event.origin!==dashboardOrigin)return;const message=event.data;if(!message||message.type!=='mcp-xray-picker-selection-received'||message.pixelId!==pixelId)return;clearTimeout(ackTimer);addButton.disabled=true;addButton.textContent='Added \u2713';addButton.dataset.state='saved';status.textContent='Draft added to X-Ray\u2014not saved yet. Return to the dashboard to review and save it.';},{capture:true});cancelButton.addEventListener('click',()=>{uxStyle.remove()},{once:true})})();`;
|
|
49992
|
+
}
|
|
49993
|
+
|
|
49858
49994
|
// src/api/analytics-privacy-lifecycle.ts
|
|
49859
49995
|
import { randomUUID as randomUUID32 } from "crypto";
|
|
49860
49996
|
var ANALYTICS_PRIVACY_FIELD_REGISTRY = Object.freeze([
|
|
@@ -50652,18 +50788,18 @@ async function discoverAnalyticsCrmObjects(input) {
|
|
|
50652
50788
|
action: ANALYTICS_CRM_NANGO_ACTIONS.discoverSchema,
|
|
50653
50789
|
payload: { provider: capability.provider, include: ["objects", "fields", "pipelines", "stages"] }
|
|
50654
50790
|
});
|
|
50655
|
-
const
|
|
50791
|
+
const record2 = isRecord5(raw) ? raw : {};
|
|
50656
50792
|
const tenantCapability = getAnalyticsCrmSyncCapability(input.provider, {
|
|
50657
|
-
compiledActions: Array.isArray(
|
|
50658
|
-
advertisedTools: Array.isArray(
|
|
50659
|
-
grantedScopes: Array.isArray(
|
|
50660
|
-
tenantObjects: Array.isArray(
|
|
50661
|
-
tenantSchemaFingerprint: safeString3(
|
|
50662
|
-
normalizedAdapter:
|
|
50663
|
-
webhookAdvertised:
|
|
50793
|
+
compiledActions: Array.isArray(record2.compiledActions) ? record2.compiledActions.filter((value) => typeof value === "string") : [],
|
|
50794
|
+
advertisedTools: Array.isArray(record2.advertisedTools) ? record2.advertisedTools.filter((value) => typeof value === "string") : [],
|
|
50795
|
+
grantedScopes: Array.isArray(record2.grantedScopes) ? record2.grantedScopes.filter((value) => typeof value === "string") : [],
|
|
50796
|
+
tenantObjects: Array.isArray(record2.objects) ? record2.objects.flatMap((value) => typeof value === "string" ? [value] : isRecord5(value) && typeof value.name === "string" ? [value.name] : []) : [],
|
|
50797
|
+
tenantSchemaFingerprint: safeString3(record2.tenantSchemaFingerprint, 64) ?? null,
|
|
50798
|
+
normalizedAdapter: record2.normalizedOutcomeContractVersion === 1,
|
|
50799
|
+
webhookAdvertised: record2.webhookAdvertised === true
|
|
50664
50800
|
});
|
|
50665
50801
|
const allowedObjects = new Set(getAnalyticsCrmProviderProfile(input.provider).supportedObjects);
|
|
50666
|
-
const objects = Array.isArray(
|
|
50802
|
+
const objects = Array.isArray(record2.objects) ? record2.objects.flatMap((item) => {
|
|
50667
50803
|
if (typeof item === "string" && allowedObjects.has(item)) return [{ name: item.slice(0, 100), fields: [] }];
|
|
50668
50804
|
if (!isRecord5(item)) return [];
|
|
50669
50805
|
const name = safeString3(item.name, 100);
|
|
@@ -50671,7 +50807,7 @@ async function discoverAnalyticsCrmObjects(input) {
|
|
|
50671
50807
|
const fields = Array.isArray(item.fields) ? item.fields.map((field) => typeof field === "string" ? field.slice(0, 160) : isRecord5(field) ? safeString3(field.internalName ?? field.name, 160) : void 0).filter((value) => Boolean(value)) : [];
|
|
50672
50808
|
return [{ name, fields }];
|
|
50673
50809
|
}) : [];
|
|
50674
|
-
return { capability: tenantCapability, source: "connected_tenant", objects, pipelines: normalizePipelines2(
|
|
50810
|
+
return { capability: tenantCapability, source: "connected_tenant", objects, pipelines: normalizePipelines2(record2.pipelines) };
|
|
50675
50811
|
}
|
|
50676
50812
|
function testAnalyticsCrmMapping(input) {
|
|
50677
50813
|
const capability = getAnalyticsCrmSyncCapability(input.provider);
|
|
@@ -51319,8 +51455,8 @@ var AdSpendSchema = z43.object({
|
|
|
51319
51455
|
function safeOrigin(origin) {
|
|
51320
51456
|
if (!origin || origin === "null") return null;
|
|
51321
51457
|
try {
|
|
51322
|
-
const
|
|
51323
|
-
return
|
|
51458
|
+
const url2 = new URL(origin);
|
|
51459
|
+
return url2.protocol === "http:" || url2.protocol === "https:" ? url2.origin : null;
|
|
51324
51460
|
} catch {
|
|
51325
51461
|
return null;
|
|
51326
51462
|
}
|
|
@@ -51338,7 +51474,8 @@ analyticsApp.use("*", async (c, next) => {
|
|
|
51338
51474
|
const origin = safeOrigin(c.req.header("origin"));
|
|
51339
51475
|
const publicIngestion = c.req.path === "/v1/events";
|
|
51340
51476
|
const publicDefinitionRead = /^\/v1\/pixels\/[^/]+\/event-definitions$/.test(c.req.path);
|
|
51341
|
-
|
|
51477
|
+
const publicConsentGateway = /^\/v1\/pixels\/[^/]+\/consent-(?:policy|receipts)$/.test(c.req.path);
|
|
51478
|
+
if (origin && (publicIngestion || publicDefinitionRead || publicConsentGateway || dashboardOriginAllowed(origin))) {
|
|
51342
51479
|
c.header("Access-Control-Allow-Origin", origin);
|
|
51343
51480
|
c.header("Vary", "Origin");
|
|
51344
51481
|
c.header(
|
|
@@ -51397,22 +51534,22 @@ function analyticsVisualPickerSdk() {
|
|
|
51397
51534
|
function pagePathFilter(value) {
|
|
51398
51535
|
return value && value.startsWith("/") ? value : void 0;
|
|
51399
51536
|
}
|
|
51400
|
-
function reportFilters(
|
|
51401
|
-
const end =
|
|
51537
|
+
function reportFilters(url2) {
|
|
51538
|
+
const end = url2.searchParams.get("end") || (/* @__PURE__ */ new Date()).toISOString();
|
|
51402
51539
|
const defaultStart = new Date(
|
|
51403
51540
|
Date.parse(end) - 30 * 24 * 60 * 60 * 1e3
|
|
51404
51541
|
).toISOString();
|
|
51405
|
-
const list = (name) =>
|
|
51406
|
-
const parseModelList = (name) =>
|
|
51542
|
+
const list = (name) => url2.searchParams.getAll(name).flatMap((value) => value.split(",")).map((value) => value.trim()).filter(Boolean);
|
|
51543
|
+
const parseModelList = (name) => url2.searchParams.getAll(name).flatMap((value) => value.split(",")).map((value) => value.trim()).filter(Boolean);
|
|
51407
51544
|
let attribution;
|
|
51408
51545
|
try {
|
|
51409
51546
|
attribution = parseAnalyticsAttributionQuery({
|
|
51410
|
-
attributionModel:
|
|
51411
|
-
clickWindow:
|
|
51412
|
-
viewWindow:
|
|
51413
|
-
journeyTier:
|
|
51547
|
+
attributionModel: url2.searchParams.get("attributionModel"),
|
|
51548
|
+
clickWindow: url2.searchParams.get("clickWindow"),
|
|
51549
|
+
viewWindow: url2.searchParams.get("viewWindow"),
|
|
51550
|
+
journeyTier: url2.searchParams.get("journeyTier"),
|
|
51414
51551
|
compareModels: parseModelList("compareModels"),
|
|
51415
|
-
customWeights:
|
|
51552
|
+
customWeights: url2.searchParams.get("customWeights")
|
|
51416
51553
|
});
|
|
51417
51554
|
} catch (error) {
|
|
51418
51555
|
if (error instanceof z43.ZodError) {
|
|
@@ -51425,29 +51562,29 @@ function reportFilters(url) {
|
|
|
51425
51562
|
throw error;
|
|
51426
51563
|
}
|
|
51427
51564
|
return {
|
|
51428
|
-
start:
|
|
51565
|
+
start: url2.searchParams.get("start") || defaultStart,
|
|
51429
51566
|
end,
|
|
51430
|
-
timezone: validTimezone(
|
|
51567
|
+
timezone: validTimezone(url2.searchParams.get("timezone")),
|
|
51431
51568
|
pixelIds: list("pixelId"),
|
|
51432
51569
|
hostnames: list("hostname"),
|
|
51433
|
-
source:
|
|
51434
|
-
medium:
|
|
51435
|
-
campaign:
|
|
51436
|
-
eventName:
|
|
51570
|
+
source: url2.searchParams.get("source") || void 0,
|
|
51571
|
+
medium: url2.searchParams.get("medium") || void 0,
|
|
51572
|
+
campaign: url2.searchParams.get("campaign") || void 0,
|
|
51573
|
+
eventName: url2.searchParams.get("eventName") || void 0,
|
|
51437
51574
|
attributionModel: attribution.model,
|
|
51438
51575
|
clickWindow: attribution.clickWindow,
|
|
51439
51576
|
viewWindow: attribution.viewWindow,
|
|
51440
51577
|
journeyTier: attribution.journeyTier,
|
|
51441
51578
|
compareModels: attribution.compareModels,
|
|
51442
51579
|
customWeights: attribution.customWeights,
|
|
51443
|
-
channelFamily:
|
|
51444
|
-
platform:
|
|
51445
|
-
referrer:
|
|
51446
|
-
path: pagePathFilter(
|
|
51447
|
-
deviceClass:
|
|
51448
|
-
countryCode:
|
|
51449
|
-
regionCode:
|
|
51450
|
-
conversionKind:
|
|
51580
|
+
channelFamily: url2.searchParams.get("channelFamily") || void 0,
|
|
51581
|
+
platform: url2.searchParams.get("platform") || void 0,
|
|
51582
|
+
referrer: url2.searchParams.get("referrer") || void 0,
|
|
51583
|
+
path: pagePathFilter(url2.searchParams.get("path")),
|
|
51584
|
+
deviceClass: url2.searchParams.get("deviceClass") || void 0,
|
|
51585
|
+
countryCode: url2.searchParams.get("countryCode") || void 0,
|
|
51586
|
+
regionCode: url2.searchParams.get("regionCode") || void 0,
|
|
51587
|
+
conversionKind: url2.searchParams.get("conversionKind") || void 0
|
|
51451
51588
|
};
|
|
51452
51589
|
}
|
|
51453
51590
|
function validTimezone(value) {
|
|
@@ -51478,11 +51615,11 @@ function validateReportWindow(filters2) {
|
|
|
51478
51615
|
);
|
|
51479
51616
|
}
|
|
51480
51617
|
}
|
|
51481
|
-
function reportPage(
|
|
51482
|
-
const rawLimit = Number(
|
|
51618
|
+
function reportPage(url2) {
|
|
51619
|
+
const rawLimit = Number(url2.searchParams.get("limit") || 25);
|
|
51483
51620
|
return {
|
|
51484
51621
|
limit: Number.isFinite(rawLimit) ? Math.min(100, Math.max(1, Math.trunc(rawLimit))) : 25,
|
|
51485
|
-
cursor:
|
|
51622
|
+
cursor: url2.searchParams.get("cursor") || void 0
|
|
51486
51623
|
};
|
|
51487
51624
|
}
|
|
51488
51625
|
function pixelSnippet(publicId) {
|
|
@@ -51547,7 +51684,98 @@ analyticsApp.get("/health/ready", async (c) => {
|
|
|
51547
51684
|
analyticsApp.get("/v1/pixel.js", (c) => {
|
|
51548
51685
|
c.header("Content-Type", "application/javascript; charset=utf-8");
|
|
51549
51686
|
c.header("Cache-Control", "public, max-age=300");
|
|
51550
|
-
return c.body(`${analyticsPixelSdk()}${analyticsIdentityBridgeSdk()}${analyticsLegacyConsentSdk()}${analyticsDefinitionLoaderSdk()}${analyticsLinkerSdk()}${analyticsVisualPickerSdk()}`);
|
|
51687
|
+
return c.body(`${analyticsPixelSdk()}${analyticsIdentityBridgeSdk()}${analyticsLegacyConsentSdk()}${analyticsDefinitionLoaderSdk()}${analyticsLinkerSdk()}${analyticsVisualPickerSdk()}${analyticsVisualPickerUxSdk()}${analyticsConsentGatewaySdk()}`);
|
|
51688
|
+
});
|
|
51689
|
+
async function publicConsentPixel(c) {
|
|
51690
|
+
const origin = safeOrigin(c.req.header("origin"));
|
|
51691
|
+
if (!origin) return null;
|
|
51692
|
+
const hostname = normalizeAnalyticsHostname(new URL(origin).hostname);
|
|
51693
|
+
const pixelResult = await getAnalyticsPool().query(
|
|
51694
|
+
`SELECT p.id, p.site_id, s.owner_user_id
|
|
51695
|
+
FROM analytics_pixels p
|
|
51696
|
+
JOIN analytics_sites s ON s.id=p.site_id
|
|
51697
|
+
JOIN analytics_pixel_domains d ON d.pixel_id=p.id
|
|
51698
|
+
WHERE p.public_id=$1 AND p.status='active'
|
|
51699
|
+
AND d.hostname=$2 AND d.state='approved'
|
|
51700
|
+
LIMIT 1`,
|
|
51701
|
+
[c.req.param("publicPixelId"), hostname]
|
|
51702
|
+
);
|
|
51703
|
+
const pixel = pixelResult.rows[0];
|
|
51704
|
+
if (!pixel || !await analyticsOwnerHasCurrentEntitlement(Number(pixel.owner_user_id))) return null;
|
|
51705
|
+
const policies = await getAnalyticsPool().query(
|
|
51706
|
+
`SELECT purpose, policy_version, jurisdiction_profile, enabled, retired_at, policy
|
|
51707
|
+
FROM analytics_purpose_policies
|
|
51708
|
+
WHERE site_id=$1
|
|
51709
|
+
AND purpose IN ('site_analytics','enhanced_matching','advertising_measurement')
|
|
51710
|
+
ORDER BY policy_version DESC, created_at DESC`,
|
|
51711
|
+
[pixel.site_id]
|
|
51712
|
+
);
|
|
51713
|
+
return {
|
|
51714
|
+
origin,
|
|
51715
|
+
pixel,
|
|
51716
|
+
config: buildAnalyticsPublicConsentGatewayConfig(policies.rows, pixel.id)
|
|
51717
|
+
};
|
|
51718
|
+
}
|
|
51719
|
+
function publicConsentCors(c, origin) {
|
|
51720
|
+
c.header("Access-Control-Allow-Origin", origin);
|
|
51721
|
+
c.header("Vary", "Origin");
|
|
51722
|
+
c.header("Cache-Control", "no-store");
|
|
51723
|
+
}
|
|
51724
|
+
analyticsApp.get("/v1/pixels/:publicPixelId/consent-policy", async (c) => {
|
|
51725
|
+
const context = await publicConsentPixel(c);
|
|
51726
|
+
if (!context) return c.json({ ok: false, error: "Approved Pixel hostname required." }, 403, noStore);
|
|
51727
|
+
publicConsentCors(c, context.origin);
|
|
51728
|
+
if (!context.config)
|
|
51729
|
+
return c.json({ ok: false, error: "A complete published consent policy was not found." }, 404, noStore);
|
|
51730
|
+
return c.json({ ok: true, config: context.config }, 200);
|
|
51731
|
+
});
|
|
51732
|
+
var PublicConsentReceiptSchema = z43.object({
|
|
51733
|
+
schemaVersion: z43.literal(1),
|
|
51734
|
+
subjectRef: z43.string().trim().min(8).max(240),
|
|
51735
|
+
policyVersion: z43.number().int().positive(),
|
|
51736
|
+
jurisdictionProfile: z43.string().trim().min(1).max(80),
|
|
51737
|
+
purposes: z43.object({
|
|
51738
|
+
site_analytics: z43.enum(["granted", "denied"]),
|
|
51739
|
+
enhanced_matching: z43.enum(["granted", "denied"]),
|
|
51740
|
+
advertising_measurement: z43.enum(["granted", "denied"])
|
|
51741
|
+
}).strict(),
|
|
51742
|
+
globalPrivacyControl: z43.boolean(),
|
|
51743
|
+
action: z43.enum(["accept_all", "reject_all", "save_choices", "withdraw", "gpc"]),
|
|
51744
|
+
presentedAt: z43.string().datetime(),
|
|
51745
|
+
choiceAt: z43.string().datetime()
|
|
51746
|
+
}).strict();
|
|
51747
|
+
analyticsApp.post("/v1/pixels/:publicPixelId/consent-receipts", async (c) => {
|
|
51748
|
+
const context = await publicConsentPixel(c);
|
|
51749
|
+
if (!context) return c.json({ ok: false, error: "Approved Pixel hostname required." }, 403, noStore);
|
|
51750
|
+
publicConsentCors(c, context.origin);
|
|
51751
|
+
if (!context.config)
|
|
51752
|
+
return c.json({ ok: false, error: "A complete published consent policy was not found." }, 404, noStore);
|
|
51753
|
+
const raw = await c.req.text();
|
|
51754
|
+
let decoded = null;
|
|
51755
|
+
if (raw.length <= 8e3) {
|
|
51756
|
+
try {
|
|
51757
|
+
decoded = JSON.parse(raw || "null");
|
|
51758
|
+
} catch {
|
|
51759
|
+
decoded = null;
|
|
51760
|
+
}
|
|
51761
|
+
}
|
|
51762
|
+
const parsed = PublicConsentReceiptSchema.safeParse(decoded);
|
|
51763
|
+
if (!parsed.success)
|
|
51764
|
+
return c.json({ ok: false, error: "Invalid consent receipt.", issues: parsed.error.issues }, 400, noStore);
|
|
51765
|
+
if (parsed.data.policyVersion !== context.config.policyVersion || parsed.data.jurisdictionProfile !== context.config.jurisdictionProfile)
|
|
51766
|
+
return c.json({ ok: false, error: "Consent policy version is no longer current." }, 409, noStore);
|
|
51767
|
+
const choiceAt = Date.parse(parsed.data.choiceAt);
|
|
51768
|
+
const presentedAt = Date.parse(parsed.data.presentedAt);
|
|
51769
|
+
const now = Date.now();
|
|
51770
|
+
if (!Number.isFinite(choiceAt) || !Number.isFinite(presentedAt) || choiceAt < presentedAt || choiceAt > now + 5 * 6e4 || presentedAt < now - 24 * 60 * 6e4)
|
|
51771
|
+
return c.json({ ok: false, error: "Consent receipt timestamps are outside the accepted window." }, 400, noStore);
|
|
51772
|
+
const receipt = await recordAnalyticsConsentReceipt({
|
|
51773
|
+
...parsed.data,
|
|
51774
|
+
siteId: context.pixel.site_id,
|
|
51775
|
+
pixelId: context.pixel.id,
|
|
51776
|
+
source: parsed.data.globalPrivacyControl ? "gpc" : "gateway"
|
|
51777
|
+
}, { actor: "website_visitor" });
|
|
51778
|
+
return c.json({ ok: true, receipt: { receiptId: receipt.id, status: "recorded" } }, 201);
|
|
51551
51779
|
});
|
|
51552
51780
|
analyticsApp.get("/v1/pixels/:publicPixelId/event-definitions", async (c) => {
|
|
51553
51781
|
const publicPixelId = c.req.param("publicPixelId");
|
|
@@ -51602,7 +51830,7 @@ analyticsApp.post("/v1/linker/issue", async (c) => {
|
|
|
51602
51830
|
const sourceHostname = normalizeAnalyticsHostname(new URL(origin).hostname);
|
|
51603
51831
|
const audienceHostname = normalizeAnalyticsHostname(parsed.data.destinationHostname);
|
|
51604
51832
|
const now = Date.now();
|
|
51605
|
-
const
|
|
51833
|
+
const record2 = await prepareAnalyticsLinkerIssue({
|
|
51606
51834
|
publicPixelId: parsed.data.pixelId,
|
|
51607
51835
|
sourceHostname,
|
|
51608
51836
|
audienceHostname,
|
|
@@ -51610,16 +51838,16 @@ analyticsApp.post("/v1/linker/issue", async (c) => {
|
|
|
51610
51838
|
expiresAt: new Date(now + ANALYTICS_LINKER_TTL_MS)
|
|
51611
51839
|
});
|
|
51612
51840
|
const token7 = issueAnalyticsLinkerToken({
|
|
51613
|
-
tokenId:
|
|
51614
|
-
siteId:
|
|
51615
|
-
pixelId:
|
|
51616
|
-
identityNodeId:
|
|
51617
|
-
sourceHostname:
|
|
51618
|
-
audienceHostname:
|
|
51841
|
+
tokenId: record2.tokenId,
|
|
51842
|
+
siteId: record2.siteId,
|
|
51843
|
+
pixelId: record2.pixelId,
|
|
51844
|
+
identityNodeId: record2.identityNodeId,
|
|
51845
|
+
sourceHostname: record2.sourceHostname,
|
|
51846
|
+
audienceHostname: record2.audienceHostname,
|
|
51619
51847
|
now
|
|
51620
51848
|
});
|
|
51621
51849
|
linkerCors(c, origin);
|
|
51622
|
-
return c.json({ ok: true, token: token7, expiresAt:
|
|
51850
|
+
return c.json({ ok: true, token: token7, expiresAt: record2.expiresAt.toISOString() });
|
|
51623
51851
|
});
|
|
51624
51852
|
analyticsApp.post("/v1/linker/consume", async (c) => {
|
|
51625
51853
|
const origin = safeOrigin(c.req.header("origin"));
|
|
@@ -52000,12 +52228,12 @@ analyticsApp.post("/v1/events", async (c) => {
|
|
|
52000
52228
|
return c.json(result2, 202);
|
|
52001
52229
|
});
|
|
52002
52230
|
function analyticsExternalRequestUrl(c) {
|
|
52003
|
-
const
|
|
52231
|
+
const url2 = new URL(c.req.url);
|
|
52004
52232
|
const forwardedProto = c.req.header("x-forwarded-proto")?.split(",")[0]?.trim();
|
|
52005
52233
|
const forwardedHost = c.req.header("x-forwarded-host")?.split(",")[0]?.trim();
|
|
52006
|
-
if (forwardedProto)
|
|
52007
|
-
if (forwardedHost)
|
|
52008
|
-
return
|
|
52234
|
+
if (forwardedProto) url2.protocol = `${forwardedProto}:`;
|
|
52235
|
+
if (forwardedHost) url2.host = forwardedHost;
|
|
52236
|
+
return url2.toString();
|
|
52009
52237
|
}
|
|
52010
52238
|
analyticsApp.post("/v1/webhooks/:provider/:connectionId", async (c) => {
|
|
52011
52239
|
const provider = c.req.param("provider").toLowerCase();
|
|
@@ -52645,8 +52873,8 @@ analyticsApp.post("/sites/:siteId/ad-spend", auth4, async (c) => {
|
|
|
52645
52873
|
return c.json({ ok: true, result: result2 }, 202);
|
|
52646
52874
|
});
|
|
52647
52875
|
analyticsApp.get("/sites/:siteId/business-metrics", auth4, async (c) => {
|
|
52648
|
-
const
|
|
52649
|
-
const filters2 = reportFilters(
|
|
52876
|
+
const url2 = new URL(c.req.url);
|
|
52877
|
+
const filters2 = reportFilters(url2);
|
|
52650
52878
|
validateReportWindow(filters2);
|
|
52651
52879
|
const data = await analyticsBusinessMetrics(
|
|
52652
52880
|
c.req.param("siteId"),
|
|
@@ -52721,11 +52949,11 @@ analyticsApp.post("/sites/:siteId/pixels/:pixelId/host-groups", auth4, async (c)
|
|
|
52721
52949
|
return c.json({ ok: true, group }, 201);
|
|
52722
52950
|
});
|
|
52723
52951
|
analyticsApp.get("/sites/:siteId/campaign-links", auth4, async (c) => {
|
|
52724
|
-
const
|
|
52952
|
+
const url2 = new URL(c.req.url);
|
|
52725
52953
|
const data = await listAnalyticsCampaignLinks(
|
|
52726
52954
|
c.req.param("siteId"),
|
|
52727
52955
|
Number(c.get("user").id),
|
|
52728
|
-
reportPage(
|
|
52956
|
+
reportPage(url2)
|
|
52729
52957
|
);
|
|
52730
52958
|
return c.json({
|
|
52731
52959
|
ok: true,
|
|
@@ -52733,17 +52961,17 @@ analyticsApp.get("/sites/:siteId/campaign-links", auth4, async (c) => {
|
|
|
52733
52961
|
...data,
|
|
52734
52962
|
links: data.links.map((link) => ({
|
|
52735
52963
|
...link,
|
|
52736
|
-
short_url: `${
|
|
52964
|
+
short_url: `${url2.origin}/analytics/l/${link.short_code}`
|
|
52737
52965
|
}))
|
|
52738
52966
|
}
|
|
52739
52967
|
});
|
|
52740
52968
|
});
|
|
52741
52969
|
analyticsApp.get("/sites/:siteId/forms", auth4, async (c) => {
|
|
52742
|
-
const
|
|
52970
|
+
const url2 = new URL(c.req.url);
|
|
52743
52971
|
const data = await listAnalyticsForms(
|
|
52744
52972
|
c.req.param("siteId"),
|
|
52745
52973
|
Number(c.get("user").id),
|
|
52746
|
-
reportPage(
|
|
52974
|
+
reportPage(url2)
|
|
52747
52975
|
);
|
|
52748
52976
|
return c.json({
|
|
52749
52977
|
ok: true,
|
|
@@ -52752,7 +52980,7 @@ analyticsApp.get("/sites/:siteId/forms", auth4, async (c) => {
|
|
|
52752
52980
|
forms: data.forms.map((form) => ({
|
|
52753
52981
|
...form,
|
|
52754
52982
|
snippet: formSnippet(form.public_id),
|
|
52755
|
-
embed_url: `${
|
|
52983
|
+
embed_url: `${url2.origin}/analytics/forms/${form.public_id}/embed`
|
|
52756
52984
|
}))
|
|
52757
52985
|
}
|
|
52758
52986
|
});
|
|
@@ -53518,13 +53746,13 @@ analyticsApp.post("/sites/:siteId/funnel-definitions/test", auth4, async (c) =>
|
|
|
53518
53746
|
start: z43.string().datetime(),
|
|
53519
53747
|
end: z43.string().datetime()
|
|
53520
53748
|
}).strict());
|
|
53521
|
-
const records = body.records.map(({ subjectRef, ...
|
|
53749
|
+
const records = body.records.map(({ subjectRef, ...record2 }) => ({ ...record2, personId: subjectRef, publicPersonRef: subjectRef }));
|
|
53522
53750
|
return c.json({ ok: true, result: testAnalyticsFunnelDefinition({ ...body, records }) }, 200, noStore);
|
|
53523
53751
|
});
|
|
53524
53752
|
analyticsApp.get("/sites/:siteId/funnel", auth4, async (c) => {
|
|
53525
53753
|
const { siteId, userId } = currentAnalyticsUser(c);
|
|
53526
|
-
const
|
|
53527
|
-
const filters2 = reportFilters(
|
|
53754
|
+
const url2 = new URL(c.req.url);
|
|
53755
|
+
const filters2 = reportFilters(url2);
|
|
53528
53756
|
validateReportWindow(filters2);
|
|
53529
53757
|
const stageKey = c.req.query("stageKey");
|
|
53530
53758
|
const status = c.req.query("status");
|
|
@@ -54341,40 +54569,40 @@ for (const [path5, load] of [
|
|
|
54341
54569
|
["/sites/:siteId/events", analyticsEventCounts]
|
|
54342
54570
|
]) {
|
|
54343
54571
|
analyticsApp.get(path5, auth4, async (c) => {
|
|
54344
|
-
const
|
|
54345
|
-
const filters2 = reportFilters(
|
|
54572
|
+
const url2 = new URL(c.req.url);
|
|
54573
|
+
const filters2 = reportFilters(url2);
|
|
54346
54574
|
validateReportWindow(filters2);
|
|
54347
54575
|
const paginated = path5.endsWith("/acquisition") || path5.endsWith("/events");
|
|
54348
54576
|
const data = await load(
|
|
54349
54577
|
c.req.param("siteId"),
|
|
54350
54578
|
Number(c.get("user").id),
|
|
54351
54579
|
filters2,
|
|
54352
|
-
paginated ? reportPage(
|
|
54580
|
+
paginated ? reportPage(url2) : void 0
|
|
54353
54581
|
);
|
|
54354
54582
|
return c.json({ ok: true, data, filters: filters2 });
|
|
54355
54583
|
});
|
|
54356
54584
|
}
|
|
54357
54585
|
analyticsApp.get("/sites/:siteId/content", auth4, async (c) => {
|
|
54358
|
-
const
|
|
54359
|
-
const filters2 = reportFilters(
|
|
54586
|
+
const url2 = new URL(c.req.url);
|
|
54587
|
+
const filters2 = reportFilters(url2);
|
|
54360
54588
|
validateReportWindow(filters2);
|
|
54361
|
-
const requestedSort =
|
|
54589
|
+
const requestedSort = url2.searchParams.get("sort");
|
|
54362
54590
|
const data = await analyticsContent(
|
|
54363
54591
|
c.req.param("siteId"),
|
|
54364
54592
|
Number(c.get("user").id),
|
|
54365
54593
|
filters2,
|
|
54366
|
-
reportPage(
|
|
54594
|
+
reportPage(url2),
|
|
54367
54595
|
{
|
|
54368
54596
|
sort: ANALYTICS_CONTENT_SORTS.includes(requestedSort) ? requestedSort : void 0,
|
|
54369
|
-
direction:
|
|
54370
|
-
search:
|
|
54597
|
+
direction: url2.searchParams.get("direction") === "asc" ? "asc" : "desc",
|
|
54598
|
+
search: url2.searchParams.get("search") || void 0
|
|
54371
54599
|
}
|
|
54372
54600
|
);
|
|
54373
54601
|
return c.json({ ok: true, data, filters: filters2 });
|
|
54374
54602
|
});
|
|
54375
54603
|
analyticsApp.get("/sites/:siteId/dashboard", auth4, async (c) => {
|
|
54376
|
-
const
|
|
54377
|
-
const filters2 = reportFilters(
|
|
54604
|
+
const url2 = new URL(c.req.url);
|
|
54605
|
+
const filters2 = reportFilters(url2);
|
|
54378
54606
|
validateReportWindow(filters2);
|
|
54379
54607
|
const siteId = c.req.param("siteId");
|
|
54380
54608
|
const userId = Number(c.get("user").id);
|
|
@@ -54400,7 +54628,7 @@ analyticsApp.get("/sites/:siteId/dashboard", auth4, async (c) => {
|
|
|
54400
54628
|
analyticsConversions(siteId, userId, filters2, { limit: 8 })
|
|
54401
54629
|
]);
|
|
54402
54630
|
let comparison = null;
|
|
54403
|
-
if (
|
|
54631
|
+
if (url2.searchParams.get("compare") === "1") {
|
|
54404
54632
|
const start = Date.parse(filters2.start);
|
|
54405
54633
|
const end = Date.parse(filters2.end);
|
|
54406
54634
|
const duration = end - start;
|
|
@@ -54432,10 +54660,10 @@ analyticsApp.get("/sites/:siteId/dashboard", auth4, async (c) => {
|
|
|
54432
54660
|
});
|
|
54433
54661
|
});
|
|
54434
54662
|
analyticsApp.get("/sites/:siteId/dimensions", auth4, async (c) => {
|
|
54435
|
-
const
|
|
54436
|
-
const filters2 = reportFilters(
|
|
54663
|
+
const url2 = new URL(c.req.url);
|
|
54664
|
+
const filters2 = reportFilters(url2);
|
|
54437
54665
|
validateReportWindow(filters2);
|
|
54438
|
-
const dimension =
|
|
54666
|
+
const dimension = url2.searchParams.get("dimension") || "device";
|
|
54439
54667
|
if (!["device", "source", "country", "region", "weekday_hour"].includes(
|
|
54440
54668
|
dimension
|
|
54441
54669
|
)) {
|
|
@@ -54453,31 +54681,31 @@ analyticsApp.get("/sites/:siteId/dimensions", auth4, async (c) => {
|
|
|
54453
54681
|
Number(c.get("user").id),
|
|
54454
54682
|
filters2,
|
|
54455
54683
|
dimension,
|
|
54456
|
-
reportPage(
|
|
54684
|
+
reportPage(url2)
|
|
54457
54685
|
);
|
|
54458
54686
|
return c.json({ ok: true, data, filters: filters2 });
|
|
54459
54687
|
});
|
|
54460
54688
|
analyticsApp.get("/sites/:siteId/conversions", auth4, async (c) => {
|
|
54461
|
-
const
|
|
54462
|
-
const filters2 = reportFilters(
|
|
54689
|
+
const url2 = new URL(c.req.url);
|
|
54690
|
+
const filters2 = reportFilters(url2);
|
|
54463
54691
|
validateReportWindow(filters2);
|
|
54464
54692
|
const data = await analyticsConversions(
|
|
54465
54693
|
c.req.param("siteId"),
|
|
54466
54694
|
Number(c.get("user").id),
|
|
54467
54695
|
filters2,
|
|
54468
|
-
reportPage(
|
|
54696
|
+
reportPage(url2)
|
|
54469
54697
|
);
|
|
54470
54698
|
return c.json({ ok: true, data, filters: filters2 });
|
|
54471
54699
|
});
|
|
54472
54700
|
analyticsApp.get("/sites/:siteId/paths", auth4, async (c) => {
|
|
54473
|
-
const
|
|
54474
|
-
const filters2 = reportFilters(
|
|
54701
|
+
const url2 = new URL(c.req.url);
|
|
54702
|
+
const filters2 = reportFilters(url2);
|
|
54475
54703
|
validateReportWindow(filters2);
|
|
54476
54704
|
const data = await analyticsPaths(
|
|
54477
54705
|
c.req.param("siteId"),
|
|
54478
54706
|
Number(c.get("user").id),
|
|
54479
54707
|
filters2,
|
|
54480
|
-
reportPage(
|
|
54708
|
+
reportPage(url2)
|
|
54481
54709
|
);
|
|
54482
54710
|
return c.json({ ok: true, data, filters: filters2 });
|
|
54483
54711
|
});
|
|
@@ -54857,8 +55085,8 @@ async function defer(job, classified, random = Math.random) {
|
|
|
54857
55085
|
}
|
|
54858
55086
|
function statusFromError(error) {
|
|
54859
55087
|
if (error && typeof error === "object") {
|
|
54860
|
-
const
|
|
54861
|
-
for (const value of [
|
|
55088
|
+
const record2 = error;
|
|
55089
|
+
for (const value of [record2.status, record2.statusCode, record2.httpStatus]) {
|
|
54862
55090
|
if (typeof value === "number" && Number.isInteger(value)) return value;
|
|
54863
55091
|
}
|
|
54864
55092
|
}
|
|
@@ -54868,19 +55096,19 @@ function statusFromError(error) {
|
|
|
54868
55096
|
function classifyActivationFailure(platform, error) {
|
|
54869
55097
|
const definition = getAnalyticsProviderDefinition(platform);
|
|
54870
55098
|
const status = statusFromError(error);
|
|
54871
|
-
const
|
|
54872
|
-
const errorCode2 = typeof
|
|
54873
|
-
const
|
|
55099
|
+
const record2 = error && typeof error === "object" ? error : {};
|
|
55100
|
+
const errorCode2 = typeof record2.code === "string" ? record2.code.toLowerCase() : "";
|
|
55101
|
+
const text2 = error instanceof Error ? error.message.toLowerCase() : "delivery_failed";
|
|
54874
55102
|
if (status && definition.retryClassification.retryableStatusCodes.includes(status)) {
|
|
54875
55103
|
return { code: status === 429 ? "activation_rate_limited" : "activation_provider_unavailable", permanent: false, responseCode: status };
|
|
54876
55104
|
}
|
|
54877
55105
|
if (status && definition.retryClassification.permanentStatusCodes.includes(status)) {
|
|
54878
55106
|
return { code: status === 401 || status === 403 ? "activation_authorization_failed" : "activation_provider_rejected", permanent: true, responseCode: status };
|
|
54879
55107
|
}
|
|
54880
|
-
if (
|
|
55108
|
+
if (record2.retryable === true || /timeout|timed out|temporar|unavailable|outcome_unknown/.test(`${errorCode2} ${text2}`)) {
|
|
54881
55109
|
return { code: errorCode2 === "action_outcome_unknown" ? "activation_outcome_unknown" : "activation_provider_unavailable", permanent: false, responseCode: status };
|
|
54882
55110
|
}
|
|
54883
|
-
if (/invalid|required|schema|validation/.test(`${errorCode2} ${
|
|
55111
|
+
if (/invalid|required|schema|validation/.test(`${errorCode2} ${text2}`)) {
|
|
54884
55112
|
return { code: "activation_payload_invalid", permanent: true, responseCode: status };
|
|
54885
55113
|
}
|
|
54886
55114
|
return { code: "activation_delivery_failed", permanent: false, responseCode: status };
|
|
@@ -54978,8 +55206,8 @@ async function dispatchPendingAnalyticsActivations(limit = 10, transport = nango
|
|
|
54978
55206
|
return result2;
|
|
54979
55207
|
}
|
|
54980
55208
|
function classifyGoogleActivationDiagnostics(raw) {
|
|
54981
|
-
const
|
|
54982
|
-
const destinations = Array.isArray(
|
|
55209
|
+
const record2 = raw && typeof raw === "object" ? raw : {};
|
|
55210
|
+
const destinations = Array.isArray(record2.destinations) ? record2.destinations : [];
|
|
54983
55211
|
const statuses = destinations.map((value) => value && typeof value === "object" ? String(value.status ?? "") : "");
|
|
54984
55212
|
const providerErrors = destinations.flatMap((value) => value && typeof value === "object" && Array.isArray(value.errors) ? value.errors : []);
|
|
54985
55213
|
if (providerErrors.length > 0 || statuses.some((status) => /FAILED|REJECTED|INVALID/i.test(status))) {
|
|
@@ -55281,26 +55509,26 @@ function crmLane(provider) {
|
|
|
55281
55509
|
tool: ANALYTICS_CRM_NANGO_ACTIONS.listOutcomes,
|
|
55282
55510
|
buildInput: (sourceAccountRef, cursor, now) => crmReadInput(provider, sourceAccountRef, cursor, now),
|
|
55283
55511
|
parsePage: parseCrmReadPage,
|
|
55284
|
-
normalize: (
|
|
55285
|
-
|
|
55512
|
+
normalize: (record2, sourceAccountRef) => analyticsCrmOutcomeToExternalEvent(
|
|
55513
|
+
record2,
|
|
55286
55514
|
sourceAccountRef
|
|
55287
55515
|
)
|
|
55288
55516
|
};
|
|
55289
55517
|
}
|
|
55290
|
-
function callRailPhase(
|
|
55291
|
-
const phase = String(
|
|
55518
|
+
function callRailPhase(record2) {
|
|
55519
|
+
const phase = String(record2.phase ?? record2.event_type ?? "").toLowerCase();
|
|
55292
55520
|
if (phase === "pre_call" || phase === "routing_complete" || phase === "call_modified") return phase;
|
|
55293
55521
|
return "post_call";
|
|
55294
55522
|
}
|
|
55295
|
-
function twilioParams(
|
|
55523
|
+
function twilioParams(record2) {
|
|
55296
55524
|
return {
|
|
55297
|
-
...
|
|
55298
|
-
CallSid:
|
|
55299
|
-
CallStatus:
|
|
55300
|
-
Timestamp:
|
|
55301
|
-
CallDuration:
|
|
55302
|
-
Direction:
|
|
55303
|
-
To:
|
|
55525
|
+
...record2,
|
|
55526
|
+
CallSid: record2.CallSid ?? record2.callSid ?? record2.sid,
|
|
55527
|
+
CallStatus: record2.CallStatus ?? record2.callStatus ?? record2.status,
|
|
55528
|
+
Timestamp: record2.Timestamp ?? record2.timestamp ?? record2.dateUpdated ?? record2.startTime,
|
|
55529
|
+
CallDuration: record2.CallDuration ?? record2.callDuration ?? record2.duration,
|
|
55530
|
+
Direction: record2.Direction ?? record2.direction,
|
|
55531
|
+
To: record2.To ?? record2.to
|
|
55304
55532
|
};
|
|
55305
55533
|
}
|
|
55306
55534
|
var ANALYTICS_RECONCILIATION_READ_LANES = {
|
|
@@ -55314,10 +55542,10 @@ var ANALYTICS_RECONCILIATION_READ_LANES = {
|
|
|
55314
55542
|
limit: MAX_PAGE_SIZE
|
|
55315
55543
|
}),
|
|
55316
55544
|
parsePage: parseReadPage,
|
|
55317
|
-
normalize: (
|
|
55318
|
-
payload:
|
|
55545
|
+
normalize: (record2, sourceAccountRef) => normalizeCallRailWebhook({
|
|
55546
|
+
payload: record2,
|
|
55319
55547
|
sourceAccountRef,
|
|
55320
|
-
phase: callRailPhase(
|
|
55548
|
+
phase: callRailPhase(record2)
|
|
55321
55549
|
})
|
|
55322
55550
|
},
|
|
55323
55551
|
twilio: {
|
|
@@ -55330,8 +55558,8 @@ var ANALYTICS_RECONCILIATION_READ_LANES = {
|
|
|
55330
55558
|
limit: MAX_PAGE_SIZE
|
|
55331
55559
|
}),
|
|
55332
55560
|
parsePage: parseReadPage,
|
|
55333
|
-
normalize: (
|
|
55334
|
-
params: twilioParams(
|
|
55561
|
+
normalize: (record2, sourceAccountRef) => normalizeTwilioWebhook({
|
|
55562
|
+
params: twilioParams(record2),
|
|
55335
55563
|
sourceAccountRef
|
|
55336
55564
|
})
|
|
55337
55565
|
},
|
|
@@ -55590,12 +55818,12 @@ async function reconcileAnalyticsConnection(job, dependencies = defaultDependenc
|
|
|
55590
55818
|
let replayed = 0;
|
|
55591
55819
|
let duplicates = 0;
|
|
55592
55820
|
let lastSourceEventId = null;
|
|
55593
|
-
for (const
|
|
55821
|
+
for (const record2 of page2.records) {
|
|
55594
55822
|
if (Date.now() >= deadlineMs) throw new Error("analytics_reconciliation_deadline");
|
|
55595
|
-
let event2 = lane.normalize(
|
|
55823
|
+
let event2 = lane.normalize(record2, scope.source_account_ref);
|
|
55596
55824
|
let trustedCrmContext;
|
|
55597
55825
|
if (lane.kind === "crm") {
|
|
55598
|
-
const outcome =
|
|
55826
|
+
const outcome = record2;
|
|
55599
55827
|
if (outcome.contact.kind === "provider_contact_id") {
|
|
55600
55828
|
trustedCrmContext = await withinDeadline2(
|
|
55601
55829
|
dependencies.resolveMappedCrmPerson({
|
|
@@ -55936,7 +56164,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
55936
56164
|
await ensureScheduledRunViewLinksSchema();
|
|
55937
56165
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
55938
56166
|
const token7 = randomBytes5(32).toString("base64url");
|
|
55939
|
-
const
|
|
56167
|
+
const record2 = {
|
|
55940
56168
|
shareId: randomUUID38(),
|
|
55941
56169
|
ownerId: input.ownerId,
|
|
55942
56170
|
runId: input.runId,
|
|
@@ -55957,19 +56185,19 @@ async function createScheduledRunViewLink(input) {
|
|
|
55957
56185
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)
|
|
55958
56186
|
`,
|
|
55959
56187
|
args: [
|
|
55960
|
-
|
|
55961
|
-
|
|
55962
|
-
|
|
55963
|
-
|
|
55964
|
-
|
|
55965
|
-
|
|
55966
|
-
|
|
55967
|
-
|
|
55968
|
-
|
|
55969
|
-
|
|
56188
|
+
record2.shareId,
|
|
56189
|
+
record2.ownerId,
|
|
56190
|
+
record2.runId,
|
|
56191
|
+
record2.artifactId,
|
|
56192
|
+
record2.title,
|
|
56193
|
+
record2.summary,
|
|
56194
|
+
record2.rendererKey,
|
|
56195
|
+
record2.tokenHash,
|
|
56196
|
+
record2.expiresAt,
|
|
56197
|
+
record2.createdAt
|
|
55970
56198
|
]
|
|
55971
56199
|
});
|
|
55972
|
-
return { record, token: token7 };
|
|
56200
|
+
return { record: record2, token: token7 };
|
|
55973
56201
|
}
|
|
55974
56202
|
async function resolveScheduledRunViewLink(token7, now = /* @__PURE__ */ new Date()) {
|
|
55975
56203
|
await ensureScheduledRunViewLinksSchema();
|
|
@@ -56553,12 +56781,12 @@ scheduledResultApp.get("/artifact-templates", async (c) => {
|
|
|
56553
56781
|
});
|
|
56554
56782
|
const templates = (Array.isArray(result2.templates) ? result2.templates : []).filter((template) => {
|
|
56555
56783
|
if (!template || typeof template !== "object") return false;
|
|
56556
|
-
const
|
|
56557
|
-
const currentVersion =
|
|
56784
|
+
const record2 = template;
|
|
56785
|
+
const currentVersion = record2.currentVersion && typeof record2.currentVersion === "object" ? record2.currentVersion : null;
|
|
56558
56786
|
const rendererKey = typeof currentVersion?.rendererKey === "string" ? currentVersion.rendererKey : "";
|
|
56559
56787
|
if (presetKey && rendererKey !== presetKey) return false;
|
|
56560
56788
|
if (!normalizedQuery) return true;
|
|
56561
|
-
return [
|
|
56789
|
+
return [record2.name, record2.description, rendererKey, currentVersion?.authoringInstructions].filter((value) => typeof value === "string").some((value) => value.toLocaleLowerCase().includes(normalizedQuery));
|
|
56562
56790
|
});
|
|
56563
56791
|
return c.json({ ok: true, presets, templates });
|
|
56564
56792
|
});
|
|
@@ -56644,7 +56872,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
56644
56872
|
const artifact = result2.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
|
|
56645
56873
|
if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
|
|
56646
56874
|
const ownerId3 = scheduledArtifactOwnerIdForApiKey(user.api_key);
|
|
56647
|
-
const { record, token: token7 } = await createScheduledRunViewLink({
|
|
56875
|
+
const { record: record2, token: token7 } = await createScheduledRunViewLink({
|
|
56648
56876
|
ownerId: ownerId3,
|
|
56649
56877
|
runId: result2.data.run.runId,
|
|
56650
56878
|
artifactId: artifact.artifactId,
|
|
@@ -56656,12 +56884,12 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
56656
56884
|
const origin = process.env.APP_ORIGIN?.trim() || "https://mcpscraper.dev";
|
|
56657
56885
|
return c.json({
|
|
56658
56886
|
ok: true,
|
|
56659
|
-
shareId:
|
|
56660
|
-
runId:
|
|
56661
|
-
artifactId:
|
|
56887
|
+
shareId: record2.shareId,
|
|
56888
|
+
runId: record2.runId,
|
|
56889
|
+
artifactId: record2.artifactId,
|
|
56662
56890
|
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token7}`,
|
|
56663
|
-
expiresAt:
|
|
56664
|
-
createdAt:
|
|
56891
|
+
expiresAt: record2.expiresAt,
|
|
56892
|
+
createdAt: record2.createdAt
|
|
56665
56893
|
}, 201);
|
|
56666
56894
|
});
|
|
56667
56895
|
scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c) => {
|
|
@@ -57925,21 +58153,21 @@ function sensitiveKey(key2) {
|
|
|
57925
58153
|
function redactSignedUrl(value) {
|
|
57926
58154
|
if (!/^https?:\/\//i.test(value)) return value;
|
|
57927
58155
|
try {
|
|
57928
|
-
const
|
|
58156
|
+
const url2 = new URL(value);
|
|
57929
58157
|
let changed = false;
|
|
57930
|
-
if (
|
|
57931
|
-
|
|
57932
|
-
|
|
58158
|
+
if (url2.username || url2.password) {
|
|
58159
|
+
url2.username = "[REDACTED]";
|
|
58160
|
+
url2.password = "[REDACTED]";
|
|
57933
58161
|
changed = true;
|
|
57934
58162
|
}
|
|
57935
|
-
for (const key2 of [...
|
|
58163
|
+
for (const key2 of [...url2.searchParams.keys()]) {
|
|
57936
58164
|
const normalized = key2.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
57937
58165
|
if (normalized.includes("signature") || normalized.includes("credential") || normalized === "token" || normalized === "sig" || normalized === "auth" || normalized === "code" || normalized === "key" || normalized === "accesskey" || normalized === "apikey") {
|
|
57938
|
-
|
|
58166
|
+
url2.searchParams.set(key2, "[REDACTED]");
|
|
57939
58167
|
changed = true;
|
|
57940
58168
|
}
|
|
57941
58169
|
}
|
|
57942
|
-
return changed ?
|
|
58170
|
+
return changed ? url2.toString() : value;
|
|
57943
58171
|
} catch {
|
|
57944
58172
|
return value;
|
|
57945
58173
|
}
|
|
@@ -58561,9 +58789,9 @@ function buildUniverseData(notes) {
|
|
|
58561
58789
|
var _sql = null;
|
|
58562
58790
|
function sql() {
|
|
58563
58791
|
if (_sql) return _sql;
|
|
58564
|
-
const
|
|
58565
|
-
if (!
|
|
58566
|
-
_sql = neon(
|
|
58792
|
+
const url2 = process.env.MEMORY_DATABASE_URL?.trim();
|
|
58793
|
+
if (!url2) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
|
|
58794
|
+
_sql = neon(url2);
|
|
58567
58795
|
return _sql;
|
|
58568
58796
|
}
|
|
58569
58797
|
function physicalVault(owner, vault) {
|
|
@@ -58815,9 +59043,9 @@ function cleanString4(value, max = 300) {
|
|
|
58815
59043
|
const trimmed = value.trim();
|
|
58816
59044
|
return trimmed ? trimmed.slice(0, max) : null;
|
|
58817
59045
|
}
|
|
58818
|
-
function firstString2(
|
|
59046
|
+
function firstString2(record2, keys, max = 300) {
|
|
58819
59047
|
for (const key2 of keys) {
|
|
58820
|
-
const value = cleanString4(
|
|
59048
|
+
const value = cleanString4(record2[key2], max);
|
|
58821
59049
|
if (value) return value;
|
|
58822
59050
|
}
|
|
58823
59051
|
return null;
|
|
@@ -58834,8 +59062,8 @@ function cleanHttpsUrl2(value) {
|
|
|
58834
59062
|
const candidate = cleanString4(value, 2e3);
|
|
58835
59063
|
if (!candidate) return null;
|
|
58836
59064
|
try {
|
|
58837
|
-
const
|
|
58838
|
-
return
|
|
59065
|
+
const url2 = new URL(candidate);
|
|
59066
|
+
return url2.protocol === "https:" ? url2.toString() : null;
|
|
58839
59067
|
} catch {
|
|
58840
59068
|
return null;
|
|
58841
59069
|
}
|
|
@@ -58843,8 +59071,8 @@ function cleanHttpsUrl2(value) {
|
|
|
58843
59071
|
function cleanAuthorizationUrl(value) {
|
|
58844
59072
|
const candidate = cleanHttpsUrl2(value);
|
|
58845
59073
|
if (!candidate) return null;
|
|
58846
|
-
const
|
|
58847
|
-
return
|
|
59074
|
+
const url2 = new URL(candidate);
|
|
59075
|
+
return url2.hostname.toLowerCase() === "api.resend.com" && url2.pathname === "/oauth/authorize" ? candidate : null;
|
|
58848
59076
|
}
|
|
58849
59077
|
function arrayFromPayload2(value, keys) {
|
|
58850
59078
|
const unwrapped = unwrapData2(value);
|
|
@@ -58868,10 +59096,10 @@ async function controlRequest2(path5, init, timeoutMs = 2e4) {
|
|
|
58868
59096
|
}).catch((err) => {
|
|
58869
59097
|
throw new ResendControlError(`Resend connection control is unavailable: ${err instanceof Error ? err.message : "network error"}`);
|
|
58870
59098
|
});
|
|
58871
|
-
const
|
|
59099
|
+
const text2 = await response.text();
|
|
58872
59100
|
let body = {};
|
|
58873
59101
|
try {
|
|
58874
|
-
body =
|
|
59102
|
+
body = text2 ? JSON.parse(text2) : {};
|
|
58875
59103
|
} catch {
|
|
58876
59104
|
body = {};
|
|
58877
59105
|
}
|
|
@@ -59609,17 +59837,17 @@ app.delete("/memory/approved-senders/:identity", auth5, async (c) => {
|
|
|
59609
59837
|
app.get("/memory/connect", auth5, async (c) => {
|
|
59610
59838
|
const { key: key2, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
59611
59839
|
if (!key2) return c.json({ error: error ?? "memory unavailable" }, 502);
|
|
59612
|
-
const
|
|
59840
|
+
const url2 = "https://memory.mcpscraper.dev/mcp";
|
|
59613
59841
|
const config = {
|
|
59614
59842
|
mcpServers: {
|
|
59615
59843
|
Memory: {
|
|
59616
|
-
url,
|
|
59844
|
+
url: url2,
|
|
59617
59845
|
headers: { Authorization: `Bearer ${key2}` }
|
|
59618
59846
|
}
|
|
59619
59847
|
}
|
|
59620
59848
|
};
|
|
59621
|
-
const codeCommand = `claude mcp add --transport http memory ${
|
|
59622
|
-
return c.json({ ok: true, url, key: key2, config, codeCommand });
|
|
59849
|
+
const codeCommand = `claude mcp add --transport http memory ${url2} --header "Authorization: Bearer ${key2}"`;
|
|
59850
|
+
return c.json({ ok: true, url: url2, key: key2, config, codeCommand });
|
|
59623
59851
|
});
|
|
59624
59852
|
app.post("/memory/vault", auth5, async (c) => {
|
|
59625
59853
|
const { key: key2, error } = await getOrCreateUserMemoryKey(c.get("user"));
|
|
@@ -60224,8 +60452,8 @@ function toolResultText(result2) {
|
|
|
60224
60452
|
const content = result2.content;
|
|
60225
60453
|
if (!Array.isArray(content)) return null;
|
|
60226
60454
|
const textItem = content.find((item) => item && typeof item === "object" && item.type === "text");
|
|
60227
|
-
const
|
|
60228
|
-
return typeof
|
|
60455
|
+
const text2 = textItem?.text;
|
|
60456
|
+
return typeof text2 === "string" ? text2 : null;
|
|
60229
60457
|
}
|
|
60230
60458
|
function parseEmailAddress(headerValue) {
|
|
60231
60459
|
const value = headerValue?.trim();
|
|
@@ -60589,9 +60817,9 @@ app.post("/schedule-connections/actions/call", auth5, requireIntegrationsTier, a
|
|
|
60589
60817
|
}
|
|
60590
60818
|
});
|
|
60591
60819
|
async function forwardIntegrationAlias(c, targetPath) {
|
|
60592
|
-
const
|
|
60593
|
-
|
|
60594
|
-
return await app.fetch(new Request(
|
|
60820
|
+
const url2 = new URL(c.req.url);
|
|
60821
|
+
url2.pathname = targetPath;
|
|
60822
|
+
return await app.fetch(new Request(url2, c.req.raw));
|
|
60595
60823
|
}
|
|
60596
60824
|
app.get("/integrations", (c) => forwardIntegrationAlias(c, "/schedule-connections"));
|
|
60597
60825
|
app.get("/integrations/catalog", (c) => forwardIntegrationAlias(c, "/schedule-connection-catalog"));
|
|
@@ -60992,8 +61220,8 @@ app.post("/harvest", auth5, async (c) => {
|
|
|
60992
61220
|
scheduleDirectPaaHarvest(jobId2);
|
|
60993
61221
|
}
|
|
60994
61222
|
if (options.serpOnly && process.env.CRON_SECRET) {
|
|
60995
|
-
const
|
|
60996
|
-
void fetch(`${
|
|
61223
|
+
const url2 = new URL(c.req.url);
|
|
61224
|
+
void fetch(`${url2.origin}/cron/tick`, {
|
|
60997
61225
|
headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
|
|
60998
61226
|
}).catch(() => {
|
|
60999
61227
|
});
|
|
@@ -61222,7 +61450,7 @@ app.post("/extract-url", auth5, async (c) => {
|
|
|
61222
61450
|
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
61223
61451
|
const bodyResult = (allowLocalRequest ? ExtractUrlLocalBodySchema : ExtractUrlBodySchema).safeParse(raw);
|
|
61224
61452
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
61225
|
-
const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, mediaTypes } = bodyResult.data;
|
|
61453
|
+
const { url: url2, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, includeFeaturedImage, mediaTypes } = bodyResult.data;
|
|
61226
61454
|
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
61227
61455
|
const requestedDelivery = bodyResult.data.delivery ?? "auto";
|
|
61228
61456
|
if (bodyResult.data.depositToVault && !["auto", "memory"].includes(requestedDelivery)) {
|
|
@@ -61236,20 +61464,20 @@ app.post("/extract-url", auth5, async (c) => {
|
|
|
61236
61464
|
const maxMediaAssets = bodyResult.data.maxMediaAssets ?? 100;
|
|
61237
61465
|
const maxInlineImages = bodyResult.data.maxInlineImages ?? 3;
|
|
61238
61466
|
if (!allowLocal) {
|
|
61239
|
-
const checked = await validatePublicHttpUrl(
|
|
61467
|
+
const checked = await validatePublicHttpUrl(url2, { field: "URL" });
|
|
61240
61468
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
61241
61469
|
} else {
|
|
61242
61470
|
try {
|
|
61243
|
-
new URL(
|
|
61471
|
+
new URL(url2);
|
|
61244
61472
|
} catch {
|
|
61245
61473
|
return c.json({ error: "Invalid URL" }, 400);
|
|
61246
61474
|
}
|
|
61247
61475
|
}
|
|
61248
61476
|
const canonicalUrl = (() => {
|
|
61249
61477
|
try {
|
|
61250
|
-
return new URL(
|
|
61478
|
+
return new URL(url2).href;
|
|
61251
61479
|
} catch {
|
|
61252
|
-
return
|
|
61480
|
+
return url2;
|
|
61253
61481
|
}
|
|
61254
61482
|
})();
|
|
61255
61483
|
const waybackReplay = parseWaybackReplayUrl(canonicalUrl);
|
|
@@ -61475,23 +61703,23 @@ app.post("/diff-page", auth5, async (c) => {
|
|
|
61475
61703
|
const allowLocalRequest = permitsLocalApiRequest(c.req.raw);
|
|
61476
61704
|
const bodyResult = (allowLocalRequest ? DiffPageLocalBodySchema : DiffPageBodySchema).safeParse(raw);
|
|
61477
61705
|
if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
61478
|
-
const { url, resetBaseline } = bodyResult.data;
|
|
61706
|
+
const { url: url2, resetBaseline } = bodyResult.data;
|
|
61479
61707
|
const allowLocal = allowLocalRequest && "allowLocal" in bodyResult.data && bodyResult.data.allowLocal === true;
|
|
61480
61708
|
if (!allowLocal) {
|
|
61481
|
-
const checked = await validatePublicHttpUrl(
|
|
61709
|
+
const checked = await validatePublicHttpUrl(url2, { field: "URL" });
|
|
61482
61710
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
61483
61711
|
} else {
|
|
61484
61712
|
try {
|
|
61485
|
-
new URL(
|
|
61713
|
+
new URL(url2);
|
|
61486
61714
|
} catch {
|
|
61487
61715
|
return c.json({ error: "Invalid URL" }, 400);
|
|
61488
61716
|
}
|
|
61489
61717
|
}
|
|
61490
61718
|
const canonicalUrl = (() => {
|
|
61491
61719
|
try {
|
|
61492
|
-
return new URL(
|
|
61720
|
+
return new URL(url2).href;
|
|
61493
61721
|
} catch {
|
|
61494
|
-
return
|
|
61722
|
+
return url2;
|
|
61495
61723
|
}
|
|
61496
61724
|
})();
|
|
61497
61725
|
const user = c.get("user");
|
|
@@ -61619,11 +61847,11 @@ app.post("/archive/read", auth5, async (c) => {
|
|
|
61619
61847
|
message: bodyResult.error.issues[0]?.message ?? "Invalid request"
|
|
61620
61848
|
}, 400);
|
|
61621
61849
|
}
|
|
61622
|
-
const { artifactId, url, path: path5, depositToLibrary } = bodyResult.data;
|
|
61850
|
+
const { artifactId, url: url2, path: path5, depositToLibrary } = bodyResult.data;
|
|
61623
61851
|
const user = c.get("user");
|
|
61624
61852
|
const gate = await acquireConcurrencyGate(user, "archive_read", {
|
|
61625
61853
|
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
61626
|
-
metadata: artifactId ? { artifactId } : { url }
|
|
61854
|
+
metadata: artifactId ? { artifactId } : { url: url2 }
|
|
61627
61855
|
});
|
|
61628
61856
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
61629
61857
|
try {
|
|
@@ -61634,7 +61862,7 @@ app.post("/archive/read", auth5, async (c) => {
|
|
|
61634
61862
|
ownedArchive = { buffer, ref: artifactId };
|
|
61635
61863
|
}
|
|
61636
61864
|
if (!path5) {
|
|
61637
|
-
const inventory = ownedArchive ? await listZipArchiveBuffer(ownedArchive.buffer, ownedArchive.ref, bodyResult.data.maxEntries ?? 200) : await listZipArchive(
|
|
61865
|
+
const inventory = ownedArchive ? await listZipArchiveBuffer(ownedArchive.buffer, ownedArchive.ref, bodyResult.data.maxEntries ?? 200) : await listZipArchive(url2, bodyResult.data.maxEntries ?? 200);
|
|
61638
61866
|
return c.json({ mode: "list", ...inventory, ...artifactId ? { artifactId } : {} });
|
|
61639
61867
|
}
|
|
61640
61868
|
const file = ownedArchive ? await readZipArchiveBuffer(
|
|
@@ -61644,7 +61872,7 @@ app.post("/archive/read", auth5, async (c) => {
|
|
|
61644
61872
|
bodyResult.data.offset ?? 0,
|
|
61645
61873
|
bodyResult.data.maxBytes ?? 5e4
|
|
61646
61874
|
) : await readZipArchiveFile(
|
|
61647
|
-
|
|
61875
|
+
url2,
|
|
61648
61876
|
path5,
|
|
61649
61877
|
bodyResult.data.offset ?? 0,
|
|
61650
61878
|
bodyResult.data.maxBytes ?? 5e4
|