mcp-scraper 0.51.2 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/bin/api-server.cjs +2819 -869
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +5 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +355 -28
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +4 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-ZSJHRZY5.js → chunk-J7TH5KU7.js} +2 -2
- package/dist/{chunk-FCZ3QCZP.js → chunk-OT2AF7SH.js} +356 -29
- package/dist/chunk-OT2AF7SH.js.map +1 -0
- package/dist/chunk-SG2PEHR3.js +562 -0
- package/dist/chunk-SG2PEHR3.js.map +1 -0
- package/dist/{chunk-GOZIG6HD.js → chunk-TVZD37SE.js} +2 -2
- package/dist/{chunk-3ZUBQQPQ.js → chunk-V7CEOUBF.js} +5 -1
- package/dist/chunk-V7CEOUBF.js.map +1 -0
- package/dist/chunk-VIAXQTZK.js +7 -0
- package/dist/chunk-VIAXQTZK.js.map +1 -0
- package/dist/{extract-bundle-RCTNANCH.js → extract-bundle-GUEUCTAE.js} +2 -2
- package/dist/index.cjs +4 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{server-KAEHVDY7.js → server-7EXEDKAU.js} +1857 -401
- package/dist/server-7EXEDKAU.js.map +1 -0
- package/dist/{worker-KQN673JF.js → worker-JL4TG6IF.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-27FMOD6S.js +0 -430
- package/dist/chunk-27FMOD6S.js.map +0 -1
- package/dist/chunk-3ZUBQQPQ.js.map +0 -1
- package/dist/chunk-FCZ3QCZP.js.map +0 -1
- package/dist/chunk-JNRSR5ZJ.js +0 -7
- package/dist/chunk-JNRSR5ZJ.js.map +0 -1
- package/dist/server-KAEHVDY7.js.map +0 -1
- /package/dist/{chunk-ZSJHRZY5.js.map → chunk-J7TH5KU7.js.map} +0 -0
- /package/dist/{chunk-GOZIG6HD.js.map → chunk-TVZD37SE.js.map} +0 -0
- /package/dist/{extract-bundle-RCTNANCH.js.map → extract-bundle-GUEUCTAE.js.map} +0 -0
- /package/dist/{worker-KQN673JF.js.map → worker-JL4TG6IF.js.map} +0 -0
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
3
3
|
cleanupExpiredSiteExtractArtifacts,
|
|
4
|
+
discoverStaticMedia,
|
|
4
5
|
harvestPageMedia,
|
|
5
6
|
readOwnedSiteExtractArtifactBuffer,
|
|
6
|
-
renewSiteExtractArtifactDownload
|
|
7
|
-
|
|
7
|
+
renewSiteExtractArtifactDownload,
|
|
8
|
+
typeFromMime
|
|
9
|
+
} from "./chunk-SG2PEHR3.js";
|
|
8
10
|
import {
|
|
9
11
|
REQUIRED_LOCATION_STATE_CODES,
|
|
10
12
|
importHostedCensusPlacesCsv,
|
|
@@ -100,7 +102,7 @@ import {
|
|
|
100
102
|
resolveKernelProxyId,
|
|
101
103
|
serpArrivalUrl,
|
|
102
104
|
withRuntimeUserAgent
|
|
103
|
-
} from "./chunk-
|
|
105
|
+
} from "./chunk-TVZD37SE.js";
|
|
104
106
|
import {
|
|
105
107
|
ApiErrorResponseSchema,
|
|
106
108
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -158,7 +160,7 @@ import {
|
|
|
158
160
|
resolveDeploymentProfile,
|
|
159
161
|
resolveLocalSourcebookSchemaType,
|
|
160
162
|
transcribeMediaUrl
|
|
161
|
-
} from "./chunk-
|
|
163
|
+
} from "./chunk-OT2AF7SH.js";
|
|
162
164
|
import {
|
|
163
165
|
auditImageUrls,
|
|
164
166
|
auditImages,
|
|
@@ -196,7 +198,7 @@ import {
|
|
|
196
198
|
workflowOutputBaseDir,
|
|
197
199
|
workflowStepCount,
|
|
198
200
|
workflowSupportsSteps
|
|
199
|
-
} from "./chunk-
|
|
201
|
+
} from "./chunk-J7TH5KU7.js";
|
|
200
202
|
import {
|
|
201
203
|
csvRecords,
|
|
202
204
|
parseCsv,
|
|
@@ -211,10 +213,10 @@ import {
|
|
|
211
213
|
RawMapsHoursRowSchema,
|
|
212
214
|
RawMapsOverviewSchema,
|
|
213
215
|
RawMapsReviewStatsSchema
|
|
214
|
-
} from "./chunk-
|
|
216
|
+
} from "./chunk-V7CEOUBF.js";
|
|
215
217
|
import {
|
|
216
218
|
PACKAGE_VERSION
|
|
217
|
-
} from "./chunk-
|
|
219
|
+
} from "./chunk-VIAXQTZK.js";
|
|
218
220
|
import {
|
|
219
221
|
abandonExtractSettlement,
|
|
220
222
|
countSuccessfulPages,
|
|
@@ -3813,39 +3815,135 @@ async function readResponseTextWithLimit(response, maxBytes, label) {
|
|
|
3813
3815
|
}
|
|
3814
3816
|
|
|
3815
3817
|
// src/api/kernel-fetch.ts
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3818
|
+
var CHALLENGE_PATTERNS = [
|
|
3819
|
+
{ reason: "cloudflare_title", pattern: /<title[^>]*>\s*just a moment(?:\.\.\.)?\s*<\/title>/i },
|
|
3820
|
+
{ reason: "cloudflare_challenge_platform", pattern: /(?:cdn-cgi\/challenge-platform|challenge-platform\/h\/|cf-chl-|cf_chl_)/i },
|
|
3821
|
+
{ reason: "cloudflare_browser_verification", pattern: /(?:cf-browser-verification|checking your browser before accessing)/i },
|
|
3822
|
+
{ reason: "cloudflare_cookie_message", pattern: /enable javascript and cookies to continue/i },
|
|
3823
|
+
{ reason: "generic_security_check", pattern: /performing security verification|verify you are human/i }
|
|
3824
|
+
];
|
|
3825
|
+
function detectBrowserChallenge(input) {
|
|
3826
|
+
if (input.status === 403 || input.status === 429) {
|
|
3827
|
+
return { detected: true, reason: `http_${input.status}` };
|
|
3828
|
+
}
|
|
3829
|
+
const sample = `${input.title ?? ""}
|
|
3830
|
+
${input.html ?? ""}`.slice(0, 25e4);
|
|
3831
|
+
for (const candidate of CHALLENGE_PATTERNS) {
|
|
3832
|
+
if (candidate.pattern.test(sample)) return { detected: true, reason: candidate.reason };
|
|
3833
|
+
}
|
|
3834
|
+
return { detected: false, reason: null };
|
|
3835
|
+
}
|
|
3836
|
+
async function runKernelAttempt(client2, auditedUrl, mode) {
|
|
3823
3837
|
const openedAtMs = Date.now();
|
|
3824
|
-
const
|
|
3825
|
-
const kb = await client2.browsers.create({
|
|
3838
|
+
const headlessSent = mode === "stealth_headless" ? true : null;
|
|
3839
|
+
const kb = await client2.browsers.create({
|
|
3840
|
+
stealth: true,
|
|
3841
|
+
...headlessSent ? { headless: true } : {},
|
|
3842
|
+
timeout_seconds: 60
|
|
3843
|
+
});
|
|
3826
3844
|
const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
|
|
3845
|
+
let attemptError = null;
|
|
3827
3846
|
try {
|
|
3828
3847
|
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
3829
|
-
const context = await browser.newContext(withRuntimeUserAgent({
|
|
3830
|
-
serviceWorkers: "block"
|
|
3831
|
-
}, runtimeUserAgent));
|
|
3848
|
+
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
3832
3849
|
const page = context.pages()[0] ?? await context.newPage();
|
|
3833
3850
|
const guard = await installBrowserRequestGuard(page, auditedUrl);
|
|
3834
|
-
await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
3851
|
+
const response = await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
3835
3852
|
if (guard.blockedReason && page.url() === "about:blank") throw new Error(guard.blockedReason);
|
|
3836
3853
|
await page.waitForTimeout(2500);
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3854
|
+
let status = response?.status() ?? null;
|
|
3855
|
+
let title = await page.title().catch(() => "");
|
|
3856
|
+
let html = await page.content();
|
|
3857
|
+
let challenge = detectBrowserChallenge({ status, title, html });
|
|
3858
|
+
if (challenge.detected && mode === "stealth_headful") {
|
|
3859
|
+
const deadline = Date.now() + 2e4;
|
|
3860
|
+
while (challenge.detected && Date.now() < deadline) {
|
|
3861
|
+
await page.waitForTimeout(1e3);
|
|
3862
|
+
title = await page.title().catch(() => "");
|
|
3863
|
+
html = await page.content();
|
|
3864
|
+
challenge = detectBrowserChallenge({ status, title, html });
|
|
3865
|
+
}
|
|
3866
|
+
status = response?.status() ?? status;
|
|
3867
|
+
}
|
|
3868
|
+
const byteLength2 = new TextEncoder().encode(html).byteLength;
|
|
3869
|
+
if (byteLength2 > MAX_PAGE_HTML_BYTES) throw new Error(`Browser HTML exceeds ${MAX_PAGE_HTML_BYTES} byte limit`);
|
|
3870
|
+
return { html, status, challenge, durationMs: Date.now() - openedAtMs };
|
|
3871
|
+
} catch (error) {
|
|
3872
|
+
attemptError = error instanceof Error ? error.message : String(error);
|
|
3873
|
+
throw error;
|
|
3842
3874
|
} finally {
|
|
3843
3875
|
await browser.close().catch(() => {
|
|
3844
3876
|
});
|
|
3845
3877
|
await client2.browsers.deleteByID(kb.session_id).catch(() => {
|
|
3846
3878
|
});
|
|
3847
|
-
void recordKernelSession({
|
|
3879
|
+
void recordKernelSession({
|
|
3880
|
+
kernelSessionId: kb.session_id,
|
|
3881
|
+
source: "kernel_fetch",
|
|
3882
|
+
stealth: typeof kb.stealth === "boolean" ? kb.stealth : null,
|
|
3883
|
+
headlessSent,
|
|
3884
|
+
proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null,
|
|
3885
|
+
fallback: true,
|
|
3886
|
+
openedAtMs,
|
|
3887
|
+
closedAtMs: Date.now(),
|
|
3888
|
+
error: attemptError
|
|
3889
|
+
});
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
async function fetchWithKernelDetailed(url) {
|
|
3893
|
+
const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
|
|
3894
|
+
if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
|
|
3895
|
+
const auditedUrl = checked.parsed.href;
|
|
3896
|
+
const apiKey = browserServiceApiKey();
|
|
3897
|
+
if (!apiKey) throw new Error("Browser backend API key not set");
|
|
3898
|
+
const client2 = new Kernel({ apiKey });
|
|
3899
|
+
const startsHeadful = currentCostContext()?.forceHeadful === true;
|
|
3900
|
+
const modes = startsHeadful ? ["stealth_headful"] : ["stealth_headless", "stealth_headful"];
|
|
3901
|
+
const attempts = [];
|
|
3902
|
+
for (const mode of modes) {
|
|
3903
|
+
const result = await runKernelAttempt(client2, auditedUrl, mode);
|
|
3904
|
+
if (result.challenge.detected) {
|
|
3905
|
+
const completion = mode === "stealth_headful" ? "challenge_unresolved" : "challenge_detected";
|
|
3906
|
+
attempts.push({
|
|
3907
|
+
mode,
|
|
3908
|
+
status: result.status,
|
|
3909
|
+
challengeDetected: true,
|
|
3910
|
+
challengeReason: result.challenge.reason,
|
|
3911
|
+
completion,
|
|
3912
|
+
durationMs: result.durationMs
|
|
3913
|
+
});
|
|
3914
|
+
if (mode === "stealth_headful") {
|
|
3915
|
+
const error = new Error(`browser_challenge_unresolved: ${result.challenge.reason ?? "challenge page remained after bounded wait"}`);
|
|
3916
|
+
error.code = "browser_challenge_unresolved";
|
|
3917
|
+
error.attempts = attempts;
|
|
3918
|
+
throw error;
|
|
3919
|
+
}
|
|
3920
|
+
continue;
|
|
3921
|
+
}
|
|
3922
|
+
if (result.status != null && (result.status < 200 || result.status >= 400)) {
|
|
3923
|
+
attempts.push({
|
|
3924
|
+
mode,
|
|
3925
|
+
status: result.status,
|
|
3926
|
+
challengeDetected: false,
|
|
3927
|
+
challengeReason: null,
|
|
3928
|
+
completion: "http_error",
|
|
3929
|
+
durationMs: result.durationMs
|
|
3930
|
+
});
|
|
3931
|
+
throw new Error(`browser_http_${result.status}: Browser fetch returned HTTP ${result.status}`);
|
|
3932
|
+
}
|
|
3933
|
+
attempts.push({
|
|
3934
|
+
mode,
|
|
3935
|
+
status: result.status,
|
|
3936
|
+
challengeDetected: false,
|
|
3937
|
+
challengeReason: null,
|
|
3938
|
+
completion: "usable_page",
|
|
3939
|
+
durationMs: result.durationMs
|
|
3940
|
+
});
|
|
3941
|
+
return { html: result.html, mode, attempts };
|
|
3848
3942
|
}
|
|
3943
|
+
throw new Error("browser_challenge_unresolved: no usable browser response");
|
|
3944
|
+
}
|
|
3945
|
+
async function fetchWithKernel(url) {
|
|
3946
|
+
return (await fetchWithKernelDetailed(url)).html;
|
|
3849
3947
|
}
|
|
3850
3948
|
|
|
3851
3949
|
// src/lib/http-user-agent.ts
|
|
@@ -3875,6 +3973,8 @@ async function extractKpo(opts) {
|
|
|
3875
3973
|
}
|
|
3876
3974
|
if (!res.ok) return null;
|
|
3877
3975
|
const text2 = await res.text();
|
|
3976
|
+
const title2 = text2.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? null;
|
|
3977
|
+
if (detectBrowserChallenge({ status: res.status, title: title2, html: text2 }).detected) return null;
|
|
3878
3978
|
if (text2.length < 2e3) return null;
|
|
3879
3979
|
const stripped = text2.replace(/<(script|style|noscript|svg|iframe|canvas)[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
3880
3980
|
const contentBlock = stripped.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? stripped.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? "";
|
|
@@ -3889,13 +3989,16 @@ async function extractKpo(opts) {
|
|
|
3889
3989
|
};
|
|
3890
3990
|
let html;
|
|
3891
3991
|
let fetchedVia = "fetch";
|
|
3992
|
+
let browserFetch;
|
|
3892
3993
|
const plainFetchStart = Date.now();
|
|
3893
3994
|
const plainHtml = await tryPlainFetch();
|
|
3894
3995
|
const plainFetchMs = Date.now() - plainFetchStart;
|
|
3895
3996
|
if (plainHtml) {
|
|
3896
3997
|
html = plainHtml;
|
|
3897
3998
|
} else if (opts.kernelApiKey) {
|
|
3898
|
-
|
|
3999
|
+
const browserResult = await fetchWithKernelDetailed(parsed.href);
|
|
4000
|
+
html = browserResult.html;
|
|
4001
|
+
browserFetch = { mode: browserResult.mode, attempts: browserResult.attempts };
|
|
3899
4002
|
fetchedVia = "headless";
|
|
3900
4003
|
} else {
|
|
3901
4004
|
throw new Error("Fetch failed: site blocked plain HTTP and no headless browser backend is configured");
|
|
@@ -4195,6 +4298,7 @@ async function extractKpo(opts) {
|
|
|
4195
4298
|
return {
|
|
4196
4299
|
url: parsed.href,
|
|
4197
4300
|
fetchedVia,
|
|
4301
|
+
browserFetch,
|
|
4198
4302
|
durationMs,
|
|
4199
4303
|
plainFetchMs,
|
|
4200
4304
|
title,
|
|
@@ -4212,13 +4316,47 @@ import { chromium as chromium2 } from "playwright";
|
|
|
4212
4316
|
import Kernel2 from "@onkernel/sdk";
|
|
4213
4317
|
|
|
4214
4318
|
// src/lib/branding-extractor.ts
|
|
4215
|
-
function
|
|
4216
|
-
const
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4319
|
+
function selectRankedLogoCandidates(candidates) {
|
|
4320
|
+
const ranked = candidates.filter((candidate) => {
|
|
4321
|
+
if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.region === "body") return false;
|
|
4322
|
+
if (candidate.region === "json_ld" || candidate.region === "favicon") return true;
|
|
4323
|
+
return candidate.evidence.includes("links_to_homepage") || candidate.evidence.includes("site_identity_marker");
|
|
4324
|
+
}).map((candidate) => ({
|
|
4325
|
+
...candidate,
|
|
4326
|
+
confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
|
|
4327
|
+
evidence: [...new Set(candidate.evidence)]
|
|
4328
|
+
})).sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
|
|
4329
|
+
const primary = ranked.find((candidate) => candidate.score >= 55 && candidate.region !== "favicon") ?? null;
|
|
4330
|
+
return {
|
|
4331
|
+
primary,
|
|
4332
|
+
variants: primary ? ranked.filter((candidate) => candidate.url !== primary.url && candidate.score >= 55 && candidate.region !== "favicon" && logoFamilyKey(candidate.url) === logoFamilyKey(primary.url)).slice(0, 5).map((candidate) => candidate.url) : [],
|
|
4333
|
+
candidates: ranked,
|
|
4334
|
+
selectionReason: primary?.evidence.join(", ") ?? null
|
|
4335
|
+
};
|
|
4336
|
+
}
|
|
4337
|
+
function logoFamilyKey(value) {
|
|
4338
|
+
try {
|
|
4339
|
+
const url = new URL(value);
|
|
4340
|
+
const host2 = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
4341
|
+
const path5 = decodeURIComponent(url.pathname).toLowerCase().replace(/-\d+x\d+(?=\.[a-z0-9]+$)/, "");
|
|
4342
|
+
return `${host2}${path5}`;
|
|
4343
|
+
} catch {
|
|
4344
|
+
return value.toLowerCase();
|
|
4345
|
+
}
|
|
4346
|
+
}
|
|
4347
|
+
function selectProofImages(candidates) {
|
|
4348
|
+
const byUrl = /* @__PURE__ */ new Map();
|
|
4349
|
+
for (const candidate of candidates) {
|
|
4350
|
+
if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.score < 45) continue;
|
|
4351
|
+
const normalized = {
|
|
4352
|
+
...candidate,
|
|
4353
|
+
confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
|
|
4354
|
+
evidence: [...new Set(candidate.evidence)]
|
|
4355
|
+
};
|
|
4356
|
+
const existing = byUrl.get(normalized.url);
|
|
4357
|
+
if (!existing || normalized.score > existing.score) byUrl.set(normalized.url, normalized);
|
|
4358
|
+
}
|
|
4359
|
+
return [...byUrl.values()].sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
|
|
4222
4360
|
}
|
|
4223
4361
|
function luminance(hex) {
|
|
4224
4362
|
const m = hex.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
|
|
@@ -4232,6 +4370,12 @@ function isTransparentOrWhite(hex) {
|
|
|
4232
4370
|
const lum = luminance(hex);
|
|
4233
4371
|
return lum > 240;
|
|
4234
4372
|
}
|
|
4373
|
+
function isChromatic(hex) {
|
|
4374
|
+
const match = hex?.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
|
|
4375
|
+
if (!match) return false;
|
|
4376
|
+
const channels = [parseInt(match[1], 16), parseInt(match[2], 16), parseInt(match[3], 16)];
|
|
4377
|
+
return Math.max(...channels) - Math.min(...channels) >= 24 && luminance(hex) > 18;
|
|
4378
|
+
}
|
|
4235
4379
|
function firstFont(fontFamily) {
|
|
4236
4380
|
if (!fontFamily) return null;
|
|
4237
4381
|
const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
|
|
@@ -4240,14 +4384,34 @@ function firstFont(fontFamily) {
|
|
|
4240
4384
|
function dominantColor(freq) {
|
|
4241
4385
|
return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
|
4242
4386
|
}
|
|
4387
|
+
function dominantChromaticColor(freq) {
|
|
4388
|
+
return Object.entries(freq).filter(([hex]) => isChromatic(hex)).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
|
4389
|
+
}
|
|
4243
4390
|
async function extractBrandingFromPage(page) {
|
|
4391
|
+
const challenge = detectBrowserChallenge({
|
|
4392
|
+
title: await page.title().catch(() => ""),
|
|
4393
|
+
html: await page.content().catch(() => "")
|
|
4394
|
+
});
|
|
4395
|
+
if (challenge.detected) {
|
|
4396
|
+
const error = new Error(`browser_challenge_unresolved: ${challenge.reason ?? "challenge page is not branding evidence"}`);
|
|
4397
|
+
error.code = "browser_challenge_unresolved";
|
|
4398
|
+
throw error;
|
|
4399
|
+
}
|
|
4244
4400
|
const evalScript = `
|
|
4245
4401
|
(function() {
|
|
4246
4402
|
function cs(el) { return el ? window.getComputedStyle(el) : null; }
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4403
|
+
var colorCanvas = document.createElement('canvas');
|
|
4404
|
+
colorCanvas.width = 1; colorCanvas.height = 1;
|
|
4405
|
+
var colorContext = colorCanvas.getContext('2d', { willReadFrequently: true });
|
|
4406
|
+
function toHex(value) {
|
|
4407
|
+
if (!value || !colorContext) return null;
|
|
4408
|
+
colorContext.clearRect(0, 0, 1, 1);
|
|
4409
|
+
colorContext.fillStyle = 'rgba(1, 2, 3, 0.001)';
|
|
4410
|
+
colorContext.fillStyle = value;
|
|
4411
|
+
colorContext.fillRect(0, 0, 1, 1);
|
|
4412
|
+
var data = colorContext.getImageData(0, 0, 1, 1).data;
|
|
4413
|
+
if (data[3] < 64) return null;
|
|
4414
|
+
return '#' + [data[0], data[1], data[2]].map(function(v){ return ('0'+v.toString(16)).slice(-2); }).join('');
|
|
4251
4415
|
}
|
|
4252
4416
|
function isUsable(hex) {
|
|
4253
4417
|
if (!hex) return false;
|
|
@@ -4256,30 +4420,63 @@ async function extractBrandingFromPage(page) {
|
|
|
4256
4420
|
return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
|
|
4257
4421
|
}
|
|
4258
4422
|
|
|
4423
|
+
function isChromatic(hex) {
|
|
4424
|
+
if (!hex) return false;
|
|
4425
|
+
var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
|
|
4426
|
+
return Math.max(r,g,b) - Math.min(r,g,b) >= 24 && (0.2126*r + 0.7152*g + 0.0722*b) > 18;
|
|
4427
|
+
}
|
|
4428
|
+
|
|
4429
|
+
function visibleRect(el) {
|
|
4430
|
+
if (!el) return null;
|
|
4431
|
+
var style = cs(el);
|
|
4432
|
+
var rect = el.getBoundingClientRect();
|
|
4433
|
+
if (!style || style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') <= 0 || rect.width < 2 || rect.height < 2) return null;
|
|
4434
|
+
return rect;
|
|
4435
|
+
}
|
|
4436
|
+
|
|
4259
4437
|
var navEl = document.querySelector('nav, header, [role="banner"]');
|
|
4260
4438
|
var bodyEl = document.body;
|
|
4261
4439
|
var h1El = document.querySelector('h1');
|
|
4262
|
-
var btnEl
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4440
|
+
var btnEl = null;
|
|
4441
|
+
var btnScore = -Infinity;
|
|
4442
|
+
var interactive = document.querySelectorAll('a[href], button, [role="button"]');
|
|
4443
|
+
for (var bi = 0; bi < interactive.length; bi++) {
|
|
4444
|
+
var candidateButton = interactive[bi];
|
|
4445
|
+
var buttonRect = visibleRect(candidateButton);
|
|
4446
|
+
if (!buttonRect || buttonRect.width < 44 || buttonRect.height < 20) continue;
|
|
4447
|
+
if (buttonRect.bottom < 0 || buttonRect.top > window.innerHeight) continue;
|
|
4448
|
+
var buttonStyle = cs(candidateButton);
|
|
4449
|
+
var buttonBg = buttonStyle ? toHex(buttonStyle.backgroundColor) : null;
|
|
4450
|
+
if (!buttonBg) continue;
|
|
4451
|
+
var buttonDescriptor = [candidateButton.textContent, candidateButton.id, candidateButton.className, candidateButton.getAttribute('aria-label')].join(' ');
|
|
4452
|
+
if (/\\b(?:skip|screen-reader|sr-only|visually-hidden)\\b/i.test(buttonDescriptor)) continue;
|
|
4453
|
+
var buttonText = String(candidateButton.textContent || candidateButton.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim();
|
|
4454
|
+
var explicitButtonEvidence = /\\b(?:btn[-_ ]?primary|button[-_ ]?primary|cta|call[-_ ]?to[-_ ]?action)\\b/i.test(buttonDescriptor);
|
|
4455
|
+
var actionTextEvidence = /\\b(?:get started|start|try|book|contact|schedule|request|sign up|join|buy|shop|call|quote|estimate|verify)\\b/i.test(buttonDescriptor);
|
|
4456
|
+
if (!explicitButtonEvidence && !actionTextEvidence && (buttonRect.height > 100 || buttonText.length > 80)) continue;
|
|
4457
|
+
var score = 0;
|
|
4458
|
+
if (isChromatic(buttonBg)) score += 35;
|
|
4459
|
+
if (explicitButtonEvidence) score += 30;
|
|
4460
|
+
if (actionTextEvidence) score += 20;
|
|
4461
|
+
if (candidateButton.closest('main, [role="main"], [class*="hero"]')) score += 12;
|
|
4462
|
+
if (candidateButton.closest('header, nav, [role="banner"]')) score += 4;
|
|
4463
|
+
score += Math.min(10, Math.round((buttonRect.width * buttonRect.height) / 5000));
|
|
4464
|
+
if (score > btnScore) { btnEl = candidateButton; btnScore = score; }
|
|
4465
|
+
}
|
|
4272
4466
|
var navStyle = cs(navEl);
|
|
4273
4467
|
var bodyStyle = cs(bodyEl);
|
|
4274
4468
|
var h1Style = cs(h1El);
|
|
4275
4469
|
var btnStyle = cs(btnEl);
|
|
4276
4470
|
|
|
4277
4471
|
var svgFreq = {};
|
|
4278
|
-
var svgScope =
|
|
4472
|
+
var svgScope = document;
|
|
4279
4473
|
if (svgScope) {
|
|
4280
|
-
var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
|
|
4474
|
+
var svgEls = svgScope.querySelectorAll('svg *, svg[fill], svg path, svg circle, svg rect, svg polygon, svg polyline');
|
|
4281
4475
|
for (var si = 0; si < svgEls.length; si++) {
|
|
4282
4476
|
var svgEl = svgEls[si];
|
|
4477
|
+
var owningSvg = svgEl.closest('svg');
|
|
4478
|
+
var owningRect = owningSvg ? owningSvg.getBoundingClientRect() : null;
|
|
4479
|
+
if (!owningRect || owningRect.bottom < 0 || owningRect.top > 300 || owningRect.width < 12 || owningRect.height < 12) continue;
|
|
4283
4480
|
var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
|
|
4284
4481
|
var fillAttr = svgEl.getAttribute('fill');
|
|
4285
4482
|
var fillHex = null;
|
|
@@ -4301,55 +4498,251 @@ async function extractBrandingFromPage(page) {
|
|
|
4301
4498
|
}
|
|
4302
4499
|
}
|
|
4303
4500
|
|
|
4501
|
+
var cssPrimaryCandidates = [];
|
|
4502
|
+
var cssAccentCandidates = [];
|
|
4503
|
+
var rootStyle = cs(document.documentElement);
|
|
4504
|
+
if (rootStyle) {
|
|
4505
|
+
for (var ci = 0; ci < rootStyle.length; ci++) {
|
|
4506
|
+
var propertyName = rootStyle[ci];
|
|
4507
|
+
if (!propertyName || propertyName.indexOf('--') !== 0) continue;
|
|
4508
|
+
var normalizedName = propertyName.toLowerCase();
|
|
4509
|
+
if (!/(?:^|[-_])(?:brand|primary|accent)(?:$|[-_])/.test(normalizedName)) continue;
|
|
4510
|
+
if (/(?:on[-_]?primary|primary[-_]?(?:text|foreground)|accent[-_]?(?:text|foreground))/.test(normalizedName)) continue;
|
|
4511
|
+
var propertyValue = rootStyle.getPropertyValue(propertyName).trim();
|
|
4512
|
+
var propertyHex = toHex(propertyValue);
|
|
4513
|
+
if (!propertyHex || !isChromatic(propertyHex)) continue;
|
|
4514
|
+
var item = { hex: propertyHex, name: propertyName };
|
|
4515
|
+
if (/(?:^|[-_])primary(?:$|[-_])|brand/.test(normalizedName)) cssPrimaryCandidates.push(item);
|
|
4516
|
+
if (/(?:^|[-_])accent(?:$|[-_])/.test(normalizedName)) cssAccentCandidates.push(item);
|
|
4517
|
+
}
|
|
4518
|
+
}
|
|
4519
|
+
|
|
4520
|
+
var themeColorEl = document.querySelector('meta[name="theme-color"]');
|
|
4521
|
+
var themeColor = themeColorEl ? toHex(themeColorEl.getAttribute('content')) : null;
|
|
4522
|
+
|
|
4523
|
+
var visibleBgFreq = {};
|
|
4524
|
+
var topBgFreq = {};
|
|
4525
|
+
var visibleElements = document.querySelectorAll('body *');
|
|
4526
|
+
for (var vi = 0; vi < visibleElements.length && vi < 1500; vi++) {
|
|
4527
|
+
var visibleElement = visibleElements[vi];
|
|
4528
|
+
var visibleElementRect = visibleRect(visibleElement);
|
|
4529
|
+
if (!visibleElementRect || visibleElementRect.width < 20 || visibleElementRect.height < 12) continue;
|
|
4530
|
+
var visibleStyle = cs(visibleElement);
|
|
4531
|
+
var visibleHex = visibleStyle ? toHex(visibleStyle.backgroundColor) : null;
|
|
4532
|
+
if (!visibleHex || !isChromatic(visibleHex)) continue;
|
|
4533
|
+
var areaWeight = Math.max(1, Math.min(20, Math.round(Math.sqrt(visibleElementRect.width * visibleElementRect.height) / 40)));
|
|
4534
|
+
visibleBgFreq[visibleHex] = (visibleBgFreq[visibleHex] || 0) + areaWeight;
|
|
4535
|
+
if (visibleElementRect.top < 300 && visibleElementRect.bottom > 0 && visibleElementRect.width >= window.innerWidth * 0.5) {
|
|
4536
|
+
topBgFreq[visibleHex] = (topBgFreq[visibleHex] || 0) + areaWeight;
|
|
4537
|
+
}
|
|
4538
|
+
}
|
|
4539
|
+
|
|
4304
4540
|
var pageHost = window.location.hostname.replace(/^www\\./, '');
|
|
4305
4541
|
function isSameDomain(src) {
|
|
4306
|
-
try {
|
|
4542
|
+
try {
|
|
4543
|
+
var host = new URL(src).hostname.replace(/^www\\./, '');
|
|
4544
|
+
return host === pageHost || host.endsWith('.' + pageHost);
|
|
4545
|
+
} catch { return false; }
|
|
4307
4546
|
}
|
|
4308
|
-
var
|
|
4309
|
-
|
|
4310
|
-
'
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
var
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4547
|
+
var logoCandidates = {};
|
|
4548
|
+
function addLogo(raw, region, evidence, score, alt, width, height) {
|
|
4549
|
+
if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
|
|
4550
|
+
var url;
|
|
4551
|
+
try { url = new URL(raw, document.baseURI); } catch (_) { return; }
|
|
4552
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
|
|
4553
|
+
url.hash = '';
|
|
4554
|
+
var key = url.href;
|
|
4555
|
+
var existing = logoCandidates[key] || {
|
|
4556
|
+
url: key, score: 0, region: region, evidence: [], alt: null, width: null, height: null
|
|
4557
|
+
};
|
|
4558
|
+
existing.score = Math.max(existing.score, score);
|
|
4559
|
+
existing.evidence = Array.from(new Set(existing.evidence.concat(evidence)));
|
|
4560
|
+
if (region === 'json_ld' || (existing.region === 'body' && region !== 'body')) existing.region = region;
|
|
4561
|
+
if (alt && !existing.alt) existing.alt = alt.replace(/\\s+/g, ' ').trim().slice(0, 300) || null;
|
|
4562
|
+
if ((width || 0) * (height || 0) > (existing.width || 0) * (existing.height || 0)) {
|
|
4563
|
+
existing.width = width || null; existing.height = height || null;
|
|
4564
|
+
}
|
|
4565
|
+
logoCandidates[key] = existing;
|
|
4566
|
+
}
|
|
4567
|
+
|
|
4568
|
+
function jsonLogo(value) {
|
|
4569
|
+
if (typeof value === 'string') return value;
|
|
4570
|
+
if (!value || typeof value !== 'object') return null;
|
|
4571
|
+
return typeof value.url === 'string' ? value.url : typeof value.contentUrl === 'string' ? value.contentUrl : null;
|
|
4572
|
+
}
|
|
4573
|
+
var jsonScripts = document.querySelectorAll('script[type="application/ld+json"]');
|
|
4574
|
+
for (var ji = 0; ji < jsonScripts.length; ji++) {
|
|
4575
|
+
try {
|
|
4576
|
+
var parsed = JSON.parse(jsonScripts[ji].textContent || 'null');
|
|
4577
|
+
var stack = Array.isArray(parsed) ? parsed.slice() : [parsed];
|
|
4578
|
+
var visited = 0;
|
|
4579
|
+
while (stack.length && visited++ < 1000) {
|
|
4580
|
+
var node = stack.pop();
|
|
4581
|
+
if (!node || typeof node !== 'object') continue;
|
|
4582
|
+
var types = Array.isArray(node['@type']) ? node['@type'] : [node['@type']];
|
|
4583
|
+
var entity = types.some(function(t) { return /^(Organization|Corporation|LocalBusiness|WebSite|Brand)$/i.test(String(t || '')); });
|
|
4584
|
+
var logo = entity ? jsonLogo(node.logo) : null;
|
|
4585
|
+
if (logo) addLogo(logo, 'json_ld', ['json_ld_logo'], 100, null, null, null);
|
|
4586
|
+
Object.keys(node).forEach(function(key) {
|
|
4587
|
+
if (key === '@context') return;
|
|
4588
|
+
var child = node[key];
|
|
4589
|
+
if (Array.isArray(child)) stack = stack.concat(child.filter(function(value) { return value && typeof value === 'object'; }));
|
|
4590
|
+
else if (child && typeof child === 'object') stack.push(child);
|
|
4591
|
+
});
|
|
4592
|
+
}
|
|
4593
|
+
} catch (_) {}
|
|
4594
|
+
}
|
|
4595
|
+
|
|
4596
|
+
var proofCandidates = {};
|
|
4597
|
+
var pageHeadings = Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6'));
|
|
4598
|
+
function nearestHeadingText(element) {
|
|
4599
|
+
var nearest = null;
|
|
4600
|
+
for (var hi = 0; hi < pageHeadings.length; hi++) {
|
|
4601
|
+
if (pageHeadings[hi].compareDocumentPosition(element) & 4) nearest = pageHeadings[hi];
|
|
4602
|
+
else break;
|
|
4603
|
+
}
|
|
4604
|
+
return nearest && nearest.textContent ? nearest.textContent.replace(/\\s+/g, ' ').trim().slice(0, 300) : null;
|
|
4605
|
+
}
|
|
4606
|
+
function proofTypeFor(descriptor, context) {
|
|
4607
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(descriptor)) return 'accreditation';
|
|
4608
|
+
if (/\\b(?:certifi(?:ed|cation)|compliance|legitscript|hipaa)\\b/i.test(descriptor)) return 'certification';
|
|
4609
|
+
if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(descriptor)) return 'award';
|
|
4610
|
+
if (/\\b(?:member|membership|association|professionals?)\\b/i.test(descriptor)) return 'membership';
|
|
4611
|
+
if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(descriptor)) return 'press_mention';
|
|
4612
|
+
if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(descriptor)) return 'partner_or_customer';
|
|
4613
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(context || '')) return 'accreditation';
|
|
4614
|
+
if (/\\b(?:certifi(?:ed|cation)|compliance)\\b/i.test(context || '')) return 'certification';
|
|
4615
|
+
if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(context || '')) return 'award';
|
|
4616
|
+
if (/\\b(?:member|membership|association)\\b/i.test(context || '')) return 'membership';
|
|
4617
|
+
if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(context || '')) return 'press_mention';
|
|
4618
|
+
if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(context || '')) return 'partner_or_customer';
|
|
4619
|
+
return 'trust_mark';
|
|
4620
|
+
}
|
|
4621
|
+
function addProof(raw, proofType, score, evidence, alt, context, width, height) {
|
|
4622
|
+
if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
|
|
4623
|
+
var url;
|
|
4624
|
+
try { url = new URL(raw, document.baseURI); } catch (_) { return; }
|
|
4625
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
|
|
4626
|
+
url.hash = '';
|
|
4627
|
+
var key = url.href;
|
|
4628
|
+
var existing = proofCandidates[key];
|
|
4629
|
+
if (!existing || score > existing.score) {
|
|
4630
|
+
proofCandidates[key] = {
|
|
4631
|
+
url: key, proofType: proofType, score: score, confidence: 'low',
|
|
4632
|
+
evidence: Array.from(new Set(evidence)), alt: alt || null, context: context || null,
|
|
4633
|
+
width: width || null, height: height || null
|
|
4634
|
+
};
|
|
4635
|
+
}
|
|
4636
|
+
}
|
|
4637
|
+
|
|
4638
|
+
var imageEls = document.querySelectorAll('img');
|
|
4639
|
+
for (var li = 0; li < imageEls.length; li++) {
|
|
4640
|
+
var image = imageEls[li];
|
|
4641
|
+
var src = image.currentSrc || image.src || image.getAttribute('data-src') || image.getAttribute('data-lazy-src');
|
|
4642
|
+
if (!src) continue;
|
|
4643
|
+
var header = image.closest('header,[role="banner"]');
|
|
4644
|
+
var nav = image.closest('nav,[role="navigation"]');
|
|
4645
|
+
var footer = image.closest('footer,[role="contentinfo"]');
|
|
4646
|
+
var region = header ? 'header' : nav ? 'nav' : footer ? 'footer' : 'body';
|
|
4647
|
+
var evidence = [];
|
|
4648
|
+
var score = 0;
|
|
4649
|
+
if (header) { score += 35; evidence.push('header_or_banner'); }
|
|
4650
|
+
else if (nav) { score += 28; evidence.push('navigation'); }
|
|
4651
|
+
else if (footer) { score += 18; evidence.push('footer_or_contentinfo'); }
|
|
4652
|
+
var descriptor = [src, image.id, image.className, image.alt, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
|
|
4653
|
+
var identityDescriptor = [image.id, image.className, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
|
|
4654
|
+
if (/\\b(?:site[-_ ]?logo|brand[-_ ]?logo|custom-logo|navbar[-_ ]?brand|site[-_ ]?branding|site[-_ ]?identity|logo[-_ ]?link)\\b/i.test(identityDescriptor)) { score += 30; evidence.push('site_identity_marker'); }
|
|
4655
|
+
else if (/\\blogo\\b/i.test(descriptor)) { score += 12; evidence.push('generic_logo_marker'); }
|
|
4656
|
+
var anchor = image.closest('a[href]');
|
|
4657
|
+
if (anchor) {
|
|
4658
|
+
try {
|
|
4659
|
+
var destination = new URL(anchor.href, document.baseURI);
|
|
4660
|
+
if (destination.origin === location.origin && destination.pathname.replace(/\\/+$/, '') === '') {
|
|
4661
|
+
score += 22; evidence.push('links_to_homepage');
|
|
4662
|
+
}
|
|
4663
|
+
} catch (_) {}
|
|
4664
|
+
}
|
|
4665
|
+
if (isSameDomain(src)) { score += 8; evidence.push('same_domain'); }
|
|
4666
|
+
var style = cs(image);
|
|
4667
|
+
var visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity || '1') > 0;
|
|
4668
|
+
if (visible) { score += 5; evidence.push('visible'); }
|
|
4669
|
+
var width = image.naturalWidth || image.width || null;
|
|
4670
|
+
var height = image.naturalHeight || image.height || null;
|
|
4671
|
+
if (width && height && width >= 80 && height >= 24) { score += 5; evidence.push('plausible_logo_dimensions'); }
|
|
4672
|
+
if ((width && width < 32) || (height && height < 16)) { score -= 45; evidence.push('tiny_asset_penalty'); }
|
|
4673
|
+
if (/\\b(?:icon|avatar|profile|search|menu|hamburger|social|badge)\\b/i.test(descriptor)) { score -= 35; evidence.push('icon_or_avatar_penalty'); }
|
|
4674
|
+
addLogo(src, region, evidence, score, image.alt || null, width, height);
|
|
4675
|
+
|
|
4676
|
+
if (region === 'body' && visible && width && height && width >= 80 && height >= 40) {
|
|
4677
|
+
var context = nearestHeadingText(image);
|
|
4678
|
+
var proofDescriptor = [src, image.alt, image.getAttribute('title')].filter(Boolean).join(' ');
|
|
4679
|
+
var proofEvidence = [];
|
|
4680
|
+
var proofScore = 0;
|
|
4681
|
+
var commercialContext = !!context && /\\b(?:insurances?|payment|financing|we accept|accepted health)\\b/i.test(context);
|
|
4682
|
+
if (/\\b(?:logo|seal|trust[-_ ]?mark|certification[-_ ]?mark)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_asset_marker'); }
|
|
4683
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation)|certifi(?:ed|cation)|compliance|legitscript|hipaa|award|winner|best of|top[-_ ]?rated|member|membership|association|professionals?|press|featured|as seen|mentioned|partner|customer|client|trusted by|used by|works with)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_descriptor'); }
|
|
4684
|
+
if (context && /\\b(?:recognized|accredit(?:ed|ation)|certifi(?:ed|cation)|award|winner|member|membership|association|featured|as seen|mentioned|partner|customer|client|trusted|trust)\\b/i.test(context)) { proofScore += 25; proofEvidence.push('proof_section_context'); }
|
|
4685
|
+
if (commercialContext) { proofScore -= 100; proofEvidence.push('commercial_acceptance_context_exclusion'); }
|
|
4686
|
+
proofScore += 10; proofEvidence.push('proof_dimensions');
|
|
4687
|
+
proofScore += 5; proofEvidence.push('visible');
|
|
4688
|
+
if (isSameDomain(src)) { proofScore += 5; proofEvidence.push('same_domain'); }
|
|
4689
|
+
else { proofScore += 3; proofEvidence.push('external_asset_domain'); }
|
|
4690
|
+
if (!commercialContext && proofScore >= 45) addProof(src, proofTypeFor(proofDescriptor, context), proofScore, proofEvidence, image.alt || null, context, width, height);
|
|
4323
4691
|
}
|
|
4324
4692
|
}
|
|
4325
4693
|
var faviconEl = document.querySelector(
|
|
4326
4694
|
'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
|
|
4327
4695
|
);
|
|
4696
|
+
if (faviconEl && faviconEl.href) addLogo(faviconEl.href, 'favicon', ['favicon_fallback_only'], 10, null, null, null);
|
|
4697
|
+
var rankedLogos = Object.values(logoCandidates).filter(function(candidate) { return candidate.region !== 'body'; }).sort(function(a, b) { return b.score - a.score || a.url.localeCompare(b.url); }).slice(0, 20);
|
|
4698
|
+
for (var ri = 0; ri < rankedLogos.length; ri++) {
|
|
4699
|
+
rankedLogos[ri].confidence = rankedLogos[ri].score >= 85 ? 'high' : rankedLogos[ri].score >= 60 ? 'medium' : 'low';
|
|
4700
|
+
}
|
|
4701
|
+
var primaryLogo = rankedLogos.length && rankedLogos[0].score >= 55 && rankedLogos[0].region !== 'favicon' ? rankedLogos[0] : null;
|
|
4328
4702
|
return {
|
|
4329
|
-
navBg: navStyle ? navStyle.backgroundColor : null,
|
|
4330
|
-
bodyBg: bodyStyle ? bodyStyle.backgroundColor : null,
|
|
4331
|
-
bodyColor: bodyStyle ? bodyStyle.color : null,
|
|
4332
|
-
h1Color: h1Style ? h1Style.color : null,
|
|
4333
|
-
btnBg: btnStyle ? btnStyle.backgroundColor : null,
|
|
4703
|
+
navBg: navStyle ? toHex(navStyle.backgroundColor) : null,
|
|
4704
|
+
bodyBg: (bodyStyle ? toHex(bodyStyle.backgroundColor) : null) || (cs(document.documentElement) ? toHex(cs(document.documentElement).backgroundColor) : null) || '#ffffff',
|
|
4705
|
+
bodyColor: bodyStyle ? toHex(bodyStyle.color) : null,
|
|
4706
|
+
h1Color: h1Style ? toHex(h1Style.color) : null,
|
|
4707
|
+
btnBg: btnStyle ? toHex(btnStyle.backgroundColor) : null,
|
|
4708
|
+
btnDetail: btnEl ? String(btnEl.textContent || btnEl.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim().slice(0, 120) : null,
|
|
4334
4709
|
bodyFont: bodyStyle ? bodyStyle.fontFamily : null,
|
|
4335
4710
|
h1Font: h1Style ? h1Style.fontFamily : null,
|
|
4336
|
-
logoSrc:
|
|
4711
|
+
logoSrc: primaryLogo ? primaryLogo.url : null,
|
|
4712
|
+
logoConfidence: primaryLogo ? primaryLogo.confidence : null,
|
|
4713
|
+
logoSelectionReason: primaryLogo ? primaryLogo.evidence.join(', ') : null,
|
|
4714
|
+
logoCandidates: rankedLogos,
|
|
4715
|
+
proofCandidates: Object.values(proofCandidates),
|
|
4337
4716
|
faviconHref: faviconEl ? faviconEl.href : null,
|
|
4338
4717
|
svgFreq: svgFreq,
|
|
4339
4718
|
navChildBgFreq: navChildBgFreq,
|
|
4719
|
+
visibleBgFreq: visibleBgFreq,
|
|
4720
|
+
topBgFreq: topBgFreq,
|
|
4721
|
+
cssPrimaryCandidates: cssPrimaryCandidates,
|
|
4722
|
+
cssAccentCandidates: cssAccentCandidates,
|
|
4723
|
+
themeColor: themeColor,
|
|
4340
4724
|
};
|
|
4341
4725
|
})()
|
|
4342
4726
|
`;
|
|
4343
4727
|
const raw = await page.evaluate(evalScript);
|
|
4344
|
-
const
|
|
4345
|
-
const
|
|
4728
|
+
const logoSelection = selectRankedLogoCandidates(raw.logoCandidates ?? []);
|
|
4729
|
+
const navBgHex = raw.navBg;
|
|
4730
|
+
const bodyBgHex = raw.bodyBg;
|
|
4346
4731
|
const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
|
|
4347
|
-
const svgPrimary =
|
|
4732
|
+
const svgPrimary = dominantChromaticColor(raw.svgFreq ?? {});
|
|
4348
4733
|
const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
|
|
4349
|
-
const
|
|
4350
|
-
const
|
|
4351
|
-
const
|
|
4352
|
-
const
|
|
4734
|
+
const visibleBg = dominantColor(raw.visibleBgFreq ?? {});
|
|
4735
|
+
const topBg = dominantColor(raw.topBgFreq ?? {});
|
|
4736
|
+
const cssPrimary = raw.cssPrimaryCandidates?.[0] ?? null;
|
|
4737
|
+
const cssAccent = raw.cssAccentCandidates?.[0] ?? null;
|
|
4738
|
+
const themeColor = isChromatic(raw.themeColor) ? raw.themeColor : null;
|
|
4739
|
+
const ctaColor = isChromatic(raw.btnBg) ? raw.btnBg : null;
|
|
4740
|
+
const primaryChoice = cssPrimary ? { value: cssPrimary.hex, source: "css_variable", confidence: "high", detail: cssPrimary.name } : themeColor ? { value: themeColor, source: "theme_color", confidence: "high", detail: 'meta[name="theme-color"]' } : navBgUsable && isChromatic(navBgUsable) ? { value: navBgUsable, source: "navigation_background", confidence: "medium", detail: "nav, header, or banner computed background" } : topBg && isChromatic(topBg) ? { value: topBg, source: "navigation_background", confidence: "medium", detail: "dominant wide rendered background near the top of the page" } : svgPrimary && isChromatic(svgPrimary) ? { value: svgPrimary, source: "header_svg", confidence: "medium", detail: "dominant rendered identity SVG fill near the top of the page" } : navChildBg && isChromatic(navChildBg) ? { value: navChildBg, source: "navigation_background", confidence: "medium", detail: "dominant rendered navigation child background" } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : visibleBg && isChromatic(visibleBg) ? { value: visibleBg, source: "visible_background", confidence: "low", detail: "area-weighted rendered element background" } : bodyBgHex && !isTransparentOrWhite(bodyBgHex) ? { value: bodyBgHex, source: "body_background", confidence: "low", detail: "computed body background fallback" } : null;
|
|
4741
|
+
const accentChoice = cssAccent ? { value: cssAccent.hex, source: "css_variable", confidence: "high", detail: cssAccent.name } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : null;
|
|
4742
|
+
const primary = primaryChoice?.value ?? null;
|
|
4743
|
+
const accent = accentChoice?.value ?? null;
|
|
4744
|
+
const text2 = raw.bodyColor;
|
|
4745
|
+
const heading = raw.h1Color;
|
|
4353
4746
|
const background = bodyBgHex;
|
|
4354
4747
|
const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
|
|
4355
4748
|
return {
|
|
@@ -4361,22 +4754,208 @@ async function extractBrandingFromPage(page) {
|
|
|
4361
4754
|
text: text2 ?? null,
|
|
4362
4755
|
heading: heading ?? null
|
|
4363
4756
|
},
|
|
4757
|
+
colorEvidence: {
|
|
4758
|
+
primary: primaryChoice,
|
|
4759
|
+
accent: accentChoice,
|
|
4760
|
+
background: background ? { value: background, source: "body_background", confidence: "high", detail: "computed body background" } : null,
|
|
4761
|
+
text: text2 ? { value: text2, source: "body_text", confidence: "high", detail: "computed body text color" } : null,
|
|
4762
|
+
heading: heading ? { value: heading, source: "heading_text", confidence: "high", detail: "computed first H1 text color" } : null
|
|
4763
|
+
},
|
|
4364
4764
|
fonts: {
|
|
4365
4765
|
body: firstFont(raw.bodyFont ?? ""),
|
|
4366
4766
|
heading: firstFont(raw.h1Font ?? "")
|
|
4367
4767
|
},
|
|
4368
4768
|
assets: {
|
|
4369
|
-
logo:
|
|
4370
|
-
favicon: raw.faviconHref || null
|
|
4769
|
+
logo: logoSelection.primary?.url ?? null,
|
|
4770
|
+
favicon: raw.faviconHref || null,
|
|
4771
|
+
logoConfidence: logoSelection.primary?.confidence ?? null,
|
|
4772
|
+
logoSelectionReason: logoSelection.selectionReason,
|
|
4773
|
+
logoVariants: logoSelection.variants,
|
|
4774
|
+
logoCandidates: logoSelection.candidates,
|
|
4775
|
+
proofImages: selectProofImages(raw.proofCandidates ?? [])
|
|
4371
4776
|
}
|
|
4372
4777
|
};
|
|
4373
4778
|
}
|
|
4374
4779
|
|
|
4780
|
+
// src/lib/rendered-media.ts
|
|
4781
|
+
var SETTLE_MS = 450;
|
|
4782
|
+
var STABLE_CHECKS_REQUIRED = 3;
|
|
4783
|
+
var MAX_SCROLL_ROUNDS = 40;
|
|
4784
|
+
var MAX_DOM_NODES = 15e3;
|
|
4785
|
+
function mergeAsset(target, incoming) {
|
|
4786
|
+
const existing = target.get(incoming.url);
|
|
4787
|
+
if (!existing) {
|
|
4788
|
+
target.set(incoming.url, incoming);
|
|
4789
|
+
return;
|
|
4790
|
+
}
|
|
4791
|
+
existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
|
|
4792
|
+
existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
|
|
4793
|
+
existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
|
|
4794
|
+
if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
|
|
4795
|
+
existing.width = incoming.width;
|
|
4796
|
+
existing.height = incoming.height;
|
|
4797
|
+
}
|
|
4798
|
+
}
|
|
4799
|
+
async function readRenderedSnapshot(page) {
|
|
4800
|
+
await page.evaluate("globalThis.__name = globalThis.__name || ((target) => target)");
|
|
4801
|
+
return page.evaluate(({ maxDomNodes }) => {
|
|
4802
|
+
const rows = /* @__PURE__ */ new Map();
|
|
4803
|
+
const resolve2 = (raw) => {
|
|
4804
|
+
if (!raw || raw.startsWith("data:") || raw.startsWith("blob:")) return null;
|
|
4805
|
+
try {
|
|
4806
|
+
const parsed = new URL(raw, document.baseURI);
|
|
4807
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
4808
|
+
parsed.hash = "";
|
|
4809
|
+
return parsed.href;
|
|
4810
|
+
} catch {
|
|
4811
|
+
return null;
|
|
4812
|
+
}
|
|
4813
|
+
};
|
|
4814
|
+
const contextFor = (element) => {
|
|
4815
|
+
const contexts = [];
|
|
4816
|
+
const section2 = element.closest('section,article,main,header,footer,aside,[role="region"]');
|
|
4817
|
+
if (section2?.id) contexts.push(`#${section2.id}`);
|
|
4818
|
+
const labelled = section2?.getAttribute("aria-label")?.trim();
|
|
4819
|
+
if (labelled) contexts.push(labelled.slice(0, 200));
|
|
4820
|
+
const heading = section2?.querySelector("h1,h2,h3,h4,h5,h6")?.textContent?.replace(/\s+/g, " ").trim();
|
|
4821
|
+
if (heading) contexts.push(heading.slice(0, 200));
|
|
4822
|
+
return contexts;
|
|
4823
|
+
};
|
|
4824
|
+
const add = (raw, type, method, element, dimensions, alt) => {
|
|
4825
|
+
const url = resolve2(raw);
|
|
4826
|
+
if (!url) return;
|
|
4827
|
+
const existing = rows.get(url) ?? {
|
|
4828
|
+
url,
|
|
4829
|
+
type,
|
|
4830
|
+
discoveryMethods: [],
|
|
4831
|
+
altTexts: [],
|
|
4832
|
+
contexts: [],
|
|
4833
|
+
width: null,
|
|
4834
|
+
height: null
|
|
4835
|
+
};
|
|
4836
|
+
existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, method])];
|
|
4837
|
+
const cleanAlt = alt?.replace(/\s+/g, " ").trim();
|
|
4838
|
+
if (cleanAlt) existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, cleanAlt.slice(0, 500)])];
|
|
4839
|
+
existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...contextFor(element)])];
|
|
4840
|
+
if (dimensions && dimensions.width > 0 && dimensions.height > 0) {
|
|
4841
|
+
const currentArea = (existing.width ?? 0) * (existing.height ?? 0);
|
|
4842
|
+
if (dimensions.width * dimensions.height > currentArea) {
|
|
4843
|
+
existing.width = dimensions.width;
|
|
4844
|
+
existing.height = dimensions.height;
|
|
4845
|
+
}
|
|
4846
|
+
}
|
|
4847
|
+
rows.set(url, existing);
|
|
4848
|
+
};
|
|
4849
|
+
const srcsetUrls = (value) => (value ?? "").split(",").map((part) => part.trim().split(/\s+/)[0] ?? "").filter(Boolean);
|
|
4850
|
+
for (const img of Array.from(document.querySelectorAll("img"))) {
|
|
4851
|
+
const image = img;
|
|
4852
|
+
const dimensions = { width: image.naturalWidth, height: image.naturalHeight };
|
|
4853
|
+
add(image.currentSrc, "image", "rendered-current-src", image, dimensions, image.alt);
|
|
4854
|
+
add(image.getAttribute("src"), "image", "rendered-src", image, dimensions, image.alt);
|
|
4855
|
+
for (const attr of ["data-src", "data-lazy-src", "data-original", "data-bg", "data-background"]) {
|
|
4856
|
+
add(image.getAttribute(attr), "image", `rendered-${attr}`, image, dimensions, image.alt);
|
|
4857
|
+
}
|
|
4858
|
+
for (const url of srcsetUrls(image.getAttribute("srcset"))) add(url, "image", "rendered-srcset", image, dimensions, image.alt);
|
|
4859
|
+
const anchor = image.closest("a[href]");
|
|
4860
|
+
const anchorAttrs = anchor ? Array.from(anchor.attributes).map((attribute) => `${attribute.name}=${attribute.value}`).join(" ") : "";
|
|
4861
|
+
if (anchor && (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(anchor.href))) add(anchor.href, "image", "rendered-lightbox-target", image, void 0, image.alt);
|
|
4862
|
+
}
|
|
4863
|
+
for (const source of Array.from(document.querySelectorAll("source"))) {
|
|
4864
|
+
const parent = source.closest("picture,video,audio");
|
|
4865
|
+
const type = parent?.tagName === "VIDEO" ? "video" : parent?.tagName === "AUDIO" ? "audio" : "image";
|
|
4866
|
+
add(source.getAttribute("src"), type, "rendered-source", source);
|
|
4867
|
+
for (const url of srcsetUrls(source.getAttribute("srcset"))) add(url, type, "rendered-source-srcset", source);
|
|
4868
|
+
}
|
|
4869
|
+
for (const video of Array.from(document.querySelectorAll("video"))) {
|
|
4870
|
+
const media = video;
|
|
4871
|
+
add(media.currentSrc || media.src, "video", "rendered-video", media);
|
|
4872
|
+
add(media.poster, "image", "rendered-video-poster", media, { width: media.videoWidth, height: media.videoHeight });
|
|
4873
|
+
}
|
|
4874
|
+
for (const audio of Array.from(document.querySelectorAll("audio"))) {
|
|
4875
|
+
const media = audio;
|
|
4876
|
+
add(media.currentSrc || media.src, "audio", "rendered-audio", media);
|
|
4877
|
+
}
|
|
4878
|
+
for (const svgImage of Array.from(document.querySelectorAll("svg image"))) {
|
|
4879
|
+
add(svgImage.getAttribute("href") || svgImage.getAttribute("xlink:href"), "image", "rendered-svg-image", svgImage);
|
|
4880
|
+
}
|
|
4881
|
+
const all = Array.from(document.querySelectorAll("*")).slice(0, maxDomNodes);
|
|
4882
|
+
for (const element of all) {
|
|
4883
|
+
const background = getComputedStyle(element).backgroundImage;
|
|
4884
|
+
if (!background || background === "none") continue;
|
|
4885
|
+
for (const match of background.matchAll(/url\(["']?(.+?)["']?\)/gi)) {
|
|
4886
|
+
add(match[1], "image", "rendered-computed-background", element);
|
|
4887
|
+
}
|
|
4888
|
+
}
|
|
4889
|
+
const documentHeight = Math.max(document.body?.scrollHeight ?? 0, document.documentElement?.scrollHeight ?? 0);
|
|
4890
|
+
const viewportHeight = window.innerHeight;
|
|
4891
|
+
return {
|
|
4892
|
+
assets: [...rows.values()],
|
|
4893
|
+
documentHeight,
|
|
4894
|
+
scrollY: window.scrollY,
|
|
4895
|
+
viewportHeight,
|
|
4896
|
+
atBottom: window.scrollY + viewportHeight >= documentHeight - 5
|
|
4897
|
+
};
|
|
4898
|
+
}, { maxDomNodes: MAX_DOM_NODES });
|
|
4899
|
+
}
|
|
4900
|
+
async function collectRenderedPageMedia(page, maxAssets = 1e3) {
|
|
4901
|
+
const boundedMax = Math.max(1, Math.min(maxAssets, 2e3));
|
|
4902
|
+
await page.evaluate(() => {
|
|
4903
|
+
for (const image of Array.from(document.querySelectorAll('img[loading="lazy"]'))) image.loading = "eager";
|
|
4904
|
+
}).catch(() => void 0);
|
|
4905
|
+
const assets = /* @__PURE__ */ new Map();
|
|
4906
|
+
let stableChecks = 0;
|
|
4907
|
+
let previousHeight = -1;
|
|
4908
|
+
let rounds = 0;
|
|
4909
|
+
let documentHeight = 0;
|
|
4910
|
+
while (rounds < MAX_SCROLL_ROUNDS) {
|
|
4911
|
+
rounds += 1;
|
|
4912
|
+
const before = assets.size;
|
|
4913
|
+
const snapshot = await readRenderedSnapshot(page);
|
|
4914
|
+
documentHeight = snapshot.documentHeight;
|
|
4915
|
+
for (const asset of snapshot.assets) mergeAsset(assets, asset);
|
|
4916
|
+
if (assets.size >= boundedMax) {
|
|
4917
|
+
return {
|
|
4918
|
+
assets: [...assets.values()].slice(0, boundedMax),
|
|
4919
|
+
exhausted: false,
|
|
4920
|
+
stopReason: "asset_limit",
|
|
4921
|
+
scrollRounds: rounds,
|
|
4922
|
+
documentHeight
|
|
4923
|
+
};
|
|
4924
|
+
}
|
|
4925
|
+
const unchanged = snapshot.atBottom && before === assets.size && previousHeight === snapshot.documentHeight;
|
|
4926
|
+
stableChecks = unchanged ? stableChecks + 1 : 0;
|
|
4927
|
+
previousHeight = snapshot.documentHeight;
|
|
4928
|
+
if (stableChecks >= STABLE_CHECKS_REQUIRED) {
|
|
4929
|
+
await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
|
|
4930
|
+
return {
|
|
4931
|
+
assets: [...assets.values()],
|
|
4932
|
+
exhausted: true,
|
|
4933
|
+
stopReason: "page_exhausted",
|
|
4934
|
+
scrollRounds: rounds,
|
|
4935
|
+
documentHeight
|
|
4936
|
+
};
|
|
4937
|
+
}
|
|
4938
|
+
await page.evaluate(({ viewportHeight, documentHeight: height }) => {
|
|
4939
|
+
const next = Math.min(Math.max(0, height - viewportHeight), window.scrollY + Math.max(800, Math.floor(viewportHeight * 0.9)));
|
|
4940
|
+
window.scrollTo(0, next);
|
|
4941
|
+
}, { viewportHeight: snapshot.viewportHeight, documentHeight: snapshot.documentHeight });
|
|
4942
|
+
await page.waitForTimeout(SETTLE_MS);
|
|
4943
|
+
}
|
|
4944
|
+
await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
|
|
4945
|
+
return {
|
|
4946
|
+
assets: [...assets.values()],
|
|
4947
|
+
exhausted: false,
|
|
4948
|
+
stopReason: "scroll_round_limit",
|
|
4949
|
+
scrollRounds: rounds,
|
|
4950
|
+
documentHeight
|
|
4951
|
+
};
|
|
4952
|
+
}
|
|
4953
|
+
|
|
4375
4954
|
// src/lib/screenshot.ts
|
|
4376
4955
|
var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
|
|
4377
4956
|
var MOBILE_VIEWPORT = { width: 390, height: 844 };
|
|
4378
4957
|
var SCROLL_STEP = 900;
|
|
4379
|
-
var
|
|
4958
|
+
var SETTLE_MS2 = 400;
|
|
4380
4959
|
async function scrollAndStitch(page, device = "desktop") {
|
|
4381
4960
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
4382
4961
|
await page.setViewportSize(vp);
|
|
@@ -4392,7 +4971,7 @@ async function scrollAndStitch(page, device = "desktop") {
|
|
|
4392
4971
|
let scrollY = 0;
|
|
4393
4972
|
while (scrollY < totalH) {
|
|
4394
4973
|
await page.evaluate((y) => window.scrollTo(0, y), scrollY);
|
|
4395
|
-
await page.waitForTimeout(
|
|
4974
|
+
await page.waitForTimeout(SETTLE_MS2);
|
|
4396
4975
|
const remaining = totalH - scrollY;
|
|
4397
4976
|
const stripH = Math.min(SCROLL_STEP, remaining);
|
|
4398
4977
|
const buf = await page.screenshot({
|
|
@@ -4423,7 +5002,7 @@ async function scrollAndStitch(page, device = "desktop") {
|
|
|
4423
5002
|
);
|
|
4424
5003
|
return Buffer.from(base64, "base64");
|
|
4425
5004
|
}
|
|
4426
|
-
async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
5005
|
+
async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
4427
5006
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
4428
5007
|
const browser = await chromium2.launch({ headless: true });
|
|
4429
5008
|
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
@@ -4434,17 +5013,18 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
|
4434
5013
|
}, runtimeUserAgent));
|
|
4435
5014
|
const page = await context.newPage();
|
|
4436
5015
|
try {
|
|
4437
|
-
await page.goto(url, { waitUntil: "
|
|
5016
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
4438
5017
|
await page.waitForTimeout(1500);
|
|
4439
5018
|
const result = {};
|
|
4440
5019
|
if (wantBranding) result.branding = await extractBrandingFromPage(page);
|
|
5020
|
+
if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
|
|
4441
5021
|
if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
|
|
4442
5022
|
return result;
|
|
4443
5023
|
} finally {
|
|
4444
5024
|
await browser.close();
|
|
4445
5025
|
}
|
|
4446
5026
|
}
|
|
4447
|
-
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
|
|
5027
|
+
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
4448
5028
|
const kernel = new Kernel2({ apiKey: kernelApiKey });
|
|
4449
5029
|
const openedAtMs = Date.now();
|
|
4450
5030
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
@@ -4457,10 +5037,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
4457
5037
|
}, runtimeUserAgent));
|
|
4458
5038
|
const page = context.pages()[0] ?? await context.newPage();
|
|
4459
5039
|
try {
|
|
4460
|
-
await page.goto(url, { waitUntil: "
|
|
5040
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
4461
5041
|
await page.waitForTimeout(1500);
|
|
4462
5042
|
const result = {};
|
|
4463
5043
|
if (wantBranding) result.branding = await extractBrandingFromPage(page);
|
|
5044
|
+
if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
|
|
4464
5045
|
if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
|
|
4465
5046
|
return result;
|
|
4466
5047
|
} finally {
|
|
@@ -4480,12 +5061,14 @@ async function capturePageData(url, opts = {}) {
|
|
|
4480
5061
|
const device = opts.device ?? "desktop";
|
|
4481
5062
|
const wantShot = opts.screenshot ?? false;
|
|
4482
5063
|
const wantBranding = opts.branding ?? false;
|
|
5064
|
+
const wantMedia = opts.media ?? false;
|
|
5065
|
+
const maxMediaAssets = opts.maxMediaAssets ?? 1e3;
|
|
4483
5066
|
try {
|
|
4484
|
-
return await captureLocal(url, device, wantShot, wantBranding);
|
|
5067
|
+
return await captureLocal(url, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
4485
5068
|
} catch (localErr) {
|
|
4486
5069
|
const msg = localErr instanceof Error ? localErr.message : String(localErr);
|
|
4487
5070
|
if (!opts.kernelApiKey) throw new Error(`Local capture failed (${msg}) and no headless browser backend is configured`);
|
|
4488
|
-
return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
|
|
5071
|
+
return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
4489
5072
|
}
|
|
4490
5073
|
}
|
|
4491
5074
|
|
|
@@ -4890,7 +5473,7 @@ function withTimeout(p, ms) {
|
|
|
4890
5473
|
async function createBrowserWithRetry(client2, headless) {
|
|
4891
5474
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
4892
5475
|
try {
|
|
4893
|
-
return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
|
|
5476
|
+
return await client2.browsers.create({ stealth: true, ...headless ? { headless: true } : {}, timeout_seconds: 120 });
|
|
4894
5477
|
} catch (err) {
|
|
4895
5478
|
if (attempt === 2) throw err;
|
|
4896
5479
|
await new Promise((r) => setTimeout(r, 5e3));
|
|
@@ -4899,7 +5482,7 @@ async function createBrowserWithRetry(client2, headless) {
|
|
|
4899
5482
|
throw new Error("unreachable");
|
|
4900
5483
|
}
|
|
4901
5484
|
var PAGE_HARD_TIMEOUT_MS = 35e3;
|
|
4902
|
-
async function crawlPage(context, url) {
|
|
5485
|
+
async function crawlPage(context, url, headless) {
|
|
4903
5486
|
const startedAt = Date.now();
|
|
4904
5487
|
let page = null;
|
|
4905
5488
|
let guardState = null;
|
|
@@ -4908,13 +5491,38 @@ async function crawlPage(context, url) {
|
|
|
4908
5491
|
guardState = await installBrowserRequestGuard(page, url);
|
|
4909
5492
|
const work = (async (p) => {
|
|
4910
5493
|
const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
4911
|
-
|
|
5494
|
+
let html = await p.evaluate((maxBytes) => {
|
|
4912
5495
|
const value = document.documentElement?.outerHTML ?? "";
|
|
4913
5496
|
if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
|
|
4914
5497
|
return value;
|
|
4915
5498
|
}, MAX_PAGE_HTML_BYTES);
|
|
4916
5499
|
const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
|
|
4917
5500
|
const finalUrl = p.url();
|
|
5501
|
+
let challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
|
|
5502
|
+
if (challenge.detected && !headless) {
|
|
5503
|
+
const deadline = Date.now() + 2e4;
|
|
5504
|
+
while (challenge.detected && Date.now() < deadline) {
|
|
5505
|
+
await p.waitForTimeout(1e3);
|
|
5506
|
+
html = await p.content();
|
|
5507
|
+
challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
|
|
5508
|
+
}
|
|
5509
|
+
}
|
|
5510
|
+
if (challenge.detected) {
|
|
5511
|
+
return {
|
|
5512
|
+
url,
|
|
5513
|
+
html: null,
|
|
5514
|
+
status: resp?.status() ?? 0,
|
|
5515
|
+
via: "browser",
|
|
5516
|
+
headers,
|
|
5517
|
+
responseTimeMs: Date.now() - startedAt,
|
|
5518
|
+
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
|
|
5519
|
+
errorCode: "browser_challenge_unresolved",
|
|
5520
|
+
error: `Browser challenge remained unresolved (${challenge.reason ?? "unknown challenge"}).`,
|
|
5521
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5522
|
+
challengeDetected: true,
|
|
5523
|
+
stopReason: "challenge_unresolved"
|
|
5524
|
+
};
|
|
5525
|
+
}
|
|
4918
5526
|
return {
|
|
4919
5527
|
url,
|
|
4920
5528
|
html,
|
|
@@ -4922,7 +5530,10 @@ async function crawlPage(context, url) {
|
|
|
4922
5530
|
via: "browser",
|
|
4923
5531
|
headers,
|
|
4924
5532
|
responseTimeMs: Date.now() - startedAt,
|
|
4925
|
-
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null
|
|
5533
|
+
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
|
|
5534
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5535
|
+
challengeDetected: false,
|
|
5536
|
+
stopReason: resp && (resp.status() < 200 || resp.status() >= 300) ? "http_error" : "usable_page"
|
|
4926
5537
|
};
|
|
4927
5538
|
})(page);
|
|
4928
5539
|
const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
|
|
@@ -4937,7 +5548,10 @@ async function crawlPage(context, url) {
|
|
|
4937
5548
|
via: "browser",
|
|
4938
5549
|
responseTimeMs: Date.now() - startedAt,
|
|
4939
5550
|
errorCode,
|
|
4940
|
-
error: message.slice(0, 500)
|
|
5551
|
+
error: message.slice(0, 500),
|
|
5552
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5553
|
+
challengeDetected: false,
|
|
5554
|
+
stopReason: "navigation_failed"
|
|
4941
5555
|
};
|
|
4942
5556
|
} finally {
|
|
4943
5557
|
if (page) await page.close().catch(() => {
|
|
@@ -4951,7 +5565,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4951
5565
|
const headless = opts.headless ?? DEFAULT_HEADLESS;
|
|
4952
5566
|
const uniqueUrls = [...new Set(urls)];
|
|
4953
5567
|
const byUrl = /* @__PURE__ */ new Map();
|
|
4954
|
-
const runPass = async (targets, phase) => {
|
|
5568
|
+
const runPass = async (targets, phase, passHeadless) => {
|
|
4955
5569
|
if (targets.length === 0) return;
|
|
4956
5570
|
let next = 0;
|
|
4957
5571
|
let done = 0;
|
|
@@ -4974,7 +5588,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4974
5588
|
const closedAtMs = Date.now();
|
|
4975
5589
|
await client2.browsers.deleteByID(k.session_id).catch(() => {
|
|
4976
5590
|
});
|
|
4977
|
-
await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent:
|
|
5591
|
+
await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: passHeadless ? true : null, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, fallback: phase === "retry", openedAtMs, closedAtMs });
|
|
4978
5592
|
}
|
|
4979
5593
|
};
|
|
4980
5594
|
try {
|
|
@@ -4986,7 +5600,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4986
5600
|
try {
|
|
4987
5601
|
if (!browser || inBrowser >= urlsPerBrowser) {
|
|
4988
5602
|
await retire();
|
|
4989
|
-
kb = await createBrowserWithRetry(client2,
|
|
5603
|
+
kb = await createBrowserWithRetry(client2, passHeadless);
|
|
4990
5604
|
kbOpenedAtMs = Date.now();
|
|
4991
5605
|
browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
|
|
4992
5606
|
runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
@@ -4994,7 +5608,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4994
5608
|
}
|
|
4995
5609
|
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
4996
5610
|
try {
|
|
4997
|
-
r = await crawlPage(context, url);
|
|
5611
|
+
r = await crawlPage(context, url, passHeadless);
|
|
4998
5612
|
} finally {
|
|
4999
5613
|
await context.close().catch(() => {
|
|
5000
5614
|
});
|
|
@@ -5008,7 +5622,10 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5008
5622
|
status: 0,
|
|
5009
5623
|
via: "browser",
|
|
5010
5624
|
errorCode: "browser_session_failed",
|
|
5011
|
-
error: message.slice(0, 500)
|
|
5625
|
+
error: message.slice(0, 500),
|
|
5626
|
+
browserMode: passHeadless ? "stealth_headless" : "stealth_headful",
|
|
5627
|
+
challengeDetected: false,
|
|
5628
|
+
stopReason: "navigation_failed"
|
|
5012
5629
|
};
|
|
5013
5630
|
await retire();
|
|
5014
5631
|
}
|
|
@@ -5032,12 +5649,12 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5032
5649
|
};
|
|
5033
5650
|
await Promise.all(Array.from({ length: workers }, () => worker()));
|
|
5034
5651
|
};
|
|
5035
|
-
await runPass(uniqueUrls, "main");
|
|
5652
|
+
await runPass(uniqueUrls, "main", headless);
|
|
5036
5653
|
const failed = uniqueUrls.filter((u) => {
|
|
5037
5654
|
const r = byUrl.get(u);
|
|
5038
5655
|
return !r || !r.html || r.status < 200 || r.status >= 300;
|
|
5039
5656
|
});
|
|
5040
|
-
if (failed.length > 0) await runPass(failed, "retry");
|
|
5657
|
+
if (failed.length > 0) await runPass(failed, "retry", headless ? false : headless);
|
|
5041
5658
|
return urls.map((u) => byUrl.get(u) ?? {
|
|
5042
5659
|
url: u,
|
|
5043
5660
|
html: null,
|
|
@@ -5749,8 +6366,6 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5749
6366
|
if (text2) headings.push({ level: Number(m[1]), text: text2.slice(0, 200) });
|
|
5750
6367
|
}
|
|
5751
6368
|
const h1 = headings.find((h) => h.level === 1)?.text ?? null;
|
|
5752
|
-
const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
5753
|
-
const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
|
|
5754
6369
|
const schemaTypes = [];
|
|
5755
6370
|
let schemaTypeBytes = 0;
|
|
5756
6371
|
const addSchemaType = (value) => {
|
|
@@ -5807,11 +6422,14 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5807
6422
|
}
|
|
5808
6423
|
}
|
|
5809
6424
|
const mainHtml = (html.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? html).replace(/<(script|style|nav|header|footer|aside|noscript|svg|iframe)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
6425
|
+
const mainText = mainHtml.replace(/<[^>]+>/g, " ").replace(/&(?:nbsp|#160);/gi, " ").replace(/&(?:amp|#38);/gi, "&").replace(/&#(?:x[0-9a-f]+|\d+);/gi, " ").replace(/\s+/g, " ").trim();
|
|
6426
|
+
const mainWords = mainText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
|
|
6427
|
+
const wordCount2 = mainWords.filter((word) => word.length > 2).length;
|
|
5810
6428
|
let bodyMarkdown = "";
|
|
5811
6429
|
try {
|
|
5812
6430
|
bodyMarkdown = turndown.turndown(mainHtml).replace(/\n{3,}/g, "\n\n").trim().slice(0, MAX_PAGE_MARKDOWN);
|
|
5813
6431
|
} catch {
|
|
5814
|
-
bodyMarkdown =
|
|
6432
|
+
bodyMarkdown = mainText.slice(0, MAX_PAGE_MARKDOWN);
|
|
5815
6433
|
}
|
|
5816
6434
|
const outlinks = [];
|
|
5817
6435
|
const discoveryLinkSet = /* @__PURE__ */ new Set();
|
|
@@ -5839,10 +6457,11 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5839
6457
|
const internal = abs.origin === origin;
|
|
5840
6458
|
if (internal) {
|
|
5841
6459
|
internalLinks++;
|
|
5842
|
-
|
|
5843
|
-
|
|
6460
|
+
const crawlable = normalizeUrl(abs.href, documentUrl);
|
|
6461
|
+
if (crawlable && !discoveryLinkSet.has(crawlable)) {
|
|
6462
|
+
const bytes2 = Buffer.byteLength(crawlable);
|
|
5844
6463
|
if (discoveryLinkSet.size < PAGE_DATA_LIMITS.discoveryLinks && discoveryBytes + bytes2 <= PAGE_DATA_LIMITS.discoveryBytes) {
|
|
5845
|
-
discoveryLinkSet.add(
|
|
6464
|
+
discoveryLinkSet.add(crawlable);
|
|
5846
6465
|
discoveryBytes += bytes2;
|
|
5847
6466
|
} else truncatedFields.add("discoveryLinks");
|
|
5848
6467
|
}
|
|
@@ -5864,8 +6483,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5864
6483
|
const h1List = headings.filter((h) => h.level === 1);
|
|
5865
6484
|
const h1_2 = h1List[1]?.text ?? null;
|
|
5866
6485
|
const h2Count = headings.filter((h) => h.level === 2).length;
|
|
5867
|
-
const textRatio = Number((
|
|
5868
|
-
const contentHash = createHash("sha1").update(
|
|
6486
|
+
const textRatio = Number((mainText.length / Math.max(1, html.length)).toFixed(3));
|
|
6487
|
+
const contentHash = createHash("sha1").update(mainText).digest("hex");
|
|
5869
6488
|
const metaRobots = boundedText(html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1], 512)?.toLowerCase() ?? null;
|
|
5870
6489
|
const relNext = boundedUrl(html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]);
|
|
5871
6490
|
const relPrev = boundedUrl(html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]);
|
|
@@ -5919,11 +6538,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5919
6538
|
}
|
|
5920
6539
|
}
|
|
5921
6540
|
}
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
addImageLink(new URL(og.image, documentUrl).href);
|
|
5925
|
-
} catch {
|
|
5926
|
-
}
|
|
6541
|
+
for (const asset of discoverStaticMedia(html, documentUrl)) {
|
|
6542
|
+
if (asset.type === "image") addImageLink(asset.url);
|
|
5927
6543
|
}
|
|
5928
6544
|
const imageLinks = [...imageLinkSet];
|
|
5929
6545
|
const canonNorm = (u) => {
|
|
@@ -5950,12 +6566,19 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5950
6566
|
indexable = false;
|
|
5951
6567
|
indexabilityReason = "canonicalised";
|
|
5952
6568
|
}
|
|
5953
|
-
const
|
|
6569
|
+
const bodyClass = html.match(/<body\b[^>]*\bclass\s*=\s*["']([^"']*)["']/i)?.[1] ?? "";
|
|
6570
|
+
const wordpressAttachment = /(?:^|\s)(?:attachment|attachment-template-default)(?:\s|$)/i.test(bodyClass);
|
|
6571
|
+
const meaningfulText = mainText.replace(/https?:\/\/\S+/gi, " ").replace(/\s+/g, " ").trim();
|
|
6572
|
+
const meaningfulWords = meaningfulText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
|
|
6573
|
+
const hasMeaningfulText = meaningfulText.length >= 24 && meaningfulWords.length >= 4;
|
|
6574
|
+
const extractionSucceeded = !wordpressAttachment && bodyMarkdown.trim().length > 0 && hasMeaningfulText;
|
|
6575
|
+
const failureCode = wordpressAttachment ? "wordpress_attachment_page" : bodyMarkdown.trim().length === 0 ? "empty_rendered_content" : "insufficient_text_content";
|
|
6576
|
+
const failureReason = wordpressAttachment ? "The HTML is a WordPress media attachment page, not a text-content page." : bodyMarkdown.trim().length === 0 ? "The HTML response produced no usable page content." : `The main content contained only ${meaningfulWords.length} word(s) and ${meaningfulText.length} text character(s).`;
|
|
5954
6577
|
if (!extractionSucceeded) {
|
|
5955
6578
|
indexable = false;
|
|
5956
6579
|
indexabilityReason = "extraction-failed";
|
|
5957
6580
|
}
|
|
5958
|
-
const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode:
|
|
6581
|
+
const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: failureCode, error: failureReason } : attempt);
|
|
5959
6582
|
return {
|
|
5960
6583
|
url,
|
|
5961
6584
|
status,
|
|
@@ -6002,9 +6625,12 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6002
6625
|
discoveryLinks,
|
|
6003
6626
|
bodyMarkdown,
|
|
6004
6627
|
schema,
|
|
6628
|
+
contentKind: extractionSucceeded ? "text_page" : "insufficient_text",
|
|
6629
|
+
mainTextChars: meaningfulText.length,
|
|
6630
|
+
mainWordCount: meaningfulWords.length,
|
|
6005
6631
|
extractionStatus: extractionSucceeded ? "successful" : "failed",
|
|
6006
|
-
failureCode: extractionSucceeded ? null :
|
|
6007
|
-
failureReason: extractionSucceeded ? null :
|
|
6632
|
+
failureCode: extractionSucceeded ? null : failureCode,
|
|
6633
|
+
failureReason: extractionSucceeded ? null : failureReason,
|
|
6008
6634
|
fetchAttempts: normalizedAttempts,
|
|
6009
6635
|
truncatedFields: [...truncatedFields]
|
|
6010
6636
|
};
|
|
@@ -6092,6 +6718,7 @@ async function fetchPagePlain(url) {
|
|
|
6092
6718
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "non_html_response", `Plain fetch returned ${contentType}, not HTML.`);
|
|
6093
6719
|
const page = emptyPageData(url, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6094
6720
|
page.contentType = contentType;
|
|
6721
|
+
page.contentKind = "non_html";
|
|
6095
6722
|
return page;
|
|
6096
6723
|
}
|
|
6097
6724
|
const html = await readResponseTextWithLimit(res, MAX_PAGE_HTML_BYTES, "Page HTML");
|
|
@@ -6134,9 +6761,24 @@ async function mapWithConcurrency(items, concurrency, fn) {
|
|
|
6134
6761
|
}
|
|
6135
6762
|
async function extractPagesRotating(urls, opts) {
|
|
6136
6763
|
const uniqueUrls = [...new Set(urls)];
|
|
6137
|
-
const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY,
|
|
6764
|
+
const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
|
|
6765
|
+
const checked = await validatePublicHttpUrl(url, { field: "page URL" });
|
|
6766
|
+
if (checked.error || !checked.parsed) return fetchPagePlain(url);
|
|
6767
|
+
if (normalizeUrl(checked.parsed.href, checked.parsed.href)) return fetchPagePlain(url);
|
|
6768
|
+
const attempt = failedAttempt("fetch", null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
|
|
6769
|
+
const page = emptyPageData(url, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
|
|
6770
|
+
page.contentKind = "excluded_url";
|
|
6771
|
+
return page;
|
|
6772
|
+
});
|
|
6138
6773
|
const byUrl = new Map(plainPages.map((page) => [page.url, page]));
|
|
6139
|
-
const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
|
|
6774
|
+
const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
|
|
6775
|
+
"unsafe_url",
|
|
6776
|
+
"unsafe_redirect",
|
|
6777
|
+
"cross_site_redirect",
|
|
6778
|
+
"non_html_response",
|
|
6779
|
+
"non_page_url",
|
|
6780
|
+
"wordpress_attachment_page"
|
|
6781
|
+
].includes(page.failureCode ?? "")).map((page) => page.url);
|
|
6140
6782
|
if (opts.kernelApiKey && browserTargets.length > 0) {
|
|
6141
6783
|
const fetched = await crawlWithRotation(browserTargets, {
|
|
6142
6784
|
apiKey: opts.kernelApiKey,
|
|
@@ -8740,7 +9382,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8740
9382
|
});
|
|
8741
9383
|
}) : null;
|
|
8742
9384
|
const finalized = await step.run("finalize", async () => {
|
|
8743
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9385
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
|
|
8744
9386
|
const current = await getExtractJob(jobId2);
|
|
8745
9387
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8746
9388
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10071,8 +10713,8 @@ async function saveDirectoryCsvLocal(result) {
|
|
|
10071
10713
|
const outDir = join3(outputBaseDir(), "directory-workflows");
|
|
10072
10714
|
await mkdir(outDir, { recursive: true });
|
|
10073
10715
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
10074
|
-
const
|
|
10075
|
-
const path5 = join3(outDir, `${stamp}-${
|
|
10716
|
+
const slug4 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
10717
|
+
const path5 = join3(outDir, `${stamp}-${slug4}-directory-workflow.csv`);
|
|
10076
10718
|
await writeFile(path5, renderDirectoryWorkflowCsv(result), "utf8");
|
|
10077
10719
|
return path5;
|
|
10078
10720
|
}
|
|
@@ -12214,9 +12856,9 @@ async function listAllPublicLocalSourcebook(category) {
|
|
|
12214
12856
|
}) : await getDb().execute(`SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') ORDER BY s.category, s.state, s.business_name`);
|
|
12215
12857
|
return result.rows.map((row) => parseJson2(row.payload_json));
|
|
12216
12858
|
}
|
|
12217
|
-
async function getPublicLocalSourcebook(category, state,
|
|
12859
|
+
async function getPublicLocalSourcebook(category, state, slug4) {
|
|
12218
12860
|
await ensureLocalSourcebookSchema();
|
|
12219
|
-
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state,
|
|
12861
|
+
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
|
|
12220
12862
|
return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
|
|
12221
12863
|
}
|
|
12222
12864
|
|
|
@@ -12520,6 +13162,235 @@ var MapsReviewCollector = class {
|
|
|
12520
13162
|
}
|
|
12521
13163
|
};
|
|
12522
13164
|
|
|
13165
|
+
// src/extractor/MapsImageCollector.ts
|
|
13166
|
+
var QUIESCENCE_MS = 700;
|
|
13167
|
+
var STABLE_CHECKS_REQUIRED2 = 3;
|
|
13168
|
+
var MAX_SCROLL_ROUNDS2 = 80;
|
|
13169
|
+
var OWNER_DISCOVERY_LIMIT = 500;
|
|
13170
|
+
function mapsImageIdentity(rawUrl) {
|
|
13171
|
+
try {
|
|
13172
|
+
const url = new URL(rawUrl);
|
|
13173
|
+
const path5 = url.hostname.toLowerCase().endsWith("googleusercontent.com") ? url.pathname.replace(/=[^/]+$/, "") : url.pathname;
|
|
13174
|
+
return `${url.hostname.toLowerCase()}${path5}`;
|
|
13175
|
+
} catch {
|
|
13176
|
+
return rawUrl.split("=")[0] ?? rawUrl;
|
|
13177
|
+
}
|
|
13178
|
+
}
|
|
13179
|
+
function isMapsPhotoUrl(rawUrl) {
|
|
13180
|
+
try {
|
|
13181
|
+
const host2 = new URL(rawUrl).hostname.toLowerCase().replace(/\.$/, "");
|
|
13182
|
+
return host2 === "googleusercontent.com" || host2.endsWith(".googleusercontent.com") || host2 === "streetviewpixels-pa.googleapis.com";
|
|
13183
|
+
} catch {
|
|
13184
|
+
return false;
|
|
13185
|
+
}
|
|
13186
|
+
}
|
|
13187
|
+
async function openPhotoGallery(page) {
|
|
13188
|
+
const seePhotos = page.getByRole("button", { name: /^see photos$/i }).first();
|
|
13189
|
+
const exact = await seePhotos.count() > 0 ? seePhotos : page.locator('[aria-label^="See photos" i], button[aria-label^="Photo of" i]').first();
|
|
13190
|
+
if (await exact.count() === 0) {
|
|
13191
|
+
return false;
|
|
13192
|
+
}
|
|
13193
|
+
const clicked = await exact.click({ timeout: 5e3 }).then(() => true, () => false);
|
|
13194
|
+
if (!clicked) return false;
|
|
13195
|
+
const galleryControl = page.locator('button[aria-label="By owner" i], [role="tab"][aria-label*="owner" i]').first();
|
|
13196
|
+
await galleryControl.waitFor({ state: "visible", timeout: 5e3 }).catch(() => void 0);
|
|
13197
|
+
return galleryControl.isVisible().catch(() => false);
|
|
13198
|
+
}
|
|
13199
|
+
async function selectGalleryTab(page, scope) {
|
|
13200
|
+
const matcher = scope === "owner" ? /by owner/i : /^all(?:\s|$)/i;
|
|
13201
|
+
const direct = page.locator(scope === "owner" ? 'button[aria-label="By owner" i]' : 'button[aria-label="All" i]');
|
|
13202
|
+
const roleTab = page.getByRole("tab", { name: matcher });
|
|
13203
|
+
const tabs = await direct.count() > 0 ? direct : roleTab;
|
|
13204
|
+
if (await tabs.count() === 0) return false;
|
|
13205
|
+
const visible = tabs.filter({ visible: true }).first();
|
|
13206
|
+
if (await visible.count() === 0) return false;
|
|
13207
|
+
const clicked = await visible.click({ timeout: 5e3 }).then(() => true, () => false);
|
|
13208
|
+
if (!clicked) return false;
|
|
13209
|
+
await page.waitForTimeout(QUIESCENCE_MS);
|
|
13210
|
+
return true;
|
|
13211
|
+
}
|
|
13212
|
+
async function readAndAdvanceGallery(page) {
|
|
13213
|
+
return page.evaluate(() => {
|
|
13214
|
+
const mediaNodes = Array.from(document.querySelectorAll('[aria-label^="Photo " i], [aria-label^="Video " i]'));
|
|
13215
|
+
const imageUrl = (root) => {
|
|
13216
|
+
const candidates = [root, ...Array.from(root.querySelectorAll("*"))];
|
|
13217
|
+
for (const node of candidates) {
|
|
13218
|
+
if (node instanceof HTMLImageElement) {
|
|
13219
|
+
const value = node.currentSrc || node.src;
|
|
13220
|
+
if (value) return value;
|
|
13221
|
+
}
|
|
13222
|
+
const inline = node.style.backgroundImage;
|
|
13223
|
+
const computed = inline || window.getComputedStyle(node).backgroundImage;
|
|
13224
|
+
const match = computed?.match(/url\(["']?(.+?)["']?\)/i);
|
|
13225
|
+
if (match?.[1]) return match[1];
|
|
13226
|
+
}
|
|
13227
|
+
return null;
|
|
13228
|
+
};
|
|
13229
|
+
const items = mediaNodes.map((node) => {
|
|
13230
|
+
const label = node.getAttribute("aria-label") ?? "";
|
|
13231
|
+
const match = label.match(/^(Photo|Video)\s+(\d+)/i);
|
|
13232
|
+
return {
|
|
13233
|
+
kind: match?.[1]?.toLowerCase() === "video" ? "video" : "photo",
|
|
13234
|
+
position: match?.[2] ? Number(match[2]) : null,
|
|
13235
|
+
url: imageUrl(node)
|
|
13236
|
+
};
|
|
13237
|
+
});
|
|
13238
|
+
const first = mediaNodes[0] ?? null;
|
|
13239
|
+
const ancestors = [];
|
|
13240
|
+
let current = first?.parentElement ?? null;
|
|
13241
|
+
while (current) {
|
|
13242
|
+
if (current.scrollHeight > current.clientHeight + 40) ancestors.push(current);
|
|
13243
|
+
current = current.parentElement;
|
|
13244
|
+
}
|
|
13245
|
+
const fallback = Array.from(document.querySelectorAll("div")).filter((node) => {
|
|
13246
|
+
const style = window.getComputedStyle(node);
|
|
13247
|
+
return /(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight + 40 && mediaNodes.some((media) => node.contains(media));
|
|
13248
|
+
});
|
|
13249
|
+
const preferred = Array.from(document.querySelectorAll("div.m6QErb.DxyBCb.kA9KIf.dS8AEf")).find((node) => mediaNodes.some((media) => node.contains(media)) && node.scrollHeight > node.clientHeight + 40);
|
|
13250
|
+
const scroller = preferred ?? [...ancestors, ...fallback].sort((a, b) => b.scrollHeight - a.scrollHeight)[0] ?? null;
|
|
13251
|
+
const atBottom = scroller ? scroller.scrollTop + scroller.clientHeight >= scroller.scrollHeight - 5 : true;
|
|
13252
|
+
const metrics = {
|
|
13253
|
+
scrollHeight: scroller?.scrollHeight ?? 0,
|
|
13254
|
+
scrollTop: scroller?.scrollTop ?? 0,
|
|
13255
|
+
clientHeight: scroller?.clientHeight ?? 0,
|
|
13256
|
+
atBottom
|
|
13257
|
+
};
|
|
13258
|
+
if (scroller && !atBottom) {
|
|
13259
|
+
scroller.scrollTop = Math.min(
|
|
13260
|
+
scroller.scrollHeight - scroller.clientHeight,
|
|
13261
|
+
scroller.scrollTop + Math.max(1200, Math.floor(scroller.clientHeight * 1.75))
|
|
13262
|
+
);
|
|
13263
|
+
}
|
|
13264
|
+
return { items, ...metrics };
|
|
13265
|
+
});
|
|
13266
|
+
}
|
|
13267
|
+
async function collectGalleryPass(page, limit) {
|
|
13268
|
+
const photos = /* @__PURE__ */ new Map();
|
|
13269
|
+
const videos = /* @__PURE__ */ new Set();
|
|
13270
|
+
let stableChecks = 0;
|
|
13271
|
+
let previousHeight = -1;
|
|
13272
|
+
let rounds = 0;
|
|
13273
|
+
while (rounds < MAX_SCROLL_ROUNDS2) {
|
|
13274
|
+
rounds += 1;
|
|
13275
|
+
const before = photos.size + videos.size;
|
|
13276
|
+
const snapshot = await readAndAdvanceGallery(page);
|
|
13277
|
+
for (const item of snapshot.items) {
|
|
13278
|
+
if (item.kind === "video") {
|
|
13279
|
+
videos.add(item.url ? mapsImageIdentity(item.url) : `position:${item.position ?? videos.size}`);
|
|
13280
|
+
continue;
|
|
13281
|
+
}
|
|
13282
|
+
if (!item.url || !isMapsPhotoUrl(item.url)) continue;
|
|
13283
|
+
const key = mapsImageIdentity(item.url);
|
|
13284
|
+
const existing = photos.get(key);
|
|
13285
|
+
if (!existing || (item.position ?? Infinity) < (existing.position ?? Infinity)) photos.set(key, item);
|
|
13286
|
+
}
|
|
13287
|
+
if (photos.size >= limit) {
|
|
13288
|
+
return {
|
|
13289
|
+
items: [...photos.values()].slice(0, limit),
|
|
13290
|
+
exhausted: false,
|
|
13291
|
+
stopReason: "limit_reached",
|
|
13292
|
+
photosDiscovered: photos.size
|
|
13293
|
+
};
|
|
13294
|
+
}
|
|
13295
|
+
await page.waitForTimeout(QUIESCENCE_MS);
|
|
13296
|
+
const unchanged = snapshot.atBottom && before === photos.size + videos.size && previousHeight === snapshot.scrollHeight;
|
|
13297
|
+
stableChecks = unchanged ? stableChecks + 1 : 0;
|
|
13298
|
+
previousHeight = snapshot.scrollHeight;
|
|
13299
|
+
if (stableChecks >= STABLE_CHECKS_REQUIRED2) {
|
|
13300
|
+
return {
|
|
13301
|
+
items: [...photos.values()],
|
|
13302
|
+
exhausted: true,
|
|
13303
|
+
stopReason: "gallery_exhausted",
|
|
13304
|
+
photosDiscovered: photos.size
|
|
13305
|
+
};
|
|
13306
|
+
}
|
|
13307
|
+
}
|
|
13308
|
+
return {
|
|
13309
|
+
items: [...photos.values()],
|
|
13310
|
+
exhausted: false,
|
|
13311
|
+
stopReason: "scroll_round_limit",
|
|
13312
|
+
photosDiscovered: photos.size
|
|
13313
|
+
};
|
|
13314
|
+
}
|
|
13315
|
+
var MapsImageCollector = class {
|
|
13316
|
+
constructor(page) {
|
|
13317
|
+
this.page = page;
|
|
13318
|
+
}
|
|
13319
|
+
page;
|
|
13320
|
+
async collect(scope, maxImages) {
|
|
13321
|
+
if (!await openPhotoGallery(this.page)) {
|
|
13322
|
+
return {
|
|
13323
|
+
images: [],
|
|
13324
|
+
scope,
|
|
13325
|
+
exhausted: false,
|
|
13326
|
+
stopReason: "gallery_unavailable",
|
|
13327
|
+
ownerGalleryAvailable: false,
|
|
13328
|
+
ownerGalleryExhausted: false,
|
|
13329
|
+
ownerPhotosDiscovered: 0,
|
|
13330
|
+
allPhotosDiscovered: 0
|
|
13331
|
+
};
|
|
13332
|
+
}
|
|
13333
|
+
const ownerGalleryAvailable = await selectGalleryTab(this.page, "owner");
|
|
13334
|
+
const ownerPass = ownerGalleryAvailable ? await collectGalleryPass(this.page, scope === "owner" ? maxImages : OWNER_DISCOVERY_LIMIT) : { items: [], exhausted: false, stopReason: "gallery_unavailable", photosDiscovered: 0 };
|
|
13335
|
+
const ownerKeys = new Set(ownerPass.items.flatMap((item) => item.url ? [mapsImageIdentity(item.url)] : []));
|
|
13336
|
+
if (scope === "owner") {
|
|
13337
|
+
return {
|
|
13338
|
+
images: ownerPass.items.flatMap((item, index) => item.url ? [{
|
|
13339
|
+
index: index + 1,
|
|
13340
|
+
galleryPosition: item.position,
|
|
13341
|
+
sourceUrl: item.url,
|
|
13342
|
+
mediaKey: mapsImageIdentity(item.url),
|
|
13343
|
+
origin: "owner",
|
|
13344
|
+
originConfidence: "owner_gallery"
|
|
13345
|
+
}] : []),
|
|
13346
|
+
scope,
|
|
13347
|
+
exhausted: ownerPass.exhausted,
|
|
13348
|
+
stopReason: ownerGalleryAvailable ? ownerPass.stopReason : "gallery_unavailable",
|
|
13349
|
+
ownerGalleryAvailable,
|
|
13350
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13351
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13352
|
+
allPhotosDiscovered: ownerPass.photosDiscovered
|
|
13353
|
+
};
|
|
13354
|
+
}
|
|
13355
|
+
if (!await selectGalleryTab(this.page, "all")) {
|
|
13356
|
+
return {
|
|
13357
|
+
images: [],
|
|
13358
|
+
scope,
|
|
13359
|
+
exhausted: false,
|
|
13360
|
+
stopReason: "gallery_unavailable",
|
|
13361
|
+
ownerGalleryAvailable,
|
|
13362
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13363
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13364
|
+
allPhotosDiscovered: 0
|
|
13365
|
+
};
|
|
13366
|
+
}
|
|
13367
|
+
const allPass = await collectGalleryPass(this.page, maxImages);
|
|
13368
|
+
const images = allPass.items.flatMap((item, index) => {
|
|
13369
|
+
if (!item.url) return [];
|
|
13370
|
+
const mediaKey = mapsImageIdentity(item.url);
|
|
13371
|
+
const isOwner = ownerKeys.has(mediaKey);
|
|
13372
|
+
return [{
|
|
13373
|
+
index: index + 1,
|
|
13374
|
+
galleryPosition: item.position,
|
|
13375
|
+
sourceUrl: item.url,
|
|
13376
|
+
mediaKey,
|
|
13377
|
+
origin: isOwner ? "owner" : ownerPass.exhausted ? "other" : "unknown",
|
|
13378
|
+
originConfidence: isOwner ? "owner_gallery_match" : ownerPass.exhausted ? "not_in_exhausted_owner_gallery" : "owner_gallery_incomplete"
|
|
13379
|
+
}];
|
|
13380
|
+
});
|
|
13381
|
+
return {
|
|
13382
|
+
images,
|
|
13383
|
+
scope,
|
|
13384
|
+
exhausted: allPass.exhausted,
|
|
13385
|
+
stopReason: allPass.stopReason,
|
|
13386
|
+
ownerGalleryAvailable,
|
|
13387
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13388
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13389
|
+
allPhotosDiscovered: allPass.photosDiscovered
|
|
13390
|
+
};
|
|
13391
|
+
}
|
|
13392
|
+
};
|
|
13393
|
+
|
|
12523
13394
|
// src/extractor/MapsExtractor.ts
|
|
12524
13395
|
import { z as z11 } from "zod";
|
|
12525
13396
|
var MapsExtractor = class {
|
|
@@ -12544,6 +13415,48 @@ var MapsExtractor = class {
|
|
|
12544
13415
|
const identifiers = this.parseUrlIdentifiers(placeUrl);
|
|
12545
13416
|
const overview = await this.extractOverview(page);
|
|
12546
13417
|
const hoursTable = await this.extractFullHours(page);
|
|
13418
|
+
let media = {
|
|
13419
|
+
status: "not_requested",
|
|
13420
|
+
scope: options.imageScope,
|
|
13421
|
+
requestedMaxImages: options.maxImages,
|
|
13422
|
+
images: [],
|
|
13423
|
+
imagesCollected: 0,
|
|
13424
|
+
imagesDownloaded: 0,
|
|
13425
|
+
ownerImagesCollected: 0,
|
|
13426
|
+
otherImagesCollected: 0,
|
|
13427
|
+
unknownOriginImagesCollected: 0,
|
|
13428
|
+
ownerGalleryAvailable: false,
|
|
13429
|
+
ownerGalleryExhausted: false,
|
|
13430
|
+
ownerPhotosDiscovered: 0,
|
|
13431
|
+
allPhotosDiscovered: 0,
|
|
13432
|
+
exhausted: false,
|
|
13433
|
+
stopReason: "not_requested",
|
|
13434
|
+
artifact: null,
|
|
13435
|
+
warnings: []
|
|
13436
|
+
};
|
|
13437
|
+
if (options.includeImages) {
|
|
13438
|
+
try {
|
|
13439
|
+
await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
|
|
13440
|
+
await page.waitForTimeout(500);
|
|
13441
|
+
const collected = await new MapsImageCollector(page).collect(options.imageScope, options.maxImages);
|
|
13442
|
+
const ownerImagesCollected = collected.images.filter((image) => image.origin === "owner").length;
|
|
13443
|
+
const otherImagesCollected = collected.images.filter((image) => image.origin === "other").length;
|
|
13444
|
+
const unknownOriginImagesCollected = collected.images.filter((image) => image.origin === "unknown").length;
|
|
13445
|
+
media = {
|
|
13446
|
+
...media,
|
|
13447
|
+
...collected,
|
|
13448
|
+
status: collected.images.length > 0 ? "collected" : collected.stopReason === "gallery_unavailable" ? "unavailable" : "none_exist",
|
|
13449
|
+
imagesCollected: collected.images.length,
|
|
13450
|
+
ownerImagesCollected,
|
|
13451
|
+
otherImagesCollected,
|
|
13452
|
+
unknownOriginImagesCollected
|
|
13453
|
+
};
|
|
13454
|
+
await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 }).catch(() => void 0);
|
|
13455
|
+
} catch (err) {
|
|
13456
|
+
console.warn("[MapsExtractor] image gallery extraction failed:", err instanceof Error ? err.message : String(err));
|
|
13457
|
+
media = { ...media, status: "unavailable", stopReason: "gallery_unavailable", warnings: ["Google Maps image gallery could not be read in this run."] };
|
|
13458
|
+
}
|
|
13459
|
+
}
|
|
12547
13460
|
let reviewStats = { reviewHistogram: [], reviewTopics: [] };
|
|
12548
13461
|
let reviews = [];
|
|
12549
13462
|
let reviewsStatus = "not_requested";
|
|
@@ -12631,6 +13544,7 @@ var MapsExtractor = class {
|
|
|
12631
13544
|
services,
|
|
12632
13545
|
areasServed,
|
|
12633
13546
|
servicesStatus,
|
|
13547
|
+
media,
|
|
12634
13548
|
durationMs: Date.now() - startMs
|
|
12635
13549
|
};
|
|
12636
13550
|
} finally {
|
|
@@ -12913,11 +13827,11 @@ function rootDomain() {
|
|
|
12913
13827
|
const configured = process.env.LOCAL_SOURCEBOOK_PUBLIC_ROOT_DOMAIN?.trim() || DEFAULT_ROOT_DOMAIN;
|
|
12914
13828
|
return configured.replace(/^https?:\/\//i, "").replace(/^www\./i, "").replace(/\/.*$/, "").replace(/\.+$/, "").toLowerCase();
|
|
12915
13829
|
}
|
|
12916
|
-
function localSourcebookPublicUrls(category, state,
|
|
13830
|
+
function localSourcebookPublicUrls(category, state, slug4) {
|
|
12917
13831
|
const categoryUrl = `https://${CATEGORY_PUBLIC_SLUGS[category]}.${rootDomain()}`;
|
|
12918
13832
|
const statePath = `/${state.toLowerCase()}`;
|
|
12919
|
-
const profilePath = `${statePath}/${
|
|
12920
|
-
const reviewsPath = `${statePath}/${
|
|
13833
|
+
const profilePath = `${statePath}/${slug4}/`;
|
|
13834
|
+
const reviewsPath = `${statePath}/${slug4}/reviews`;
|
|
12921
13835
|
return {
|
|
12922
13836
|
categoryUrl,
|
|
12923
13837
|
nationwideDirectoryUrl: `${categoryUrl}/directory`,
|
|
@@ -13465,13 +14379,13 @@ function host(value) {
|
|
|
13465
14379
|
}
|
|
13466
14380
|
}
|
|
13467
14381
|
function tokens(value) {
|
|
13468
|
-
return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((
|
|
14382
|
+
return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token6) => token6.length > 1 && !BUSINESS_SUFFIXES.has(token6)));
|
|
13469
14383
|
}
|
|
13470
14384
|
function nameOverlap(left, right) {
|
|
13471
14385
|
const a = tokens(left);
|
|
13472
14386
|
const b = tokens(right);
|
|
13473
14387
|
if (!a.size || !b.size) return 0;
|
|
13474
|
-
const intersection = [...a].filter((
|
|
14388
|
+
const intersection = [...a].filter((token6) => b.has(token6)).length;
|
|
13475
14389
|
return intersection / Math.max(a.size, b.size);
|
|
13476
14390
|
}
|
|
13477
14391
|
function validateLocalSourcebookMapsIdentity(submission, maps) {
|
|
@@ -13520,6 +14434,10 @@ function createDefaultLocalSourcebookProvider() {
|
|
|
13520
14434
|
includeReviews: true,
|
|
13521
14435
|
maxReviews,
|
|
13522
14436
|
includeServices: true,
|
|
14437
|
+
includeImages: false,
|
|
14438
|
+
imageScope: "all",
|
|
14439
|
+
maxImages: 100,
|
|
14440
|
+
maxInlineImages: 3,
|
|
13523
14441
|
kernelApiKey,
|
|
13524
14442
|
headless: true
|
|
13525
14443
|
});
|
|
@@ -14405,12 +15323,12 @@ function normalizeIdempotencyKey(value) {
|
|
|
14405
15323
|
function tokenFor(id, idempotencyKey3) {
|
|
14406
15324
|
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
|
|
14407
15325
|
}
|
|
14408
|
-
function tokenHash(
|
|
14409
|
-
return createHash4("sha256").update(
|
|
15326
|
+
function tokenHash(token6) {
|
|
15327
|
+
return createHash4("sha256").update(token6).digest("hex");
|
|
14410
15328
|
}
|
|
14411
|
-
function verifyToken(row,
|
|
15329
|
+
function verifyToken(row, token6) {
|
|
14412
15330
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
14413
|
-
const actual = Buffer.from(tokenHash(
|
|
15331
|
+
const actual = Buffer.from(tokenHash(token6), "hex");
|
|
14414
15332
|
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
14415
15333
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
14416
15334
|
}
|
|
@@ -14527,11 +15445,11 @@ async function authorizeScheduledRun(args) {
|
|
|
14527
15445
|
);
|
|
14528
15446
|
}
|
|
14529
15447
|
const id = randomUUID4();
|
|
14530
|
-
const
|
|
15448
|
+
const token6 = tokenFor(id, idempotencyKey3);
|
|
14531
15449
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
14532
15450
|
const inserted = await getDb().execute({
|
|
14533
15451
|
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
14534
|
-
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(
|
|
15452
|
+
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token6), safeMetadata2(args.metadata), expiresAt]
|
|
14535
15453
|
});
|
|
14536
15454
|
if (inserted.rowsAffected === 0) {
|
|
14537
15455
|
const raced = await getDb().execute({
|
|
@@ -14547,7 +15465,7 @@ async function authorizeScheduledRun(args) {
|
|
|
14547
15465
|
}
|
|
14548
15466
|
return {
|
|
14549
15467
|
id,
|
|
14550
|
-
token:
|
|
15468
|
+
token: token6,
|
|
14551
15469
|
status: "authorized",
|
|
14552
15470
|
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
14553
15471
|
maximumMc,
|
|
@@ -16157,9 +17075,9 @@ var YouTubeExtractor = class {
|
|
|
16157
17075
|
}
|
|
16158
17076
|
async fetchContinuation(page, endpoint, contToken, context) {
|
|
16159
17077
|
const result = await page.evaluate(
|
|
16160
|
-
async ({ ep, token:
|
|
17078
|
+
async ({ ep, token: token6, ctx }) => {
|
|
16161
17079
|
const body = {
|
|
16162
|
-
continuation:
|
|
17080
|
+
continuation: token6,
|
|
16163
17081
|
context: {
|
|
16164
17082
|
client: {
|
|
16165
17083
|
clientName: ctx.clientName,
|
|
@@ -16403,23 +17321,23 @@ async function extractOnce(options) {
|
|
|
16403
17321
|
}
|
|
16404
17322
|
async function writeOutputs(result, outputDir) {
|
|
16405
17323
|
await fs.mkdir(outputDir, { recursive: true });
|
|
16406
|
-
const
|
|
17324
|
+
const slug4 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16407
17325
|
const ts = Date.now();
|
|
16408
17326
|
await fs.writeFile(
|
|
16409
|
-
path4.join(outputDir, `${
|
|
17327
|
+
path4.join(outputDir, `${slug4}-${ts}.json`),
|
|
16410
17328
|
JSON.stringify(result, null, 2),
|
|
16411
17329
|
"utf8"
|
|
16412
17330
|
);
|
|
16413
17331
|
if (result.videos.length > 0) {
|
|
16414
17332
|
await fs.writeFile(
|
|
16415
|
-
path4.join(outputDir, `${
|
|
17333
|
+
path4.join(outputDir, `${slug4}-videos-${ts}.csv`),
|
|
16416
17334
|
Papa.unparse(result.videos, { header: true }),
|
|
16417
17335
|
"utf8"
|
|
16418
17336
|
);
|
|
16419
17337
|
}
|
|
16420
17338
|
if (result.downloads.length > 0) {
|
|
16421
17339
|
await fs.writeFile(
|
|
16422
|
-
path4.join(outputDir, `${
|
|
17340
|
+
path4.join(outputDir, `${slug4}-downloads-${ts}.csv`),
|
|
16423
17341
|
Papa.unparse(result.downloads, { header: true }),
|
|
16424
17342
|
"utf8"
|
|
16425
17343
|
);
|
|
@@ -16783,6 +17701,8 @@ var ExtractUrlBodyShape = {
|
|
|
16783
17701
|
includeFeaturedImage: z16.boolean().optional(),
|
|
16784
17702
|
downloadMedia: z16.boolean().optional(),
|
|
16785
17703
|
mediaTypes: z16.array(z16.enum(["image", "video", "audio"])).optional(),
|
|
17704
|
+
maxMediaAssets: z16.number().int().min(1).max(250).optional(),
|
|
17705
|
+
maxInlineImages: z16.number().int().min(0).max(5).optional(),
|
|
16786
17706
|
delivery: z16.enum(["auto", "inline", "artifact", "memory"]).optional(),
|
|
16787
17707
|
preserveMedia: z16.boolean().optional(),
|
|
16788
17708
|
depositToVault: z16.boolean().optional(),
|
|
@@ -18059,7 +18979,7 @@ function advertiserTokenOverlap(query, candidate) {
|
|
|
18059
18979
|
const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
|
|
18060
18980
|
if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
|
|
18061
18981
|
let overlap = 0;
|
|
18062
|
-
for (const
|
|
18982
|
+
for (const token6 of queryTokens) if (candidateTokens.has(token6)) overlap++;
|
|
18063
18983
|
return overlap / queryTokens.size;
|
|
18064
18984
|
}
|
|
18065
18985
|
function assessPageIntelMatch(body, result) {
|
|
@@ -19294,8 +20214,8 @@ function getBlobStore() {
|
|
|
19294
20214
|
return cached;
|
|
19295
20215
|
}
|
|
19296
20216
|
var VercelBlobStore = class {
|
|
19297
|
-
constructor(
|
|
19298
|
-
this.token =
|
|
20217
|
+
constructor(token6) {
|
|
20218
|
+
this.token = token6;
|
|
19299
20219
|
}
|
|
19300
20220
|
token;
|
|
19301
20221
|
kind = "vercel-blob";
|
|
@@ -19751,8 +20671,8 @@ function safeFilePart(input) {
|
|
|
19751
20671
|
}
|
|
19752
20672
|
function blobKeyPrefix(shortcode, sourceUrl) {
|
|
19753
20673
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
19754
|
-
const
|
|
19755
|
-
return `instagram/${stamp}-${
|
|
20674
|
+
const slug4 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
20675
|
+
return `instagram/${stamp}-${slug4}`;
|
|
19756
20676
|
}
|
|
19757
20677
|
async function fetchMediaBytes(url, referer, runtimeUserAgent) {
|
|
19758
20678
|
const res = await fetch(url, {
|
|
@@ -20664,6 +21584,212 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
|
|
|
20664
21584
|
|
|
20665
21585
|
// src/api/maps-routes.ts
|
|
20666
21586
|
import { Hono as Hono13 } from "hono";
|
|
21587
|
+
|
|
21588
|
+
// src/api/maps-media-artifacts.ts
|
|
21589
|
+
import { homedir as homedir2 } from "os";
|
|
21590
|
+
import { join as join6 } from "path";
|
|
21591
|
+
import { randomBytes as randomBytes2 } from "crypto";
|
|
21592
|
+
import { ZipFile } from "yazl";
|
|
21593
|
+
import pLimit4 from "p-limit";
|
|
21594
|
+
var MAPS_MEDIA_ARTIFACT_PREFIX = "maps-media/";
|
|
21595
|
+
var MAPS_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
21596
|
+
var MAPS_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
21597
|
+
var MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
|
21598
|
+
var MAX_ARCHIVE_IMAGE_BYTES = 40 * 1024 * 1024;
|
|
21599
|
+
var MAX_INLINE_IMAGE_BYTES = 15e5;
|
|
21600
|
+
var MAX_INLINE_TOTAL_BYTES = 25e5;
|
|
21601
|
+
var DOWNLOAD_CONCURRENCY = 6;
|
|
21602
|
+
var MAX_REDIRECTS = 4;
|
|
21603
|
+
function token() {
|
|
21604
|
+
return process.env.MAPS_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
21605
|
+
}
|
|
21606
|
+
function policy() {
|
|
21607
|
+
return {
|
|
21608
|
+
prefix: MAPS_MEDIA_ARTIFACT_PREFIX,
|
|
21609
|
+
artifactTtlMs: MAPS_MEDIA_ARTIFACT_TTL_MS,
|
|
21610
|
+
downloadTtlMs: MAPS_MEDIA_DOWNLOAD_TTL_MS,
|
|
21611
|
+
token: token()
|
|
21612
|
+
};
|
|
21613
|
+
}
|
|
21614
|
+
function allowedMapsImageUrl(raw) {
|
|
21615
|
+
try {
|
|
21616
|
+
const url = new URL(raw);
|
|
21617
|
+
if (url.protocol !== "https:") return null;
|
|
21618
|
+
const host2 = url.hostname.toLowerCase().replace(/\.$/, "");
|
|
21619
|
+
if (host2 !== "googleusercontent.com" && !host2.endsWith(".googleusercontent.com") && host2 !== "streetviewpixels-pa.googleapis.com") return null;
|
|
21620
|
+
return url;
|
|
21621
|
+
} catch {
|
|
21622
|
+
return null;
|
|
21623
|
+
}
|
|
21624
|
+
}
|
|
21625
|
+
function preferredMapsImageRendition(rawUrl) {
|
|
21626
|
+
const url = allowedMapsImageUrl(rawUrl);
|
|
21627
|
+
if (!url) return rawUrl;
|
|
21628
|
+
if (url.hostname.toLowerCase().endsWith("googleusercontent.com") && /=[^/]+$/.test(url.pathname)) {
|
|
21629
|
+
url.pathname = url.pathname.replace(/=[^/]+$/, "=s2048");
|
|
21630
|
+
}
|
|
21631
|
+
return url.href;
|
|
21632
|
+
}
|
|
21633
|
+
function imageMime(bytes) {
|
|
21634
|
+
if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
|
|
21635
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
|
|
21636
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
|
|
21637
|
+
if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
|
|
21638
|
+
return null;
|
|
21639
|
+
}
|
|
21640
|
+
async function readBounded(response) {
|
|
21641
|
+
const declared = Number(response.headers.get("content-length") ?? "");
|
|
21642
|
+
if (Number.isFinite(declared) && declared > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21643
|
+
if (!response.body) {
|
|
21644
|
+
const bytes2 = new Uint8Array(await response.arrayBuffer());
|
|
21645
|
+
if (bytes2.byteLength > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21646
|
+
return bytes2;
|
|
21647
|
+
}
|
|
21648
|
+
const reader = response.body.getReader();
|
|
21649
|
+
const chunks = [];
|
|
21650
|
+
let total = 0;
|
|
21651
|
+
try {
|
|
21652
|
+
while (true) {
|
|
21653
|
+
const { done, value } = await reader.read();
|
|
21654
|
+
if (done) break;
|
|
21655
|
+
if (!value) continue;
|
|
21656
|
+
total += value.byteLength;
|
|
21657
|
+
if (total > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21658
|
+
chunks.push(value);
|
|
21659
|
+
}
|
|
21660
|
+
} finally {
|
|
21661
|
+
reader.releaseLock();
|
|
21662
|
+
}
|
|
21663
|
+
const bytes = new Uint8Array(total);
|
|
21664
|
+
let offset = 0;
|
|
21665
|
+
for (const chunk of chunks) {
|
|
21666
|
+
bytes.set(chunk, offset);
|
|
21667
|
+
offset += chunk.byteLength;
|
|
21668
|
+
}
|
|
21669
|
+
return bytes;
|
|
21670
|
+
}
|
|
21671
|
+
async function fetchMapsImage(rawUrl) {
|
|
21672
|
+
let url = allowedMapsImageUrl(preferredMapsImageRendition(rawUrl));
|
|
21673
|
+
if (!url) throw new Error("image_host_not_allowed");
|
|
21674
|
+
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
|
|
21675
|
+
const checked = await validatePublicHttpUrl(url.href, { field: "Maps image URL" });
|
|
21676
|
+
if (checked.error || !checked.parsed) throw new Error("image_url_rejected");
|
|
21677
|
+
const response = await fetch(checked.parsed.href, {
|
|
21678
|
+
redirect: "manual",
|
|
21679
|
+
headers: { Accept: "image/jpeg,image/png,image/webp,image/gif;q=0.8" },
|
|
21680
|
+
signal: AbortSignal.timeout(15e3)
|
|
21681
|
+
});
|
|
21682
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
21683
|
+
const location2 = response.headers.get("location");
|
|
21684
|
+
if (!location2 || redirect === MAX_REDIRECTS) throw new Error("image_redirect_rejected");
|
|
21685
|
+
url = allowedMapsImageUrl(new URL(location2, url).href);
|
|
21686
|
+
if (!url) throw new Error("image_redirect_host_not_allowed");
|
|
21687
|
+
continue;
|
|
21688
|
+
}
|
|
21689
|
+
if (!response.ok) throw new Error(`image_fetch_${response.status}`);
|
|
21690
|
+
const body = await readBounded(response);
|
|
21691
|
+
const detected = imageMime(body);
|
|
21692
|
+
if (!detected) throw new Error("image_content_invalid");
|
|
21693
|
+
return { bytes: Buffer.from(body), ...detected, finalUrl: url.href };
|
|
21694
|
+
}
|
|
21695
|
+
throw new Error("image_redirect_rejected");
|
|
21696
|
+
}
|
|
21697
|
+
function zipBuffer(files) {
|
|
21698
|
+
return new Promise((resolve2, reject) => {
|
|
21699
|
+
const zip = new ZipFile();
|
|
21700
|
+
const chunks = [];
|
|
21701
|
+
zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
|
21702
|
+
zip.outputStream.once("error", reject);
|
|
21703
|
+
zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
|
|
21704
|
+
for (const file of files) zip.addBuffer(file.content, file.path);
|
|
21705
|
+
zip.end();
|
|
21706
|
+
});
|
|
21707
|
+
}
|
|
21708
|
+
function slug(value) {
|
|
21709
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "google-maps-place";
|
|
21710
|
+
}
|
|
21711
|
+
function mapsMediaArtifactOwnerId(artifactId) {
|
|
21712
|
+
return privateArtifactOwnerId(artifactId, MAPS_MEDIA_ARTIFACT_PREFIX);
|
|
21713
|
+
}
|
|
21714
|
+
async function readOwnedMapsMediaArtifactBuffer(args) {
|
|
21715
|
+
if (mapsMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
21716
|
+
return readPrivateArtifactBuffer({ policy: policy(), artifactId: args.artifactId, maxBytes: 50 * 1024 * 1024 });
|
|
21717
|
+
}
|
|
21718
|
+
async function packageMapsMedia(args) {
|
|
21719
|
+
const limit = pLimit4(DOWNLOAD_CONCURRENCY);
|
|
21720
|
+
const files = [];
|
|
21721
|
+
let storedBytes = 0;
|
|
21722
|
+
let inlineBytes = 0;
|
|
21723
|
+
let inlineCount = 0;
|
|
21724
|
+
await Promise.all(args.media.images.map((image) => limit(async () => {
|
|
21725
|
+
try {
|
|
21726
|
+
const fetched = await fetchMapsImage(image.sourceUrl);
|
|
21727
|
+
if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_IMAGE_BYTES) throw new Error("archive_byte_limit_reached");
|
|
21728
|
+
storedBytes += fetched.bytes.length;
|
|
21729
|
+
const filename = `images/${String(image.index).padStart(4, "0")}-${image.origin}.${fetched.extension}`;
|
|
21730
|
+
files.push({ path: filename, content: fetched.bytes });
|
|
21731
|
+
image.sourceUrl = fetched.finalUrl;
|
|
21732
|
+
image.filename = filename;
|
|
21733
|
+
image.mimeType = fetched.mimeType;
|
|
21734
|
+
image.bytes = fetched.bytes.length;
|
|
21735
|
+
image.downloadStatus = "downloaded";
|
|
21736
|
+
image.downloadError = null;
|
|
21737
|
+
if (inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES) {
|
|
21738
|
+
image.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
|
|
21739
|
+
inlineBytes += fetched.bytes.length;
|
|
21740
|
+
inlineCount += 1;
|
|
21741
|
+
}
|
|
21742
|
+
} catch (error) {
|
|
21743
|
+
image.filename = null;
|
|
21744
|
+
image.mimeType = null;
|
|
21745
|
+
image.bytes = null;
|
|
21746
|
+
image.downloadStatus = "failed";
|
|
21747
|
+
image.downloadError = error instanceof Error ? error.message : "image_download_failed";
|
|
21748
|
+
image.inlinePreview = null;
|
|
21749
|
+
}
|
|
21750
|
+
})));
|
|
21751
|
+
args.media.imagesDownloaded = args.media.images.filter((image) => image.downloadStatus === "downloaded").length;
|
|
21752
|
+
const failed = args.media.images.length - args.media.imagesDownloaded;
|
|
21753
|
+
if (failed > 0) args.media.warnings.push(`${failed} image(s) could not be downloaded; their manifest rows include a bounded error code.`);
|
|
21754
|
+
if (args.media.imagesDownloaded === 0) return { media: args.media, artifact: null };
|
|
21755
|
+
const cleanImages = args.media.images.map(({ inlinePreview: _preview, ...image }) => image);
|
|
21756
|
+
const summary = {
|
|
21757
|
+
businessName: args.businessName,
|
|
21758
|
+
placeUrl: args.placeUrl,
|
|
21759
|
+
extractedAt: args.extractedAt,
|
|
21760
|
+
scope: args.media.scope,
|
|
21761
|
+
imagesCollected: args.media.imagesCollected,
|
|
21762
|
+
imagesDownloaded: args.media.imagesDownloaded,
|
|
21763
|
+
ownerImagesCollected: args.media.ownerImagesCollected,
|
|
21764
|
+
otherImagesCollected: args.media.otherImagesCollected,
|
|
21765
|
+
unknownOriginImagesCollected: args.media.unknownOriginImagesCollected,
|
|
21766
|
+
ownerGalleryExhausted: args.media.ownerGalleryExhausted,
|
|
21767
|
+
galleryExhausted: args.media.exhausted,
|
|
21768
|
+
stopReason: args.media.stopReason,
|
|
21769
|
+
notes: [
|
|
21770
|
+
"origin=owner is backed by the Google Maps By owner gallery or an exact media-key match to it.",
|
|
21771
|
+
"origin=other means absent from an exhausted owner gallery; it can include customer, review, Street View, Google, or unattributed media.",
|
|
21772
|
+
"Google-hosted photo files use a bounded s2048 rendition when that CDN form is available; each file is still subject to byte and signature validation."
|
|
21773
|
+
]
|
|
21774
|
+
};
|
|
21775
|
+
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
21776
|
+
files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
|
|
21777
|
+
const archive = await zipBuffer(files);
|
|
21778
|
+
const id = randomBytes2(6).toString("hex");
|
|
21779
|
+
const pointer = await createPrivateArtifact({
|
|
21780
|
+
policy: policy(),
|
|
21781
|
+
ownerId: args.ownerId,
|
|
21782
|
+
artifactKey: `${id}.zip`,
|
|
21783
|
+
createdAt: new Date(args.extractedAt),
|
|
21784
|
+
filename: `${slug(args.businessName)}-maps-images.zip`,
|
|
21785
|
+
contentType: "application/zip",
|
|
21786
|
+
content: archive
|
|
21787
|
+
});
|
|
21788
|
+
const localPath = token() ? null : join6(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
21789
|
+
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
21790
|
+
}
|
|
21791
|
+
|
|
21792
|
+
// src/api/maps-routes.ts
|
|
20667
21793
|
function mapsErrorResponse(c, err, errorCode) {
|
|
20668
21794
|
const msg = err instanceof Error ? err.message : String(err);
|
|
20669
21795
|
const rotationError = err instanceof MapsSearchRotationError ? err : null;
|
|
@@ -20782,6 +21908,30 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
20782
21908
|
result.reviews = [];
|
|
20783
21909
|
}
|
|
20784
21910
|
}
|
|
21911
|
+
if (parsed.data.includeImages && result.media?.images.length > 0) {
|
|
21912
|
+
try {
|
|
21913
|
+
const packaged = await packageMapsMedia({
|
|
21914
|
+
ownerId: String(user.id),
|
|
21915
|
+
businessName: result.name ?? parsed.data.businessName,
|
|
21916
|
+
placeUrl: result.placeUrl,
|
|
21917
|
+
extractedAt: result.extractedAt,
|
|
21918
|
+
media: result.media,
|
|
21919
|
+
maxInlineImages: parsed.data.maxInlineImages
|
|
21920
|
+
});
|
|
21921
|
+
result.media = packaged.media;
|
|
21922
|
+
result.media.artifact = packaged.artifact;
|
|
21923
|
+
} catch (error) {
|
|
21924
|
+
console.warn("[maps/place] image artifact packaging failed:", error instanceof Error ? error.message : String(error));
|
|
21925
|
+
result.media.warnings.push("The image manifest was collected, but the downloadable ZIP artifact could not be created.");
|
|
21926
|
+
}
|
|
21927
|
+
}
|
|
21928
|
+
const logResult = result.media ? {
|
|
21929
|
+
...result,
|
|
21930
|
+
media: {
|
|
21931
|
+
...result.media,
|
|
21932
|
+
images: result.media.images.map(({ inlinePreview: _preview, ...image }) => image)
|
|
21933
|
+
}
|
|
21934
|
+
} : result;
|
|
20785
21935
|
await logRequestEvent({
|
|
20786
21936
|
userId: user.id,
|
|
20787
21937
|
source: "maps_place",
|
|
@@ -20789,7 +21939,7 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
20789
21939
|
query: `${parsed.data.businessName} ${parsed.data.location}`,
|
|
20790
21940
|
location: parsed.data.location,
|
|
20791
21941
|
resultCount: reviewCount,
|
|
20792
|
-
result
|
|
21942
|
+
result: logResult
|
|
20793
21943
|
});
|
|
20794
21944
|
return c.json(result);
|
|
20795
21945
|
} catch (err) {
|
|
@@ -21729,7 +22879,7 @@ function getLocalSourcebookContract(category) {
|
|
|
21729
22879
|
}
|
|
21730
22880
|
async function prepareLocalSourcebookWrite(input) {
|
|
21731
22881
|
const parsed = DirectoryIdentitySchema.parse(input);
|
|
21732
|
-
const
|
|
22882
|
+
const slug4 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
|
|
21733
22883
|
const categoryTag = CATEGORY_TAG[parsed.category];
|
|
21734
22884
|
const candidates = [
|
|
21735
22885
|
{ tag: categoryTag, central: true, reusable: true },
|
|
@@ -21737,12 +22887,12 @@ async function prepareLocalSourcebookWrite(input) {
|
|
|
21737
22887
|
...parsed.tags.filter((tag) => tag !== categoryTag).map((tag) => ({ tag }))
|
|
21738
22888
|
];
|
|
21739
22889
|
const tagResolutions = await resolveLocalSourcebookTags(candidates);
|
|
21740
|
-
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state,
|
|
22890
|
+
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug4);
|
|
21741
22891
|
const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
|
|
21742
22892
|
return {
|
|
21743
22893
|
ok: true,
|
|
21744
|
-
route: { category: parsed.category, state: parsed.state, slug:
|
|
21745
|
-
proposedWrite: { ...parsed, schemaOrgType, slug:
|
|
22894
|
+
route: { category: parsed.category, state: parsed.state, slug: slug4, ...publicUrls },
|
|
22895
|
+
proposedWrite: { ...parsed, schemaOrgType, slug: slug4, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21746
22896
|
contract: getLocalSourcebookContract(parsed.category),
|
|
21747
22897
|
tagResolutions,
|
|
21748
22898
|
instructions: [
|
|
@@ -21807,7 +22957,7 @@ async function persistLocalSourcebookTags(submissionId, userId, tags, decisions
|
|
|
21807
22957
|
}
|
|
21808
22958
|
|
|
21809
22959
|
// src/api/local-sourcebook-routes.ts
|
|
21810
|
-
var
|
|
22960
|
+
var slug2 = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
|
|
21811
22961
|
var SubmitSchema = z26.object({
|
|
21812
22962
|
category: z26.enum(LOCAL_SOURCEBOOK_CATEGORIES),
|
|
21813
22963
|
state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
@@ -21878,7 +23028,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21878
23028
|
if (!idempotencyKey3) {
|
|
21879
23029
|
return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
|
|
21880
23030
|
}
|
|
21881
|
-
const listingSlug = identity.slug ??
|
|
23031
|
+
const listingSlug = identity.slug ?? slug2(identity.businessName);
|
|
21882
23032
|
const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
|
|
21883
23033
|
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
|
|
21884
23034
|
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
@@ -21929,7 +23079,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21929
23079
|
if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
|
|
21930
23080
|
const input = parsed.data;
|
|
21931
23081
|
if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
|
|
21932
|
-
const listingSlug = input.slug ??
|
|
23082
|
+
const listingSlug = input.slug ?? slug2(input.businessName);
|
|
21933
23083
|
const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
|
|
21934
23084
|
const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
|
|
21935
23085
|
let billing = null;
|
|
@@ -22986,7 +24136,7 @@ import { createHash as createHash8, randomUUID as randomUUID9 } from "crypto";
|
|
|
22986
24136
|
|
|
22987
24137
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
22988
24138
|
import { createHash as createHash7 } from "crypto";
|
|
22989
|
-
import
|
|
24139
|
+
import pLimit5 from "p-limit";
|
|
22990
24140
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
22991
24141
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
22992
24142
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
@@ -23267,7 +24417,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
23267
24417
|
async function capturePageSnapshots(targets, options = {}) {
|
|
23268
24418
|
const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
|
|
23269
24419
|
const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
|
|
23270
|
-
const limit =
|
|
24420
|
+
const limit = pLimit5(maxConcurrency);
|
|
23271
24421
|
const pageSnapshotArtifacts = await Promise.all(
|
|
23272
24422
|
targets.map((target) => limit(() => capturePageSnapshot(target, { ...options, timeoutMs })))
|
|
23273
24423
|
);
|
|
@@ -24108,15 +25258,15 @@ var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
|
|
|
24108
25258
|
["image/webp", "webp"],
|
|
24109
25259
|
["image/gif", "gif"]
|
|
24110
25260
|
]);
|
|
24111
|
-
function
|
|
25261
|
+
function token2() {
|
|
24112
25262
|
return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
24113
25263
|
}
|
|
24114
|
-
function
|
|
25264
|
+
function policy2() {
|
|
24115
25265
|
return {
|
|
24116
25266
|
prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
|
|
24117
25267
|
artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
|
|
24118
25268
|
downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
|
|
24119
|
-
token:
|
|
25269
|
+
token: token2()
|
|
24120
25270
|
};
|
|
24121
25271
|
}
|
|
24122
25272
|
function imageSourceArtifactOwnerId(artifactId) {
|
|
@@ -24130,7 +25280,7 @@ async function createImageSourceArtifact(args) {
|
|
|
24130
25280
|
}
|
|
24131
25281
|
const id = randomUUID10().replaceAll("-", "");
|
|
24132
25282
|
return createPrivateArtifact({
|
|
24133
|
-
policy:
|
|
25283
|
+
policy: policy2(),
|
|
24134
25284
|
ownerId: args.ownerId,
|
|
24135
25285
|
artifactKey: `${id}.${extension}`,
|
|
24136
25286
|
createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
|
|
@@ -24142,7 +25292,7 @@ async function createImageSourceArtifact(args) {
|
|
|
24142
25292
|
async function readOwnedImageSourceArtifact(args) {
|
|
24143
25293
|
if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
24144
25294
|
return readPrivateArtifactBuffer({
|
|
24145
|
-
policy:
|
|
25295
|
+
policy: policy2(),
|
|
24146
25296
|
artifactId: args.artifactId,
|
|
24147
25297
|
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
24148
25298
|
});
|
|
@@ -24728,8 +25878,8 @@ function protectedResourceMetadata() {
|
|
|
24728
25878
|
function wwwAuthenticateValue() {
|
|
24729
25879
|
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
24730
25880
|
}
|
|
24731
|
-
function isBearerJwt(
|
|
24732
|
-
return !
|
|
25881
|
+
function isBearerJwt(token6) {
|
|
25882
|
+
return !token6.startsWith("sk_") && token6.split(".").length === 3;
|
|
24733
25883
|
}
|
|
24734
25884
|
var jwksSingleton = null;
|
|
24735
25885
|
function getJwks() {
|
|
@@ -24737,8 +25887,8 @@ function getJwks() {
|
|
|
24737
25887
|
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
24738
25888
|
return jwksSingleton;
|
|
24739
25889
|
}
|
|
24740
|
-
async function verifyScraperOAuthJwt(
|
|
24741
|
-
const { payload } = await jwtVerify(
|
|
25890
|
+
async function verifyScraperOAuthJwt(token6) {
|
|
25891
|
+
const { payload } = await jwtVerify(token6, getJwks(), {
|
|
24742
25892
|
issuer: oauthIssuer(),
|
|
24743
25893
|
audience: oauthResourceUrl()
|
|
24744
25894
|
});
|
|
@@ -27461,7 +28611,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
27461
28611
|
// src/api/oauth-routes.ts
|
|
27462
28612
|
import { Hono as Hono24 } from "hono";
|
|
27463
28613
|
import { getCookie, setCookie } from "hono/cookie";
|
|
27464
|
-
import { createHash as createHash9, randomBytes as
|
|
28614
|
+
import { createHash as createHash9, randomBytes as randomBytes3, randomUUID as randomUUID13 } from "crypto";
|
|
27465
28615
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
27466
28616
|
var ISSUER = "https://mcpscraper.dev";
|
|
27467
28617
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -27560,9 +28710,9 @@ function negotiateScope(requested, user, resource) {
|
|
|
27560
28710
|
if (operator && !unique3.includes("memory.admin")) unique3.push("memory.admin");
|
|
27561
28711
|
return unique3.join(" ");
|
|
27562
28712
|
}
|
|
27563
|
-
async function sessionUser(
|
|
27564
|
-
if (!
|
|
27565
|
-
const userId = verifySession(
|
|
28713
|
+
async function sessionUser(token6) {
|
|
28714
|
+
if (!token6) return null;
|
|
28715
|
+
const userId = verifySession(token6);
|
|
27566
28716
|
if (!userId) return null;
|
|
27567
28717
|
const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
|
|
27568
28718
|
return user ?? null;
|
|
@@ -27747,7 +28897,7 @@ oauthApp.post("/oauth/register", async (c) => {
|
|
|
27747
28897
|
}
|
|
27748
28898
|
}
|
|
27749
28899
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
27750
|
-
const clientId = `client_${
|
|
28900
|
+
const clientId = `client_${randomBytes3(16).toString("hex")}`;
|
|
27751
28901
|
await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
|
|
27752
28902
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
27753
28903
|
return c.json({
|
|
@@ -27822,7 +28972,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
27822
28972
|
if (action2 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
27823
28973
|
if (action2 !== "approve") return c.text("unsupported action", 400);
|
|
27824
28974
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
27825
|
-
const code = `code_${
|
|
28975
|
+
const code = `code_${randomBytes3(32).toString("base64url")}`;
|
|
27826
28976
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
27827
28977
|
await withOAuthDatabaseDeadline("put-authorization-code", putCode({
|
|
27828
28978
|
code,
|
|
@@ -27873,7 +29023,7 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27873
29023
|
const plan = user ? resolvePlan(user) : "free";
|
|
27874
29024
|
const audience = record.resource ?? RESOURCE();
|
|
27875
29025
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
27876
|
-
const refreshToken = `rt_${
|
|
29026
|
+
const refreshToken = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
27877
29027
|
await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
|
|
27878
29028
|
refresh_token: refreshToken,
|
|
27879
29029
|
client_id: clientId,
|
|
@@ -27903,7 +29053,7 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27903
29053
|
const plan = user ? resolvePlan(user) : "free";
|
|
27904
29054
|
const audience = record.resource ?? RESOURCE();
|
|
27905
29055
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
27906
|
-
const nextRefresh = `rt_${
|
|
29056
|
+
const nextRefresh = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
27907
29057
|
await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
|
|
27908
29058
|
refresh_token: nextRefresh,
|
|
27909
29059
|
client_id: record.client_id,
|
|
@@ -27921,35 +29071,35 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27921
29071
|
// src/api/chat-routes.ts
|
|
27922
29072
|
import { Hono as Hono25 } from "hono";
|
|
27923
29073
|
var chatApp = new Hono25();
|
|
27924
|
-
function tokenOk(
|
|
27925
|
-
return !!
|
|
29074
|
+
function tokenOk(token6) {
|
|
29075
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 10;
|
|
27926
29076
|
}
|
|
27927
29077
|
function jsonError(message, status = 400) {
|
|
27928
29078
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
27929
29079
|
}
|
|
27930
29080
|
chatApp.get("/api/channels", async (c) => {
|
|
27931
|
-
const
|
|
27932
|
-
if (!tokenOk(
|
|
27933
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
29081
|
+
const token6 = c.req.query("token");
|
|
29082
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29083
|
+
const r = await memoryCall("listVaultsTool", {}, token6);
|
|
27934
29084
|
if (!r.ok) return c.json(r);
|
|
27935
29085
|
const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
|
|
27936
29086
|
return c.json({ ok: true, channels: vaults });
|
|
27937
29087
|
});
|
|
27938
29088
|
chatApp.get("/api/messages", async (c) => {
|
|
27939
|
-
const
|
|
27940
|
-
if (!tokenOk(
|
|
29089
|
+
const token6 = c.req.query("token");
|
|
29090
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27941
29091
|
const vault = c.req.query("vault");
|
|
27942
29092
|
if (!vault) return jsonError("vault is required");
|
|
27943
29093
|
const parentMessageId = c.req.query("parentMessageId") || void 0;
|
|
27944
|
-
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId },
|
|
29094
|
+
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token6);
|
|
27945
29095
|
return c.json(r);
|
|
27946
29096
|
});
|
|
27947
29097
|
chatApp.get("/api/poll", async (c) => {
|
|
27948
|
-
const
|
|
27949
|
-
if (!tokenOk(
|
|
29098
|
+
const token6 = c.req.query("token");
|
|
29099
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27950
29100
|
const vault = c.req.query("vault");
|
|
27951
29101
|
if (!vault) return jsonError("vault is required");
|
|
27952
|
-
const r = await memoryCall("pollChannelTool", { vault },
|
|
29102
|
+
const r = await memoryCall("pollChannelTool", { vault }, token6);
|
|
27953
29103
|
return c.json(r);
|
|
27954
29104
|
});
|
|
27955
29105
|
function parseAttachNote(v) {
|
|
@@ -27974,12 +29124,12 @@ chatApp.post("/api/reply", async (c) => {
|
|
|
27974
29124
|
return c.json(r);
|
|
27975
29125
|
});
|
|
27976
29126
|
chatApp.get("/api/notes-own", async (c) => {
|
|
27977
|
-
const
|
|
27978
|
-
if (!tokenOk(
|
|
27979
|
-
const vaultsRes = await memoryCall("listVaultsTool", {},
|
|
29127
|
+
const token6 = c.req.query("token");
|
|
29128
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29129
|
+
const vaultsRes = await memoryCall("listVaultsTool", {}, token6);
|
|
27980
29130
|
if (!vaultsRes.ok) return c.json(vaultsRes);
|
|
27981
29131
|
const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
|
|
27982
|
-
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle },
|
|
29132
|
+
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token6)));
|
|
27983
29133
|
const notes = ownVaults.flatMap((v, i) => {
|
|
27984
29134
|
const r = lists[i];
|
|
27985
29135
|
return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
|
|
@@ -27987,12 +29137,12 @@ chatApp.get("/api/notes-own", async (c) => {
|
|
|
27987
29137
|
return c.json({ ok: true, notes });
|
|
27988
29138
|
});
|
|
27989
29139
|
chatApp.get("/api/message-note", async (c) => {
|
|
27990
|
-
const
|
|
27991
|
-
if (!tokenOk(
|
|
29140
|
+
const token6 = c.req.query("token");
|
|
29141
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27992
29142
|
const vault = c.req.query("vault");
|
|
27993
29143
|
const messageId = c.req.query("messageId");
|
|
27994
29144
|
if (!vault || !messageId) return jsonError("vault and messageId are required");
|
|
27995
|
-
const r = await memoryCall("getMessageNoteTool", { vault, messageId },
|
|
29145
|
+
const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token6);
|
|
27996
29146
|
return c.json(r);
|
|
27997
29147
|
});
|
|
27998
29148
|
chatApp.post("/api/react", async (c) => {
|
|
@@ -28003,17 +29153,17 @@ chatApp.post("/api/react", async (c) => {
|
|
|
28003
29153
|
return c.json(r);
|
|
28004
29154
|
});
|
|
28005
29155
|
chatApp.get("/api/mentions", async (c) => {
|
|
28006
|
-
const
|
|
28007
|
-
if (!tokenOk(
|
|
28008
|
-
const r = await memoryCall("myMentionsTool", {},
|
|
29156
|
+
const token6 = c.req.query("token");
|
|
29157
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29158
|
+
const r = await memoryCall("myMentionsTool", {}, token6);
|
|
28009
29159
|
return c.json(r);
|
|
28010
29160
|
});
|
|
28011
29161
|
chatApp.get("/api/members", async (c) => {
|
|
28012
|
-
const
|
|
28013
|
-
if (!tokenOk(
|
|
29162
|
+
const token6 = c.req.query("token");
|
|
29163
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
28014
29164
|
const vault = c.req.query("vault");
|
|
28015
29165
|
if (!vault) return jsonError("vault is required");
|
|
28016
|
-
const r = await memoryCall("listChannelMembersTool", { vault },
|
|
29166
|
+
const r = await memoryCall("listChannelMembersTool", { vault }, token6);
|
|
28017
29167
|
return c.json(r);
|
|
28018
29168
|
});
|
|
28019
29169
|
chatApp.post("/api/channel", async (c) => {
|
|
@@ -28049,13 +29199,13 @@ chatApp.post("/api/member-remove", async (c) => {
|
|
|
28049
29199
|
return c.json(r);
|
|
28050
29200
|
});
|
|
28051
29201
|
chatApp.get("/:token", (c) => {
|
|
28052
|
-
const
|
|
28053
|
-
if (!tokenOk(
|
|
29202
|
+
const token6 = c.req.param("token");
|
|
29203
|
+
if (!tokenOk(token6)) {
|
|
28054
29204
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
|
|
28055
29205
|
}
|
|
28056
|
-
return c.html(renderChatPage(
|
|
29206
|
+
return c.html(renderChatPage(token6));
|
|
28057
29207
|
});
|
|
28058
|
-
function renderChatPage(
|
|
29208
|
+
function renderChatPage(token6) {
|
|
28059
29209
|
return `<!doctype html>
|
|
28060
29210
|
<html>
|
|
28061
29211
|
<head>
|
|
@@ -28240,7 +29390,7 @@ function renderChatPage(token4) {
|
|
|
28240
29390
|
</div>
|
|
28241
29391
|
|
|
28242
29392
|
<script>
|
|
28243
|
-
const TOKEN = ${JSON.stringify(
|
|
29393
|
+
const TOKEN = ${JSON.stringify(token6)};
|
|
28244
29394
|
const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
|
|
28245
29395
|
|
|
28246
29396
|
function api(path, opts) {
|
|
@@ -28610,8 +29760,8 @@ var MAX_VAULT_RECORDS = 200;
|
|
|
28610
29760
|
var MAX_MEMORY_READ_CONCURRENCY = 4;
|
|
28611
29761
|
var MAX_MEMORY_READ_ATTEMPTS = 3;
|
|
28612
29762
|
var vaultApp = new Hono26();
|
|
28613
|
-
function validToken(
|
|
28614
|
-
return !!
|
|
29763
|
+
function validToken(token6) {
|
|
29764
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 20 && token6.length < 256;
|
|
28615
29765
|
}
|
|
28616
29766
|
function isVault(value) {
|
|
28617
29767
|
return typeof value === "string" && VAULTS.includes(value);
|
|
@@ -28737,11 +29887,11 @@ function shouldRetryMemoryRead(error) {
|
|
|
28737
29887
|
function sleep(ms) {
|
|
28738
29888
|
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
28739
29889
|
}
|
|
28740
|
-
async function readMemory(tool, args,
|
|
28741
|
-
let result = await memoryCall(tool, args,
|
|
29890
|
+
async function readMemory(tool, args, token6) {
|
|
29891
|
+
let result = await memoryCall(tool, args, token6);
|
|
28742
29892
|
for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
|
|
28743
29893
|
await sleep(attempt * 150);
|
|
28744
|
-
result = await memoryCall(tool, args,
|
|
29894
|
+
result = await memoryCall(tool, args, token6);
|
|
28745
29895
|
}
|
|
28746
29896
|
return result;
|
|
28747
29897
|
}
|
|
@@ -28758,27 +29908,27 @@ async function mapWithConcurrency3(items, limit, map) {
|
|
|
28758
29908
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
28759
29909
|
return values;
|
|
28760
29910
|
}
|
|
28761
|
-
async function readVault(vault,
|
|
28762
|
-
const list = await readMemory("listTool", { vault },
|
|
29911
|
+
async function readVault(vault, token6) {
|
|
29912
|
+
const list = await readMemory("listTool", { vault }, token6);
|
|
28763
29913
|
if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
|
|
28764
29914
|
const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
|
|
28765
29915
|
const notes = await mapWithConcurrency3(
|
|
28766
29916
|
noteRefs,
|
|
28767
29917
|
MAX_MEMORY_READ_CONCURRENCY,
|
|
28768
|
-
({ path: path5 }) => readMemory("getTool", { vault, path: path5 },
|
|
29918
|
+
({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token6)
|
|
28769
29919
|
);
|
|
28770
29920
|
const failed = notes.find((result) => !result.ok);
|
|
28771
29921
|
if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
|
|
28772
29922
|
return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
|
|
28773
29923
|
}
|
|
28774
29924
|
vaultApp.get("/api/workspace", async (c) => {
|
|
28775
|
-
const
|
|
28776
|
-
if (!validToken(
|
|
29925
|
+
const token6 = c.req.query("token");
|
|
29926
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28777
29927
|
try {
|
|
28778
29928
|
const requested = c.req.query("vault");
|
|
28779
29929
|
const vaults = isVault(requested) ? [requested] : VAULTS;
|
|
28780
29930
|
const results = [];
|
|
28781
|
-
for (const vault of vaults) results.push(await readVault(vault,
|
|
29931
|
+
for (const vault of vaults) results.push(await readVault(vault, token6));
|
|
28782
29932
|
const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
28783
29933
|
return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
|
|
28784
29934
|
} catch (error) {
|
|
@@ -28787,8 +29937,8 @@ vaultApp.get("/api/workspace", async (c) => {
|
|
|
28787
29937
|
});
|
|
28788
29938
|
vaultApp.post("/api/records", async (c) => {
|
|
28789
29939
|
const body = await c.req.json().catch(() => ({}));
|
|
28790
|
-
const
|
|
28791
|
-
if (!validToken(
|
|
29940
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29941
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28792
29942
|
const vault = body.vault;
|
|
28793
29943
|
const path5 = stringValue(body.path, 320);
|
|
28794
29944
|
const title = stringValue(body.title, 320);
|
|
@@ -28799,19 +29949,19 @@ vaultApp.post("/api/records", async (c) => {
|
|
|
28799
29949
|
const validationError2 = validateNewRecord(vault, normalizedProps);
|
|
28800
29950
|
if (validationError2) return c.json({ ok: false, error: validationError2 }, 400);
|
|
28801
29951
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28802
|
-
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } },
|
|
29952
|
+
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token6);
|
|
28803
29953
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28804
29954
|
});
|
|
28805
29955
|
vaultApp.post("/api/records/transition", async (c) => {
|
|
28806
29956
|
const body = await c.req.json().catch(() => ({}));
|
|
28807
|
-
const
|
|
28808
|
-
if (!validToken(
|
|
29957
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29958
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28809
29959
|
const vault = typeof body.vault === "string" ? body.vault : "";
|
|
28810
29960
|
const path5 = typeof body.path === "string" ? body.path : "";
|
|
28811
29961
|
const status = stringValue(body.status, 80);
|
|
28812
29962
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28813
29963
|
if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
|
|
28814
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
29964
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28815
29965
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28816
29966
|
const result = await memoryCall("putTool", {
|
|
28817
29967
|
vault,
|
|
@@ -28820,13 +29970,13 @@ vaultApp.post("/api/records/transition", async (c) => {
|
|
|
28820
29970
|
content: current.note.content,
|
|
28821
29971
|
props: { ...current.note.props ?? {}, status },
|
|
28822
29972
|
baseRevision: baseRevision ?? current.note.revision
|
|
28823
|
-
},
|
|
29973
|
+
}, token6);
|
|
28824
29974
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28825
29975
|
});
|
|
28826
29976
|
vaultApp.post("/api/records/update", async (c) => {
|
|
28827
29977
|
const body = await c.req.json().catch(() => ({}));
|
|
28828
|
-
const
|
|
28829
|
-
if (!validToken(
|
|
29978
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29979
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28830
29980
|
const vault = body.vault;
|
|
28831
29981
|
const path5 = stringValue(body.path, 320);
|
|
28832
29982
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -28836,7 +29986,7 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
28836
29986
|
if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
|
|
28837
29987
|
return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
|
|
28838
29988
|
}
|
|
28839
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
29989
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28840
29990
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28841
29991
|
const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
|
|
28842
29992
|
const validationError2 = validateNewRecord(vault, nextProps);
|
|
@@ -28848,18 +29998,18 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
28848
29998
|
content: content ?? current.note.content,
|
|
28849
29999
|
props: nextProps,
|
|
28850
30000
|
baseRevision: baseRevision ?? current.note.revision
|
|
28851
|
-
},
|
|
30001
|
+
}, token6);
|
|
28852
30002
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28853
30003
|
});
|
|
28854
30004
|
vaultApp.delete("/api/records", async (c) => {
|
|
28855
30005
|
const body = await c.req.json().catch(() => ({}));
|
|
28856
|
-
const
|
|
28857
|
-
if (!validToken(
|
|
30006
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30007
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28858
30008
|
const vault = body.vault;
|
|
28859
30009
|
const path5 = stringValue(body.path, 320);
|
|
28860
30010
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28861
30011
|
if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
|
|
28862
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
30012
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28863
30013
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28864
30014
|
if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
|
|
28865
30015
|
return c.json({ ok: false, error: "only task records can be deleted" }, 400);
|
|
@@ -28868,13 +30018,13 @@ vaultApp.delete("/api/records", async (c) => {
|
|
|
28868
30018
|
vault,
|
|
28869
30019
|
path: path5,
|
|
28870
30020
|
baseRevision: baseRevision ?? current.note.revision
|
|
28871
|
-
},
|
|
30021
|
+
}, token6);
|
|
28872
30022
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28873
30023
|
});
|
|
28874
30024
|
vaultApp.post("/api/communications/review", async (c) => {
|
|
28875
30025
|
const body = await c.req.json().catch(() => ({}));
|
|
28876
|
-
const
|
|
28877
|
-
if (!validToken(
|
|
30026
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30027
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28878
30028
|
const path5 = stringValue(body.path, 320);
|
|
28879
30029
|
const action2 = stringValue(body.action, 24);
|
|
28880
30030
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -28885,7 +30035,7 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
28885
30035
|
if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
|
|
28886
30036
|
return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
|
|
28887
30037
|
}
|
|
28888
|
-
const current = await readMemory("getTool", { vault: "Communications", path: path5 },
|
|
30038
|
+
const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token6);
|
|
28889
30039
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28890
30040
|
if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
|
|
28891
30041
|
const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
|
|
@@ -28909,13 +30059,13 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
28909
30059
|
content: content ?? current.note.content,
|
|
28910
30060
|
props,
|
|
28911
30061
|
baseRevision: baseRevision ?? current.note.revision
|
|
28912
|
-
},
|
|
30062
|
+
}, token6);
|
|
28913
30063
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28914
30064
|
});
|
|
28915
30065
|
vaultApp.post("/api/activity", async (c) => {
|
|
28916
30066
|
const body = await c.req.json().catch(() => ({}));
|
|
28917
|
-
const
|
|
28918
|
-
if (!validToken(
|
|
30067
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30068
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28919
30069
|
const title = stringValue(body.title, 320);
|
|
28920
30070
|
const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
|
|
28921
30071
|
const props = propsValue(body.props) ?? {};
|
|
@@ -28925,7 +30075,7 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
28925
30075
|
return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
|
|
28926
30076
|
}
|
|
28927
30077
|
for (const ref of personRefs) {
|
|
28928
|
-
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") },
|
|
30078
|
+
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token6);
|
|
28929
30079
|
if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
|
|
28930
30080
|
return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
|
|
28931
30081
|
}
|
|
@@ -28953,37 +30103,37 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
28953
30103
|
...props,
|
|
28954
30104
|
person_refs: personRefs
|
|
28955
30105
|
}
|
|
28956
|
-
},
|
|
30106
|
+
}, token6);
|
|
28957
30107
|
return c.json(result, result.ok ? 200 : 502);
|
|
28958
30108
|
});
|
|
28959
30109
|
|
|
28960
30110
|
// src/api/schedule-routes.ts
|
|
28961
30111
|
import { Hono as Hono27 } from "hono";
|
|
28962
30112
|
var scheduleApp = new Hono27();
|
|
28963
|
-
function tokenOk2(
|
|
28964
|
-
return !!
|
|
30113
|
+
function tokenOk2(token6) {
|
|
30114
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 10;
|
|
28965
30115
|
}
|
|
28966
30116
|
function jsonError2(message, status = 400) {
|
|
28967
30117
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
28968
30118
|
}
|
|
28969
30119
|
scheduleApp.get("/api/status", async (c) => {
|
|
28970
|
-
const
|
|
28971
|
-
if (!tokenOk2(
|
|
28972
|
-
const r = await memoryCall("getScheduleStatusTool", {},
|
|
30120
|
+
const token6 = c.req.query("token");
|
|
30121
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30122
|
+
const r = await memoryCall("getScheduleStatusTool", {}, token6);
|
|
28973
30123
|
return c.json(r);
|
|
28974
30124
|
});
|
|
28975
30125
|
scheduleApp.get("/api/vaults", async (c) => {
|
|
28976
|
-
const
|
|
28977
|
-
if (!tokenOk2(
|
|
28978
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
30126
|
+
const token6 = c.req.query("token");
|
|
30127
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30128
|
+
const r = await memoryCall("listVaultsTool", {}, token6);
|
|
28979
30129
|
if (!r.ok) return c.json(r);
|
|
28980
30130
|
const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
|
|
28981
30131
|
return c.json({ ok: true, vaults });
|
|
28982
30132
|
});
|
|
28983
30133
|
scheduleApp.get("/api/actions", async (c) => {
|
|
28984
|
-
const
|
|
28985
|
-
if (!tokenOk2(
|
|
28986
|
-
const r = await memoryCall("listScheduledActionsTool", {},
|
|
30134
|
+
const token6 = c.req.query("token");
|
|
30135
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30136
|
+
const r = await memoryCall("listScheduledActionsTool", {}, token6);
|
|
28987
30137
|
return c.json(r);
|
|
28988
30138
|
});
|
|
28989
30139
|
scheduleApp.post("/api/propose", async (c) => {
|
|
@@ -29019,13 +30169,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
|
|
|
29019
30169
|
return c.json(r);
|
|
29020
30170
|
});
|
|
29021
30171
|
scheduleApp.get("/:token", (c) => {
|
|
29022
|
-
const
|
|
29023
|
-
if (!tokenOk2(
|
|
30172
|
+
const token6 = c.req.param("token");
|
|
30173
|
+
if (!tokenOk2(token6)) {
|
|
29024
30174
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
|
|
29025
30175
|
}
|
|
29026
|
-
return c.html(renderSchedulePage(
|
|
30176
|
+
return c.html(renderSchedulePage(token6));
|
|
29027
30177
|
});
|
|
29028
|
-
function renderSchedulePage(
|
|
30178
|
+
function renderSchedulePage(token6) {
|
|
29029
30179
|
return `<!doctype html>
|
|
29030
30180
|
<html>
|
|
29031
30181
|
<head>
|
|
@@ -29123,7 +30273,7 @@ function renderSchedulePage(token4) {
|
|
|
29123
30273
|
</div>
|
|
29124
30274
|
|
|
29125
30275
|
<script>
|
|
29126
|
-
const TOKEN = ${JSON.stringify(
|
|
30276
|
+
const TOKEN = ${JSON.stringify(token6)};
|
|
29127
30277
|
const state = { actions: [], vaults: [] };
|
|
29128
30278
|
|
|
29129
30279
|
function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
|
|
@@ -29366,8 +30516,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
|
|
|
29366
30516
|
return shortHash(`${customerEmail.trim().toLowerCase()}
|
|
29367
30517
|
${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
|
|
29368
30518
|
}
|
|
29369
|
-
function supportThreadAddress(
|
|
29370
|
-
const safeToken =
|
|
30519
|
+
function supportThreadAddress(token6, receivingDomain) {
|
|
30520
|
+
const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
29371
30521
|
const domain = receivingDomain.trim().toLowerCase();
|
|
29372
30522
|
if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
|
|
29373
30523
|
return `support+${safeToken}@${domain}`;
|
|
@@ -29394,8 +30544,8 @@ function personPath(identity) {
|
|
|
29394
30544
|
const local = identity.email.split("@")[0];
|
|
29395
30545
|
return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
|
|
29396
30546
|
}
|
|
29397
|
-
function issuePath(
|
|
29398
|
-
return `support/threads/${
|
|
30547
|
+
function issuePath(token6) {
|
|
30548
|
+
return `support/threads/${token6}`;
|
|
29399
30549
|
}
|
|
29400
30550
|
function communicationPath(email) {
|
|
29401
30551
|
const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
|
|
@@ -29579,8 +30729,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
|
|
|
29579
30729
|
`- Person: ${noteRef("People", personPathValue)}`
|
|
29580
30730
|
].join("\n");
|
|
29581
30731
|
}
|
|
29582
|
-
async function findOrCreateIssue(memoryKey, email, customer, personPathValue,
|
|
29583
|
-
const path5 = issuePath(
|
|
30732
|
+
async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token6, threadAddressValue) {
|
|
30733
|
+
const path5 = issuePath(token6);
|
|
29584
30734
|
const existing = await getNote(memoryKey, "Issues", path5);
|
|
29585
30735
|
if (existing) return existing;
|
|
29586
30736
|
const subject = normalizeSupportSubject(email.subject);
|
|
@@ -29590,7 +30740,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29590
30740
|
path: path5,
|
|
29591
30741
|
title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
|
|
29592
30742
|
content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
|
|
29593
|
-
source: `resend:thread:${
|
|
30743
|
+
source: `resend:thread:${token6}`,
|
|
29594
30744
|
type: "user_friction",
|
|
29595
30745
|
tagCandidates: ISSUE_TAGS,
|
|
29596
30746
|
props: {
|
|
@@ -29599,7 +30749,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29599
30749
|
summary,
|
|
29600
30750
|
pinned: false,
|
|
29601
30751
|
source_type: "thread",
|
|
29602
|
-
source_ref: `resend:thread:${
|
|
30752
|
+
source_ref: `resend:thread:${token6}`,
|
|
29603
30753
|
related: [],
|
|
29604
30754
|
related_vault_notes: [noteRef("People", personPathValue)],
|
|
29605
30755
|
embed: true,
|
|
@@ -29614,7 +30764,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29614
30764
|
customer_email: customer.email,
|
|
29615
30765
|
customer_name: customer.name,
|
|
29616
30766
|
normalized_subject: subject,
|
|
29617
|
-
thread_token:
|
|
30767
|
+
thread_token: token6,
|
|
29618
30768
|
thread_address: threadAddressValue,
|
|
29619
30769
|
communication_refs: [],
|
|
29620
30770
|
message_count: 0,
|
|
@@ -29657,7 +30807,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
|
|
|
29657
30807
|
...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
|
|
29658
30808
|
].join("\n");
|
|
29659
30809
|
}
|
|
29660
|
-
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue,
|
|
30810
|
+
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token6, parentPath) {
|
|
29661
30811
|
const path5 = communicationPath(email);
|
|
29662
30812
|
const existing = await getNote(memoryKey, "Communication", path5);
|
|
29663
30813
|
if (existing) return existing;
|
|
@@ -29696,8 +30846,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
|
|
|
29696
30846
|
provider: "resend",
|
|
29697
30847
|
provider_connection_ref: "resend:updates.mcpscraper.dev",
|
|
29698
30848
|
provider_message_ref: providerMessageRef,
|
|
29699
|
-
provider_thread_ref:
|
|
29700
|
-
thread_ref:
|
|
30849
|
+
provider_thread_ref: token6,
|
|
30850
|
+
thread_ref: token6,
|
|
29701
30851
|
issue_ref: issuePathValue,
|
|
29702
30852
|
sender_email: supportSender(email)?.email ?? email.from,
|
|
29703
30853
|
recipient_emails: email.to ?? [],
|
|
@@ -29757,8 +30907,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
|
|
|
29757
30907
|
if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
|
|
29758
30908
|
}
|
|
29759
30909
|
}
|
|
29760
|
-
async function getSupportThreadByToken(memoryKey,
|
|
29761
|
-
const safeToken =
|
|
30910
|
+
async function getSupportThreadByToken(memoryKey, token6) {
|
|
30911
|
+
const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
29762
30912
|
if (safeToken.length !== 24) return null;
|
|
29763
30913
|
const path5 = issuePath(safeToken);
|
|
29764
30914
|
const issue = await getNote(memoryKey, "Issues", path5);
|
|
@@ -29786,8 +30936,8 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29786
30936
|
const customer = options.customer ?? supportSender(email);
|
|
29787
30937
|
if (!customer) throw new Error("support sender does not contain a usable email address");
|
|
29788
30938
|
const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
|
|
29789
|
-
const
|
|
29790
|
-
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(
|
|
30939
|
+
const token6 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
|
|
30940
|
+
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token6, options.receivingDomain);
|
|
29791
30941
|
const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
|
|
29792
30942
|
if (!person) throw new Error("support person could not be resolved");
|
|
29793
30943
|
const issue = options.existingThread?.issue ?? await findOrCreateIssue(
|
|
@@ -29795,7 +30945,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29795
30945
|
email,
|
|
29796
30946
|
customer,
|
|
29797
30947
|
person.path,
|
|
29798
|
-
|
|
30948
|
+
token6,
|
|
29799
30949
|
threadAddressValue
|
|
29800
30950
|
);
|
|
29801
30951
|
const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
|
|
@@ -29806,7 +30956,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29806
30956
|
options.direction,
|
|
29807
30957
|
person.path,
|
|
29808
30958
|
issue.path,
|
|
29809
|
-
|
|
30959
|
+
token6,
|
|
29810
30960
|
parentPath
|
|
29811
30961
|
);
|
|
29812
30962
|
await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
|
|
@@ -29815,7 +30965,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29815
30965
|
communicationPath: communication.path,
|
|
29816
30966
|
personPath: person.path,
|
|
29817
30967
|
parentCommunicationPath: parentPath,
|
|
29818
|
-
threadToken:
|
|
30968
|
+
threadToken: token6,
|
|
29819
30969
|
threadAddress: threadAddressValue,
|
|
29820
30970
|
customerEmail: customer.email,
|
|
29821
30971
|
normalizedSubject,
|
|
@@ -29988,8 +31138,8 @@ resendInboundApp.post("/webhooks", async (c) => {
|
|
|
29988
31138
|
threadAddress: captured.threadAddress
|
|
29989
31139
|
});
|
|
29990
31140
|
}
|
|
29991
|
-
const
|
|
29992
|
-
const thread =
|
|
31141
|
+
const token6 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
|
|
31142
|
+
const thread = token6 ? await getSupportThreadByToken(memoryKey, token6) : null;
|
|
29993
31143
|
if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
|
|
29994
31144
|
const operator = parseMailbox(destination);
|
|
29995
31145
|
if (operator && sender.email === operator.email) {
|
|
@@ -30275,8 +31425,8 @@ var workshopFieldNotes = [
|
|
|
30275
31425
|
function escapeHtml3(value) {
|
|
30276
31426
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
30277
31427
|
}
|
|
30278
|
-
function workshopFieldNoteUrl(
|
|
30279
|
-
return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(
|
|
31428
|
+
function workshopFieldNoteUrl(slug4) {
|
|
31429
|
+
return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug4)}`;
|
|
30280
31430
|
}
|
|
30281
31431
|
function renderWorkshopFieldNotesBody() {
|
|
30282
31432
|
let activeCategory = "";
|
|
@@ -30835,16 +31985,16 @@ import { z as z31 } from "zod";
|
|
|
30835
31985
|
// src/editorial-reading-room/render.ts
|
|
30836
31986
|
import { createHash as createHash11 } from "crypto";
|
|
30837
31987
|
import { readFileSync as readFileSync3 } from "fs";
|
|
30838
|
-
import { dirname as dirname3, join as
|
|
31988
|
+
import { dirname as dirname3, join as join7, resolve } from "path";
|
|
30839
31989
|
import { Marked, Renderer } from "marked";
|
|
30840
31990
|
var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
|
|
30841
31991
|
var assetCache = /* @__PURE__ */ new Map();
|
|
30842
31992
|
function assetPath(name) {
|
|
30843
31993
|
const candidates = [
|
|
30844
|
-
|
|
30845
|
-
|
|
30846
|
-
|
|
30847
|
-
|
|
31994
|
+
join7(runtimeEntryDir, "editorial-reading-room", "assets", name),
|
|
31995
|
+
join7(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
|
|
31996
|
+
join7(process.cwd(), "src", "editorial-reading-room", "assets", name),
|
|
31997
|
+
join7(process.cwd(), "dist", "editorial-reading-room", "assets", name)
|
|
30848
31998
|
];
|
|
30849
31999
|
for (const candidate of candidates) {
|
|
30850
32000
|
try {
|
|
@@ -30918,8 +32068,8 @@ function addHeadingIds(html) {
|
|
|
30918
32068
|
});
|
|
30919
32069
|
return { rendered, toc };
|
|
30920
32070
|
}
|
|
30921
|
-
function safeFilename(requested,
|
|
30922
|
-
const stem = (requested ||
|
|
32071
|
+
function safeFilename(requested, slug4) {
|
|
32072
|
+
const stem = (requested || slug4).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
|
|
30923
32073
|
return `${stem}.html`;
|
|
30924
32074
|
}
|
|
30925
32075
|
function escapedJsonForHtml(value) {
|
|
@@ -31075,20 +32225,20 @@ ${provenance}`;
|
|
|
31075
32225
|
var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
|
|
31076
32226
|
var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
31077
32227
|
var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
31078
|
-
function
|
|
32228
|
+
function token3() {
|
|
31079
32229
|
return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
31080
32230
|
}
|
|
31081
|
-
function
|
|
32231
|
+
function policy3() {
|
|
31082
32232
|
return {
|
|
31083
32233
|
prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
|
|
31084
32234
|
artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
|
|
31085
32235
|
downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
|
|
31086
|
-
token:
|
|
32236
|
+
token: token3()
|
|
31087
32237
|
};
|
|
31088
32238
|
}
|
|
31089
32239
|
function createEditorialReadingRoomArtifact(args) {
|
|
31090
32240
|
return createPrivateArtifact({
|
|
31091
|
-
policy:
|
|
32241
|
+
policy: policy3(),
|
|
31092
32242
|
ownerId: args.ownerId,
|
|
31093
32243
|
artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
|
|
31094
32244
|
createdAt: args.rendered.generatedAt,
|
|
@@ -31099,7 +32249,7 @@ function createEditorialReadingRoomArtifact(args) {
|
|
|
31099
32249
|
}
|
|
31100
32250
|
function renewEditorialReadingRoomArtifactDownload(args) {
|
|
31101
32251
|
return renewPrivateArtifactDownload({
|
|
31102
|
-
policy:
|
|
32252
|
+
policy: policy3(),
|
|
31103
32253
|
ownerId: args.ownerId,
|
|
31104
32254
|
artifactId: args.artifactId
|
|
31105
32255
|
});
|
|
@@ -31879,7 +33029,7 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
31879
33029
|
async function prepareCommonsEntity(input, user) {
|
|
31880
33030
|
await ensureCommonsSchema();
|
|
31881
33031
|
const title = optionalText2(input.title, 180) ?? "";
|
|
31882
|
-
const
|
|
33032
|
+
const slug4 = normalizeSlug(input.slug || title);
|
|
31883
33033
|
const schemaOrgType = optionalText2(input.schemaOrgType, 80);
|
|
31884
33034
|
const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
|
|
31885
33035
|
const entityType = contract.entityType;
|
|
@@ -31887,7 +33037,7 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31887
33037
|
const headings = extractMarkdownHeadings(bodyMarkdown);
|
|
31888
33038
|
const duplicateCandidates = await potentialDuplicateCandidates({
|
|
31889
33039
|
title,
|
|
31890
|
-
slug:
|
|
33040
|
+
slug: slug4,
|
|
31891
33041
|
source: normalizeSource(input.source),
|
|
31892
33042
|
maxCandidates: clampNumber(input.maxCandidates, 1, 20, 8)
|
|
31893
33043
|
});
|
|
@@ -31904,8 +33054,8 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31904
33054
|
entityIdPrefix: "TPW-Q",
|
|
31905
33055
|
userId: user?.id ?? null,
|
|
31906
33056
|
normalizedEntityType: entityType,
|
|
31907
|
-
proposedSlug:
|
|
31908
|
-
publicUrl:
|
|
33057
|
+
proposedSlug: slug4 || null,
|
|
33058
|
+
publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
|
|
31909
33059
|
writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
|
|
31910
33060
|
},
|
|
31911
33061
|
contract,
|
|
@@ -31940,7 +33090,7 @@ async function validateCommonsEntity(input, user) {
|
|
|
31940
33090
|
const warnings = [];
|
|
31941
33091
|
const title = optionalText2(input.title, 180);
|
|
31942
33092
|
const description = optionalText2(input.description, 600);
|
|
31943
|
-
const
|
|
33093
|
+
const slug4 = normalizeSlug(input.slug || title || "");
|
|
31944
33094
|
const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
|
|
31945
33095
|
const source = normalizeSource(input.source);
|
|
31946
33096
|
const featuredImage = normalizeFeaturedImage(input.featuredImage);
|
|
@@ -31949,11 +33099,11 @@ async function validateCommonsEntity(input, user) {
|
|
|
31949
33099
|
const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
|
|
31950
33100
|
const duplicateCandidates = await potentialDuplicateCandidates({
|
|
31951
33101
|
title: title ?? "",
|
|
31952
|
-
slug:
|
|
33102
|
+
slug: slug4,
|
|
31953
33103
|
source,
|
|
31954
33104
|
maxCandidates: 8
|
|
31955
33105
|
});
|
|
31956
|
-
const existing =
|
|
33106
|
+
const existing = slug4 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug4, input.entityId) : null;
|
|
31957
33107
|
try {
|
|
31958
33108
|
normalizeCommonsClaims(input.claims);
|
|
31959
33109
|
} catch (error) {
|
|
@@ -31961,7 +33111,7 @@ async function validateCommonsEntity(input, user) {
|
|
|
31961
33111
|
}
|
|
31962
33112
|
if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
|
|
31963
33113
|
if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
|
|
31964
|
-
if (!
|
|
33114
|
+
if (!slug4) errors.push("A Commons entity needs a URL slug.");
|
|
31965
33115
|
if (!featuredImage?.url) errors.push("A featured image is required before a Commons entity can publish.");
|
|
31966
33116
|
if (!bodyMarkdown.trim() && !source.originalUrl && !source.resolvedUrl) {
|
|
31967
33117
|
errors.push("A public entity needs article body text or source evidence before publishing.");
|
|
@@ -32422,13 +33572,13 @@ async function findProposalByIdempotency(userId, idempotencyKey3) {
|
|
|
32422
33572
|
});
|
|
32423
33573
|
return result.rows[0] ? rowToProposal(result.rows[0]) : null;
|
|
32424
33574
|
}
|
|
32425
|
-
async function findEntityBySlugOrIdAnyStatus(
|
|
33575
|
+
async function findEntityBySlugOrIdAnyStatus(slug4, entityId) {
|
|
32426
33576
|
const result = entityId ? await getDb().execute({
|
|
32427
33577
|
sql: "SELECT * FROM commons_entities WHERE slug = ? OR id = ? LIMIT 1",
|
|
32428
|
-
args: [
|
|
33578
|
+
args: [slug4, entityId]
|
|
32429
33579
|
}) : await getDb().execute({
|
|
32430
33580
|
sql: "SELECT * FROM commons_entities WHERE slug = ? LIMIT 1",
|
|
32431
|
-
args: [
|
|
33581
|
+
args: [slug4]
|
|
32432
33582
|
});
|
|
32433
33583
|
return result.rows[0] ? rowToEntity(result.rows[0]) : null;
|
|
32434
33584
|
}
|
|
@@ -32620,13 +33770,13 @@ function canonicalizeSourceUrl(value) {
|
|
|
32620
33770
|
function normalizeSubmitInput(input) {
|
|
32621
33771
|
const title = requiredText(input.title, "title", 3, 180);
|
|
32622
33772
|
const description = requiredText(input.description, "description", 8, 600);
|
|
32623
|
-
const
|
|
32624
|
-
if (!
|
|
33773
|
+
const slug4 = normalizeSlug(input.slug || title);
|
|
33774
|
+
if (!slug4) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
|
|
32625
33775
|
return {
|
|
32626
33776
|
idempotencyKey: requiredText(input.idempotencyKey, "idempotencyKey", 8, 200),
|
|
32627
33777
|
title,
|
|
32628
33778
|
description,
|
|
32629
|
-
slug:
|
|
33779
|
+
slug: slug4,
|
|
32630
33780
|
entityId: normalizeEntityId(input.entityId),
|
|
32631
33781
|
entityType: normalizeEntityType(input.entityType),
|
|
32632
33782
|
disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
|
|
@@ -33175,13 +34325,13 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
33175
34325
|
return relatedLinks.map((link) => {
|
|
33176
34326
|
const title = requiredText(link.title, "relatedLinks.title", 1, 180);
|
|
33177
34327
|
const url = optionalText2(link.url, 2e3);
|
|
33178
|
-
const
|
|
34328
|
+
const slug4 = link.slug ? normalizeSlug(link.slug) : void 0;
|
|
33179
34329
|
const entityId = normalizeEntityId(link.entityId);
|
|
33180
|
-
const needsLink = link.needsLink === true || !url && !
|
|
34330
|
+
const needsLink = link.needsLink === true || !url && !slug4 && !entityId;
|
|
33181
34331
|
return {
|
|
33182
34332
|
title,
|
|
33183
34333
|
url,
|
|
33184
|
-
slug:
|
|
34334
|
+
slug: slug4,
|
|
33185
34335
|
entityId,
|
|
33186
34336
|
relationship: optionalText2(link.relationship, 120) ?? void 0,
|
|
33187
34337
|
summary: optionalText2(link.summary, 300) ?? void 0,
|
|
@@ -33412,8 +34562,8 @@ function actorLabel(input, user) {
|
|
|
33412
34562
|
function publicBaseUrl() {
|
|
33413
34563
|
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
|
|
33414
34564
|
}
|
|
33415
|
-
function publicUrlForSlug(
|
|
33416
|
-
return `${publicBaseUrl()}/wiki/${encodeURIComponent(
|
|
34565
|
+
function publicUrlForSlug(slug4) {
|
|
34566
|
+
return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug4)}`;
|
|
33417
34567
|
}
|
|
33418
34568
|
function allocateEntityId() {
|
|
33419
34569
|
return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
|
|
@@ -33615,11 +34765,11 @@ async function validateCommonsPublication(input, user) {
|
|
|
33615
34765
|
} catch (error) {
|
|
33616
34766
|
errors.push(error instanceof Error ? error.message : String(error));
|
|
33617
34767
|
}
|
|
33618
|
-
const
|
|
33619
|
-
const latest = publication ? await getLatestEdition(publication.id,
|
|
33620
|
-
if (latest && input.baseRevision === void 0) errors.push(`Edition ${
|
|
34768
|
+
const slug4 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
|
|
34769
|
+
const latest = publication ? await getLatestEdition(publication.id, slug4) : null;
|
|
34770
|
+
if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug4} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
|
|
33621
34771
|
if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
|
|
33622
|
-
errors.push(`Edition ${
|
|
34772
|
+
errors.push(`Edition ${slug4} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
|
|
33623
34773
|
}
|
|
33624
34774
|
}
|
|
33625
34775
|
}
|
|
@@ -34368,7 +35518,7 @@ import Papa2 from "papaparse";
|
|
|
34368
35518
|
import {
|
|
34369
35519
|
createHash as createHash15,
|
|
34370
35520
|
createHmac as createHmac3,
|
|
34371
|
-
randomBytes as
|
|
35521
|
+
randomBytes as randomBytes4,
|
|
34372
35522
|
timingSafeEqual as timingSafeEqual2
|
|
34373
35523
|
} from "crypto";
|
|
34374
35524
|
import { z as z33 } from "zod";
|
|
@@ -34580,7 +35730,7 @@ function dashboardCallbackUrl() {
|
|
|
34580
35730
|
}
|
|
34581
35731
|
async function createThorbitConnectUrl(userId) {
|
|
34582
35732
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
34583
|
-
const state =
|
|
35733
|
+
const state = randomBytes4(32).toString("base64url");
|
|
34584
35734
|
const stateHash = createHash15("sha256").update(state).digest("hex");
|
|
34585
35735
|
await getAnalyticsPool().query(
|
|
34586
35736
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
@@ -34593,8 +35743,8 @@ async function createThorbitConnectUrl(userId) {
|
|
|
34593
35743
|
url.searchParams.set("state", state);
|
|
34594
35744
|
return url.toString();
|
|
34595
35745
|
}
|
|
34596
|
-
function verifyThorbitAssertion(
|
|
34597
|
-
const [encoded, signature] =
|
|
35746
|
+
function verifyThorbitAssertion(token6) {
|
|
35747
|
+
const [encoded, signature] = token6.split(".");
|
|
34598
35748
|
if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
|
|
34599
35749
|
const expected = createHmac3("sha256", bridgeSecret()).update(encoded).digest();
|
|
34600
35750
|
const actual = Buffer.from(signature, "base64url");
|
|
@@ -36072,7 +37222,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
36072
37222
|
}
|
|
36073
37223
|
|
|
36074
37224
|
// src/api/scheduled-run-view-links.ts
|
|
36075
|
-
import { createHash as createHash18, randomBytes as
|
|
37225
|
+
import { createHash as createHash18, randomBytes as randomBytes5, randomUUID as randomUUID17 } from "crypto";
|
|
36076
37226
|
var schemaReady2 = false;
|
|
36077
37227
|
async function ensureScheduledRunViewLinksSchema() {
|
|
36078
37228
|
if (schemaReady2) return;
|
|
@@ -36125,8 +37275,8 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
36125
37275
|
`);
|
|
36126
37276
|
schemaReady2 = true;
|
|
36127
37277
|
}
|
|
36128
|
-
function tokenHash2(
|
|
36129
|
-
return createHash18("sha256").update(
|
|
37278
|
+
function tokenHash2(token6) {
|
|
37279
|
+
return createHash18("sha256").update(token6).digest("hex");
|
|
36130
37280
|
}
|
|
36131
37281
|
function mapRow(row) {
|
|
36132
37282
|
return {
|
|
@@ -36146,7 +37296,7 @@ function mapRow(row) {
|
|
|
36146
37296
|
async function createScheduledRunViewLink(input) {
|
|
36147
37297
|
await ensureScheduledRunViewLinksSchema();
|
|
36148
37298
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
36149
|
-
const
|
|
37299
|
+
const token6 = randomBytes5(32).toString("base64url");
|
|
36150
37300
|
const record = {
|
|
36151
37301
|
shareId: randomUUID17(),
|
|
36152
37302
|
ownerId: input.ownerId,
|
|
@@ -36155,7 +37305,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
36155
37305
|
title: input.title.slice(0, 300),
|
|
36156
37306
|
summary: input.summary.slice(0, 2e3),
|
|
36157
37307
|
rendererKey: input.rendererKey,
|
|
36158
|
-
tokenHash: tokenHash2(
|
|
37308
|
+
tokenHash: tokenHash2(token6),
|
|
36159
37309
|
expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
|
|
36160
37310
|
revokedAt: null,
|
|
36161
37311
|
createdAt: now.toISOString()
|
|
@@ -36180,9 +37330,9 @@ async function createScheduledRunViewLink(input) {
|
|
|
36180
37330
|
record.createdAt
|
|
36181
37331
|
]
|
|
36182
37332
|
});
|
|
36183
|
-
return { record, token:
|
|
37333
|
+
return { record, token: token6 };
|
|
36184
37334
|
}
|
|
36185
|
-
async function resolveScheduledRunViewLink(
|
|
37335
|
+
async function resolveScheduledRunViewLink(token6, now = /* @__PURE__ */ new Date()) {
|
|
36186
37336
|
await ensureScheduledRunViewLinksSchema();
|
|
36187
37337
|
const result = await getDb().execute({
|
|
36188
37338
|
sql: `
|
|
@@ -36194,7 +37344,7 @@ async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Dat
|
|
|
36194
37344
|
AND expires_at > ?
|
|
36195
37345
|
LIMIT 1
|
|
36196
37346
|
`,
|
|
36197
|
-
args: [tokenHash2(
|
|
37347
|
+
args: [tokenHash2(token6), now.toISOString()]
|
|
36198
37348
|
});
|
|
36199
37349
|
const row = result.rows[0];
|
|
36200
37350
|
return row ? mapRow(row) : null;
|
|
@@ -36609,7 +37759,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
36609
37759
|
const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
|
|
36610
37760
|
if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
|
|
36611
37761
|
const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
|
|
36612
|
-
const { record, token:
|
|
37762
|
+
const { record, token: token6 } = await createScheduledRunViewLink({
|
|
36613
37763
|
ownerId,
|
|
36614
37764
|
runId: result.data.run.runId,
|
|
36615
37765
|
artifactId: artifact.artifactId,
|
|
@@ -36624,7 +37774,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
36624
37774
|
shareId: record.shareId,
|
|
36625
37775
|
runId: record.runId,
|
|
36626
37776
|
artifactId: record.artifactId,
|
|
36627
|
-
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${
|
|
37777
|
+
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token6}`,
|
|
36628
37778
|
expiresAt: record.expiresAt,
|
|
36629
37779
|
createdAt: record.createdAt
|
|
36630
37780
|
}, 201);
|
|
@@ -36650,15 +37800,15 @@ import { createHash as createHash21 } from "crypto";
|
|
|
36650
37800
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
36651
37801
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
36652
37802
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
36653
|
-
function
|
|
37803
|
+
function token4() {
|
|
36654
37804
|
return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
36655
37805
|
}
|
|
36656
|
-
function
|
|
37806
|
+
function policy4() {
|
|
36657
37807
|
return {
|
|
36658
37808
|
prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
|
|
36659
37809
|
artifactTtlMs: null,
|
|
36660
37810
|
downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
|
|
36661
|
-
token:
|
|
37811
|
+
token: token4()
|
|
36662
37812
|
};
|
|
36663
37813
|
}
|
|
36664
37814
|
function runStorageSegment(runId) {
|
|
@@ -36669,7 +37819,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
36669
37819
|
throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
|
|
36670
37820
|
}
|
|
36671
37821
|
const stored = await createPrivateArtifact({
|
|
36672
|
-
policy:
|
|
37822
|
+
policy: policy4(),
|
|
36673
37823
|
ownerId: args.request.ownerId,
|
|
36674
37824
|
scopeSegments: [runStorageSegment(args.request.runId)],
|
|
36675
37825
|
artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
|
|
@@ -36695,7 +37845,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
36695
37845
|
async function readScheduledRunArtifact(args) {
|
|
36696
37846
|
if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
|
|
36697
37847
|
const window2 = await readPrivateArtifactWindow({
|
|
36698
|
-
policy:
|
|
37848
|
+
policy: policy4(),
|
|
36699
37849
|
artifactId: args.artifactId,
|
|
36700
37850
|
offset: 0,
|
|
36701
37851
|
maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
|
|
@@ -37258,6 +38408,282 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
37258
38408
|
return input.background === true || input.downloadImages === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
|
|
37259
38409
|
}
|
|
37260
38410
|
|
|
38411
|
+
// src/api/page-media-artifacts.ts
|
|
38412
|
+
import { createHash as createHash23, randomBytes as randomBytes6 } from "crypto";
|
|
38413
|
+
import { homedir as homedir3 } from "os";
|
|
38414
|
+
import { basename, extname, join as join8 } from "path";
|
|
38415
|
+
import pLimit6 from "p-limit";
|
|
38416
|
+
import { ZipFile as ZipFile2 } from "yazl";
|
|
38417
|
+
var PAGE_MEDIA_ARTIFACT_PREFIX = "page-media/";
|
|
38418
|
+
var PAGE_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
38419
|
+
var PAGE_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
38420
|
+
var MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
38421
|
+
var MAX_ARCHIVE_MEDIA_BYTES = 45 * 1024 * 1024;
|
|
38422
|
+
var MAX_INLINE_IMAGE_BYTES2 = 15e5;
|
|
38423
|
+
var MAX_INLINE_TOTAL_BYTES2 = 25e5;
|
|
38424
|
+
var DOWNLOAD_CONCURRENCY2 = 6;
|
|
38425
|
+
var MAX_REDIRECTS2 = 5;
|
|
38426
|
+
function token5() {
|
|
38427
|
+
return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
38428
|
+
}
|
|
38429
|
+
function policy5() {
|
|
38430
|
+
return {
|
|
38431
|
+
prefix: PAGE_MEDIA_ARTIFACT_PREFIX,
|
|
38432
|
+
artifactTtlMs: PAGE_MEDIA_ARTIFACT_TTL_MS,
|
|
38433
|
+
downloadTtlMs: PAGE_MEDIA_DOWNLOAD_TTL_MS,
|
|
38434
|
+
token: token5()
|
|
38435
|
+
};
|
|
38436
|
+
}
|
|
38437
|
+
function imageSignature(bytes, contentType) {
|
|
38438
|
+
if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
|
|
38439
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
|
|
38440
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
|
|
38441
|
+
if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
|
|
38442
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 12)).match(/^ftyp(?:avif|avis)$/)) return { mimeType: "image/avif", extension: "avif" };
|
|
38443
|
+
const prefix = Buffer.from(bytes.slice(0, Math.min(bytes.length, 1024))).toString("utf8").replace(/^\uFEFF/, "").trimStart();
|
|
38444
|
+
if ((contentType === "image/svg+xml" || prefix.startsWith("<svg") || /^<\?xml[\s\S]*?<svg/i.test(prefix)) && /<svg[\s>]/i.test(prefix)) {
|
|
38445
|
+
return { mimeType: "image/svg+xml", extension: "svg" };
|
|
38446
|
+
}
|
|
38447
|
+
return null;
|
|
38448
|
+
}
|
|
38449
|
+
function mediaSignature(bytes, contentType, expectedType) {
|
|
38450
|
+
if (expectedType === "image") return imageSignature(bytes, contentType);
|
|
38451
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 8)) === "ftyp") return { mimeType: "video/mp4", extension: "mp4" };
|
|
38452
|
+
if (bytes.length >= 4 && bytes[0] === 26 && bytes[1] === 69 && bytes[2] === 223 && bytes[3] === 163) return { mimeType: "video/webm", extension: "webm" };
|
|
38453
|
+
if (bytes.length >= 4 && String.fromCharCode(...bytes.slice(0, 4)) === "OggS") {
|
|
38454
|
+
return expectedType === "audio" ? { mimeType: "audio/ogg", extension: "ogg" } : { mimeType: "video/ogg", extension: "ogv" };
|
|
38455
|
+
}
|
|
38456
|
+
if (expectedType === "audio" && bytes.length >= 3 && String.fromCharCode(...bytes.slice(0, 3)) === "ID3") return { mimeType: "audio/mpeg", extension: "mp3" };
|
|
38457
|
+
if (contentType && typeFromMime(contentType) === expectedType) {
|
|
38458
|
+
const extension = contentType.split("/")[1]?.replace("jpeg", "jpg").replace("mpeg", "mp3").replace("svg+xml", "svg");
|
|
38459
|
+
if (extension && /^[a-z0-9]+$/i.test(extension)) return { mimeType: contentType, extension };
|
|
38460
|
+
}
|
|
38461
|
+
return null;
|
|
38462
|
+
}
|
|
38463
|
+
function imageDimensions(bytes, mimeType) {
|
|
38464
|
+
const buffer = Buffer.from(bytes);
|
|
38465
|
+
if (mimeType === "image/png" && buffer.length >= 24) return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
|
|
38466
|
+
if (mimeType === "image/gif" && buffer.length >= 10) return { width: buffer.readUInt16LE(6), height: buffer.readUInt16LE(8) };
|
|
38467
|
+
if (mimeType === "image/webp" && buffer.length >= 30) {
|
|
38468
|
+
const chunk = buffer.toString("ascii", 12, 16);
|
|
38469
|
+
if (chunk === "VP8X") return { width: 1 + buffer.readUIntLE(24, 3), height: 1 + buffer.readUIntLE(27, 3) };
|
|
38470
|
+
if (chunk === "VP8L" && buffer[20] === 47) {
|
|
38471
|
+
const bits = buffer.readUInt32LE(21);
|
|
38472
|
+
return { width: (bits & 16383) + 1, height: (bits >> 14 & 16383) + 1 };
|
|
38473
|
+
}
|
|
38474
|
+
}
|
|
38475
|
+
if (mimeType === "image/jpeg") {
|
|
38476
|
+
let offset = 2;
|
|
38477
|
+
while (offset + 9 < buffer.length) {
|
|
38478
|
+
if (buffer[offset] !== 255) {
|
|
38479
|
+
offset += 1;
|
|
38480
|
+
continue;
|
|
38481
|
+
}
|
|
38482
|
+
const marker = buffer[offset + 1];
|
|
38483
|
+
if (marker === 216 || marker === 217) {
|
|
38484
|
+
offset += 2;
|
|
38485
|
+
continue;
|
|
38486
|
+
}
|
|
38487
|
+
const length = buffer.readUInt16BE(offset + 2);
|
|
38488
|
+
if (length < 2 || offset + length + 2 > buffer.length) break;
|
|
38489
|
+
if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker)) {
|
|
38490
|
+
return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
|
|
38491
|
+
}
|
|
38492
|
+
offset += length + 2;
|
|
38493
|
+
}
|
|
38494
|
+
}
|
|
38495
|
+
if (mimeType === "image/svg+xml") {
|
|
38496
|
+
const source = buffer.toString("utf8", 0, Math.min(buffer.length, 64 * 1024));
|
|
38497
|
+
const tag = source.match(/<svg\b[^>]*>/i)?.[0] ?? "";
|
|
38498
|
+
const width = Number(tag.match(/\bwidth=["']?([\d.]+)/i)?.[1] ?? 0);
|
|
38499
|
+
const height = Number(tag.match(/\bheight=["']?([\d.]+)/i)?.[1] ?? 0);
|
|
38500
|
+
if (width > 0 && height > 0) return { width: Math.round(width), height: Math.round(height) };
|
|
38501
|
+
const viewBox = tag.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);
|
|
38502
|
+
if (viewBox?.length === 4 && viewBox[2] > 0 && viewBox[3] > 0) return { width: Math.round(viewBox[2]), height: Math.round(viewBox[3]) };
|
|
38503
|
+
}
|
|
38504
|
+
return null;
|
|
38505
|
+
}
|
|
38506
|
+
async function readBounded2(response) {
|
|
38507
|
+
const declared = Number(response.headers.get("content-length") ?? "");
|
|
38508
|
+
if (Number.isFinite(declared) && declared > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38509
|
+
if (!response.body) {
|
|
38510
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
38511
|
+
if (bytes.length > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38512
|
+
return bytes;
|
|
38513
|
+
}
|
|
38514
|
+
const reader = response.body.getReader();
|
|
38515
|
+
const chunks = [];
|
|
38516
|
+
let total = 0;
|
|
38517
|
+
try {
|
|
38518
|
+
for (; ; ) {
|
|
38519
|
+
const { done, value } = await reader.read();
|
|
38520
|
+
if (done) break;
|
|
38521
|
+
if (!value) continue;
|
|
38522
|
+
total += value.byteLength;
|
|
38523
|
+
if (total > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38524
|
+
chunks.push(Buffer.from(value));
|
|
38525
|
+
}
|
|
38526
|
+
} finally {
|
|
38527
|
+
reader.releaseLock();
|
|
38528
|
+
}
|
|
38529
|
+
return Buffer.concat(chunks);
|
|
38530
|
+
}
|
|
38531
|
+
async function fetchMedia(rawUrl, expectedType) {
|
|
38532
|
+
let target = rawUrl;
|
|
38533
|
+
for (let redirects = 0; redirects <= MAX_REDIRECTS2; redirects += 1) {
|
|
38534
|
+
const checked = await validatePublicHttpUrl(target, { field: "website media URL" });
|
|
38535
|
+
if (checked.error || !checked.parsed) throw new Error("media_url_rejected");
|
|
38536
|
+
const response = await fetch(checked.parsed.href, {
|
|
38537
|
+
redirect: "manual",
|
|
38538
|
+
headers: { Accept: expectedType === "image" ? "image/*" : `${expectedType}/*` },
|
|
38539
|
+
signal: AbortSignal.timeout(15e3)
|
|
38540
|
+
});
|
|
38541
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
38542
|
+
const location2 = response.headers.get("location");
|
|
38543
|
+
if (!location2 || redirects === MAX_REDIRECTS2) throw new Error("media_redirect_rejected");
|
|
38544
|
+
target = new URL(location2, checked.parsed).href;
|
|
38545
|
+
continue;
|
|
38546
|
+
}
|
|
38547
|
+
if (!response.ok) throw new Error(`media_fetch_${response.status}`);
|
|
38548
|
+
const bytes = await readBounded2(response);
|
|
38549
|
+
const declared = response.headers.get("content-type")?.split(";")[0].trim().toLowerCase() ?? null;
|
|
38550
|
+
const detected = mediaSignature(bytes, declared, expectedType);
|
|
38551
|
+
if (!detected || typeFromMime(detected.mimeType) !== expectedType) throw new Error("media_content_invalid");
|
|
38552
|
+
const dimensions = expectedType === "image" ? imageDimensions(bytes, detected.mimeType) : null;
|
|
38553
|
+
return {
|
|
38554
|
+
bytes,
|
|
38555
|
+
...detected,
|
|
38556
|
+
finalUrl: checked.parsed.href,
|
|
38557
|
+
width: dimensions?.width ?? null,
|
|
38558
|
+
height: dimensions?.height ?? null,
|
|
38559
|
+
sha256: createHash23("sha256").update(bytes).digest("hex")
|
|
38560
|
+
};
|
|
38561
|
+
}
|
|
38562
|
+
throw new Error("media_redirect_rejected");
|
|
38563
|
+
}
|
|
38564
|
+
function zipBuffer2(files) {
|
|
38565
|
+
return new Promise((resolve2, reject) => {
|
|
38566
|
+
const zip = new ZipFile2();
|
|
38567
|
+
const chunks = [];
|
|
38568
|
+
zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
|
38569
|
+
zip.outputStream.once("error", reject);
|
|
38570
|
+
zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
|
|
38571
|
+
for (const file of files) zip.addBuffer(file.content, file.path);
|
|
38572
|
+
zip.end();
|
|
38573
|
+
});
|
|
38574
|
+
}
|
|
38575
|
+
function slug3(value) {
|
|
38576
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "website";
|
|
38577
|
+
}
|
|
38578
|
+
function safeOriginalName(rawUrl) {
|
|
38579
|
+
try {
|
|
38580
|
+
const name = basename(new URL(rawUrl).pathname).replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
38581
|
+
return name.slice(0, 80);
|
|
38582
|
+
} catch {
|
|
38583
|
+
return "";
|
|
38584
|
+
}
|
|
38585
|
+
}
|
|
38586
|
+
function pageMediaArtifactOwnerId(artifactId) {
|
|
38587
|
+
return privateArtifactOwnerId(artifactId, PAGE_MEDIA_ARTIFACT_PREFIX);
|
|
38588
|
+
}
|
|
38589
|
+
async function readOwnedPageMediaArtifactBuffer(args) {
|
|
38590
|
+
if (pageMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
38591
|
+
return readPrivateArtifactBuffer({ policy: policy5(), artifactId: args.artifactId, maxBytes: 60 * 1024 * 1024 });
|
|
38592
|
+
}
|
|
38593
|
+
async function packagePageMedia(args) {
|
|
38594
|
+
const files = [];
|
|
38595
|
+
const limit = pLimit6(DOWNLOAD_CONCURRENCY2);
|
|
38596
|
+
const digestToFilename = /* @__PURE__ */ new Map();
|
|
38597
|
+
let storedBytes = 0;
|
|
38598
|
+
let inlineBytes = 0;
|
|
38599
|
+
let inlineCount = 0;
|
|
38600
|
+
await Promise.all(args.media.assets.map((asset, position) => limit(async () => {
|
|
38601
|
+
try {
|
|
38602
|
+
const fetched = await fetchMedia(asset.url, asset.type);
|
|
38603
|
+
asset.finalUrl = fetched.finalUrl;
|
|
38604
|
+
asset.mimeType = fetched.mimeType;
|
|
38605
|
+
asset.sizeBytes = fetched.bytes.length;
|
|
38606
|
+
asset.sha256 = fetched.sha256;
|
|
38607
|
+
asset.width = fetched.width ?? asset.width;
|
|
38608
|
+
asset.height = fetched.height ?? asset.height;
|
|
38609
|
+
const duplicate = digestToFilename.get(fetched.sha256);
|
|
38610
|
+
if (duplicate) {
|
|
38611
|
+
asset.filename = duplicate;
|
|
38612
|
+
asset.duplicateOf = duplicate;
|
|
38613
|
+
asset.downloadStatus = "downloaded";
|
|
38614
|
+
asset.downloadError = null;
|
|
38615
|
+
return;
|
|
38616
|
+
}
|
|
38617
|
+
if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_MEDIA_BYTES) throw new Error("archive_byte_limit_reached");
|
|
38618
|
+
storedBytes += fetched.bytes.length;
|
|
38619
|
+
const original = safeOriginalName(fetched.finalUrl);
|
|
38620
|
+
const extension = extname(original);
|
|
38621
|
+
const stem = original ? extension ? original.slice(0, -extension.length) : original : asset.type;
|
|
38622
|
+
const filename = `media/${String(position + 1).padStart(4, "0")}-${slug3(stem)}.${fetched.extension}`;
|
|
38623
|
+
digestToFilename.set(fetched.sha256, filename);
|
|
38624
|
+
files.push({ path: filename, content: fetched.bytes });
|
|
38625
|
+
asset.filename = filename;
|
|
38626
|
+
asset.duplicateOf = null;
|
|
38627
|
+
asset.downloadStatus = "downloaded";
|
|
38628
|
+
asset.downloadError = null;
|
|
38629
|
+
if (asset.type === "image" && inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES2 && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES2) {
|
|
38630
|
+
asset.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
|
|
38631
|
+
inlineBytes += fetched.bytes.length;
|
|
38632
|
+
inlineCount += 1;
|
|
38633
|
+
}
|
|
38634
|
+
} catch (error) {
|
|
38635
|
+
asset.finalUrl = null;
|
|
38636
|
+
asset.mimeType = null;
|
|
38637
|
+
asset.sizeBytes = null;
|
|
38638
|
+
asset.sha256 = null;
|
|
38639
|
+
asset.savedPath = null;
|
|
38640
|
+
asset.downloadStatus = "failed";
|
|
38641
|
+
asset.downloadError = error instanceof Error ? error.message : "media_download_failed";
|
|
38642
|
+
asset.inlinePreview = null;
|
|
38643
|
+
}
|
|
38644
|
+
})));
|
|
38645
|
+
const downloaded = args.media.assets.filter((asset) => asset.downloadStatus === "downloaded").length;
|
|
38646
|
+
const failed = args.media.assets.length - downloaded;
|
|
38647
|
+
if (failed > 0) args.media.warnings.push(`${failed} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`);
|
|
38648
|
+
if (downloaded === 0) return { media: args.media, artifact: null };
|
|
38649
|
+
const cleanAssets = args.media.assets.map(({ inlinePreview: _preview, ...asset }) => asset);
|
|
38650
|
+
const summary = {
|
|
38651
|
+
pageUrl: args.pageUrl,
|
|
38652
|
+
title: args.title,
|
|
38653
|
+
extractedAt: args.extractedAt,
|
|
38654
|
+
completeness: args.media.completeness,
|
|
38655
|
+
exhausted: args.media.exhausted,
|
|
38656
|
+
stopReason: args.media.stopReason,
|
|
38657
|
+
scrollRounds: args.media.scrollRounds,
|
|
38658
|
+
staticFound: args.media.staticFound,
|
|
38659
|
+
renderedFound: args.media.renderedFound,
|
|
38660
|
+
totalFound: args.media.totalFound,
|
|
38661
|
+
retainedAfterFilteringAndVariantCollapse: args.media.assets.length,
|
|
38662
|
+
downloaded,
|
|
38663
|
+
failed,
|
|
38664
|
+
notes: [
|
|
38665
|
+
"The inventory unions static-source and rendered-page discovery.",
|
|
38666
|
+
"complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.",
|
|
38667
|
+
"Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."
|
|
38668
|
+
]
|
|
38669
|
+
};
|
|
38670
|
+
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
38671
|
+
files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
|
|
38672
|
+
const archive = await zipBuffer2(files);
|
|
38673
|
+
const id = randomBytes6(6).toString("hex");
|
|
38674
|
+
const pointer = await createPrivateArtifact({
|
|
38675
|
+
policy: policy5(),
|
|
38676
|
+
ownerId: args.ownerId,
|
|
38677
|
+
artifactKey: `${id}.zip`,
|
|
38678
|
+
createdAt: new Date(args.extractedAt),
|
|
38679
|
+
filename: `${slug3(args.title || new URL(args.pageUrl).hostname)}-website-media.zip`,
|
|
38680
|
+
contentType: "application/zip",
|
|
38681
|
+
content: archive
|
|
38682
|
+
});
|
|
38683
|
+
const localPath = token5() ? null : join8(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join8(homedir3(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
38684
|
+
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
38685
|
+
}
|
|
38686
|
+
|
|
37261
38687
|
// src/api/site-extract-reconciliation.ts
|
|
37262
38688
|
async function redispatchFundedSiteExtractJobs(limit = 25) {
|
|
37263
38689
|
const jobs = await listFundedPendingExtractJobs(limit);
|
|
@@ -37323,13 +38749,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
37323
38749
|
}
|
|
37324
38750
|
|
|
37325
38751
|
// src/api/page-diff.ts
|
|
37326
|
-
import { createHash as
|
|
38752
|
+
import { createHash as createHash24 } from "crypto";
|
|
37327
38753
|
import { diffLines } from "diff";
|
|
37328
38754
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
37329
38755
|
var MAX_DIFF_HUNKS = 200;
|
|
37330
38756
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
37331
38757
|
function sha256Hex(value) {
|
|
37332
|
-
return
|
|
38758
|
+
return createHash24("sha256").update(value).digest("hex");
|
|
37333
38759
|
}
|
|
37334
38760
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
37335
38761
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -37422,8 +38848,8 @@ async function uploadFallbackBlob(opts) {
|
|
|
37422
38848
|
try {
|
|
37423
38849
|
const content = (opts.content ?? "").trim();
|
|
37424
38850
|
if (!content) return null;
|
|
37425
|
-
const
|
|
37426
|
-
const key = `${SCRAPE_FALLBACK_PREFIX}${
|
|
38851
|
+
const slug4 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
|
|
38852
|
+
const key = `${SCRAPE_FALLBACK_PREFIX}${slug4}.md`;
|
|
37427
38853
|
const doc = `# ${opts.title}
|
|
37428
38854
|
Source: ${opts.source}
|
|
37429
38855
|
|
|
@@ -37443,10 +38869,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
37443
38869
|
}
|
|
37444
38870
|
|
|
37445
38871
|
// src/api/scrape-image-sink.ts
|
|
37446
|
-
import { createHash as
|
|
38872
|
+
import { createHash as createHash25 } from "crypto";
|
|
37447
38873
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
37448
38874
|
function idempotencyKey2(userId, vault, input) {
|
|
37449
|
-
return `scrape-image-${
|
|
38875
|
+
return `scrape-image-${createHash25("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
37450
38876
|
}
|
|
37451
38877
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
37452
38878
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -37490,7 +38916,7 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
|
37490
38916
|
}
|
|
37491
38917
|
|
|
37492
38918
|
// src/api/archive-reader.ts
|
|
37493
|
-
import { basename, extname } from "path";
|
|
38919
|
+
import { basename as basename2, extname as extname2 } from "path";
|
|
37494
38920
|
import yauzl from "yauzl";
|
|
37495
38921
|
var MAX_ARCHIVE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
|
|
37496
38922
|
var MAX_ARCHIVE_ENTRIES = 1e4;
|
|
@@ -37545,9 +38971,9 @@ var ArchiveReadError = class extends Error {
|
|
|
37545
38971
|
retryable;
|
|
37546
38972
|
};
|
|
37547
38973
|
function readableContentType(path5) {
|
|
37548
|
-
const extensionType = READABLE_EXTENSIONS.get(
|
|
38974
|
+
const extensionType = READABLE_EXTENSIONS.get(extname2(path5).toLowerCase());
|
|
37549
38975
|
if (extensionType) return extensionType;
|
|
37550
|
-
return READABLE_FILENAMES.has(
|
|
38976
|
+
return READABLE_FILENAMES.has(basename2(path5).toLowerCase()) ? "text/plain" : null;
|
|
37551
38977
|
}
|
|
37552
38978
|
function safeEntryPath(path5) {
|
|
37553
38979
|
const normalized = path5.replaceAll("\\", "/");
|
|
@@ -37908,7 +39334,7 @@ function archiveLibrarySource(archiveUrl, path5) {
|
|
|
37908
39334
|
return source.href;
|
|
37909
39335
|
}
|
|
37910
39336
|
function archiveEntryTitle(path5) {
|
|
37911
|
-
return
|
|
39337
|
+
return basename2(path5).slice(0, 200) || "Archive entry";
|
|
37912
39338
|
}
|
|
37913
39339
|
function inferredWaybackCapturedAt(content) {
|
|
37914
39340
|
const timestamp = content.slice(0, 4e3).match(/^Archive timestamp:\s*(\d{14})\s*$/m)?.[1];
|
|
@@ -37918,7 +39344,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
37918
39344
|
}
|
|
37919
39345
|
|
|
37920
39346
|
// src/api/connection-memory-import.ts
|
|
37921
|
-
import { createHash as
|
|
39347
|
+
import { createHash as createHash26 } from "crypto";
|
|
37922
39348
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
37923
39349
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
37924
39350
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -37948,7 +39374,7 @@ function canonicalJson(value) {
|
|
|
37948
39374
|
return JSON.stringify(value);
|
|
37949
39375
|
}
|
|
37950
39376
|
function sha2562(value) {
|
|
37951
|
-
return
|
|
39377
|
+
return createHash26("sha256").update(value).digest("hex");
|
|
37952
39378
|
}
|
|
37953
39379
|
function sensitiveKey(key) {
|
|
37954
39380
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -38166,17 +39592,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
38166
39592
|
|
|
38167
39593
|
// src/api/scrape-blob-cleanup.ts
|
|
38168
39594
|
import { readdir, stat, unlink } from "fs/promises";
|
|
38169
|
-
import { homedir as
|
|
38170
|
-
import { join as
|
|
38171
|
-
async function cleanupVercel(
|
|
39595
|
+
import { homedir as homedir4 } from "os";
|
|
39596
|
+
import { join as join9 } from "path";
|
|
39597
|
+
async function cleanupVercel(token6, cutoff) {
|
|
38172
39598
|
const { list, del } = await import("@vercel/blob");
|
|
38173
39599
|
let cursor;
|
|
38174
39600
|
let deleted = 0;
|
|
38175
39601
|
do {
|
|
38176
|
-
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token:
|
|
39602
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token6 });
|
|
38177
39603
|
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
38178
39604
|
if (expired.length) {
|
|
38179
|
-
await del(expired.map((b) => b.url), { token:
|
|
39605
|
+
await del(expired.map((b) => b.url), { token: token6 });
|
|
38180
39606
|
deleted += expired.length;
|
|
38181
39607
|
}
|
|
38182
39608
|
cursor = res.hasMore ? res.cursor : void 0;
|
|
@@ -38184,8 +39610,8 @@ async function cleanupVercel(token4, cutoff) {
|
|
|
38184
39610
|
return { deleted, store: "vercel-blob" };
|
|
38185
39611
|
}
|
|
38186
39612
|
async function cleanupLocal(cutoff) {
|
|
38187
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
38188
|
-
const dir =
|
|
39613
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join9(homedir4(), "Downloads", "mcp-scraper");
|
|
39614
|
+
const dir = join9(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
38189
39615
|
let deleted = 0;
|
|
38190
39616
|
let entries;
|
|
38191
39617
|
try {
|
|
@@ -38194,7 +39620,7 @@ async function cleanupLocal(cutoff) {
|
|
|
38194
39620
|
return { deleted: 0, store: "local" };
|
|
38195
39621
|
}
|
|
38196
39622
|
for (const name of entries) {
|
|
38197
|
-
const path5 =
|
|
39623
|
+
const path5 = join9(dir, name);
|
|
38198
39624
|
try {
|
|
38199
39625
|
const s = await stat(path5);
|
|
38200
39626
|
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
@@ -38209,8 +39635,8 @@ async function cleanupLocal(cutoff) {
|
|
|
38209
39635
|
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
38210
39636
|
const cutoff = Date.now() - maxAgeMs;
|
|
38211
39637
|
try {
|
|
38212
|
-
const
|
|
38213
|
-
return
|
|
39638
|
+
const token6 = process.env.BLOB_READ_WRITE_TOKEN;
|
|
39639
|
+
return token6 ? await cleanupVercel(token6, cutoff) : await cleanupLocal(cutoff);
|
|
38214
39640
|
} catch {
|
|
38215
39641
|
return { deleted: 0, store: "none" };
|
|
38216
39642
|
}
|
|
@@ -38959,7 +40385,7 @@ async function dbUsage(identity, plan) {
|
|
|
38959
40385
|
}
|
|
38960
40386
|
|
|
38961
40387
|
// src/api/nango-control.ts
|
|
38962
|
-
import { createHash as
|
|
40388
|
+
import { createHash as createHash28, randomUUID as randomUUID22 } from "crypto";
|
|
38963
40389
|
|
|
38964
40390
|
// src/api/slack-archive-analysis.ts
|
|
38965
40391
|
function isRecord2(value) {
|
|
@@ -39333,7 +40759,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
39333
40759
|
}
|
|
39334
40760
|
|
|
39335
40761
|
// src/api/main-nango-transport.ts
|
|
39336
|
-
import { createHash as
|
|
40762
|
+
import { createHash as createHash27, randomUUID as randomUUID21 } from "crypto";
|
|
39337
40763
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
39338
40764
|
|
|
39339
40765
|
// src/api/connected-tool-policy.ts
|
|
@@ -39376,9 +40802,9 @@ function buildConnectedToolPolicyRegistry(args) {
|
|
|
39376
40802
|
return policies;
|
|
39377
40803
|
}
|
|
39378
40804
|
function connectedToolPolicyOrThrow(registry, toolName) {
|
|
39379
|
-
const
|
|
39380
|
-
if (!
|
|
39381
|
-
return
|
|
40805
|
+
const policy6 = registry.get(toolName);
|
|
40806
|
+
if (!policy6) throw new Error("connected_tool_policy_missing");
|
|
40807
|
+
return policy6;
|
|
39382
40808
|
}
|
|
39383
40809
|
|
|
39384
40810
|
// src/api/main-nango-transport.ts
|
|
@@ -39724,9 +41150,9 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
39724
41150
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39725
41151
|
actionsEnabled: connection.actionsEnabled
|
|
39726
41152
|
});
|
|
39727
|
-
const readTools = [...policies.values()].filter((
|
|
39728
|
-
const actionTools = [...policies.values()].filter((
|
|
39729
|
-
const revision =
|
|
41153
|
+
const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
|
|
41154
|
+
const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
|
|
41155
|
+
const revision = createHash27("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
39730
41156
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
39731
41157
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
39732
41158
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -39735,18 +41161,18 @@ async function callNangoToolDirect(args) {
|
|
|
39735
41161
|
const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
|
|
39736
41162
|
const selected = tools.find((tool) => tool.name === args.tool);
|
|
39737
41163
|
if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
39738
|
-
const
|
|
41164
|
+
const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
39739
41165
|
tools,
|
|
39740
41166
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39741
41167
|
actionsEnabled: connection.actionsEnabled
|
|
39742
41168
|
}), selected.name);
|
|
39743
|
-
if (
|
|
41169
|
+
if (policy6.classification !== args.classification) {
|
|
39744
41170
|
throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
|
|
39745
41171
|
}
|
|
39746
|
-
if (!
|
|
41172
|
+
if (!policy6.available && policy6.blockedReason === "actions_disabled") {
|
|
39747
41173
|
throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
|
|
39748
41174
|
}
|
|
39749
|
-
if (!
|
|
41175
|
+
if (!policy6.available) {
|
|
39750
41176
|
throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
|
|
39751
41177
|
}
|
|
39752
41178
|
await preflightConnectedUsage({
|
|
@@ -39754,7 +41180,7 @@ async function callNangoToolDirect(args) {
|
|
|
39754
41180
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
39755
41181
|
});
|
|
39756
41182
|
const requestId = args.requestId?.trim() || randomUUID21();
|
|
39757
|
-
const idempotencyKey3 = `main-nango:${
|
|
41183
|
+
const idempotencyKey3 = `main-nango:${createHash27("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
39758
41184
|
const startedAt = /* @__PURE__ */ new Date();
|
|
39759
41185
|
const started = performance.now();
|
|
39760
41186
|
let result;
|
|
@@ -39779,7 +41205,7 @@ async function callNangoToolDirect(args) {
|
|
|
39779
41205
|
toolName: args.tool,
|
|
39780
41206
|
operationKind: args.operationKind ?? args.classification,
|
|
39781
41207
|
outcome: providerError ? "error" : "partial",
|
|
39782
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
41208
|
+
requestId: requestId.length <= 200 ? requestId : createHash27("sha256").update(requestId).digest("hex"),
|
|
39783
41209
|
startedAt: startedAt.toISOString(),
|
|
39784
41210
|
completedAt: completedAt.toISOString()
|
|
39785
41211
|
}
|
|
@@ -39791,13 +41217,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
39791
41217
|
const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
|
|
39792
41218
|
const tool = tools.find((candidate) => candidate.name === toolName);
|
|
39793
41219
|
if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
39794
|
-
const
|
|
41220
|
+
const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
39795
41221
|
tools,
|
|
39796
41222
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39797
41223
|
actionsEnabled: connection.actionsEnabled
|
|
39798
41224
|
}), tool.name);
|
|
39799
41225
|
const classification = classifyConnectedTool(tool);
|
|
39800
|
-
const callable =
|
|
41226
|
+
const callable = policy6.available;
|
|
39801
41227
|
const projected = {
|
|
39802
41228
|
name: tool.name,
|
|
39803
41229
|
...tool.title ? { title: tool.title } : {},
|
|
@@ -39810,18 +41236,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
39810
41236
|
...projected,
|
|
39811
41237
|
classification,
|
|
39812
41238
|
callable,
|
|
39813
|
-
blockedReason:
|
|
39814
|
-
requiredPermissions:
|
|
39815
|
-
missingPermissions:
|
|
39816
|
-
requiredFeatures:
|
|
39817
|
-
missingFeatures:
|
|
41239
|
+
blockedReason: policy6.blockedReason,
|
|
41240
|
+
requiredPermissions: policy6.requiredPermissions,
|
|
41241
|
+
missingPermissions: policy6.missingPermissions,
|
|
41242
|
+
requiredFeatures: policy6.requiredFeatures,
|
|
41243
|
+
missingFeatures: policy6.missingFeatures,
|
|
39818
41244
|
transport: "nango",
|
|
39819
41245
|
providerConfigKey: connection.providerConfigKey,
|
|
39820
41246
|
providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
|
|
39821
41247
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
39822
41248
|
protocolVersion: null,
|
|
39823
41249
|
schemaSource: "live_tools_list",
|
|
39824
|
-
schemaHash:
|
|
41250
|
+
schemaHash: createHash27("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
39825
41251
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
39826
41252
|
};
|
|
39827
41253
|
}
|
|
@@ -40545,8 +41971,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
40545
41971
|
return data.connection.actionsEnabled === true;
|
|
40546
41972
|
}
|
|
40547
41973
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
40548
|
-
const requestId = `main-connected-action:${
|
|
40549
|
-
const requestDigest =
|
|
41974
|
+
const requestId = `main-connected-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
|
|
41975
|
+
const requestDigest = createHash28("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
40550
41976
|
if (mainOwnsIntegrations()) {
|
|
40551
41977
|
const selectedTool = tool?.trim();
|
|
40552
41978
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -40697,7 +42123,7 @@ function canonicalJson2(value) {
|
|
|
40697
42123
|
return JSON.stringify(value);
|
|
40698
42124
|
}
|
|
40699
42125
|
function projectedToolSchemaHash(tool) {
|
|
40700
|
-
return
|
|
42126
|
+
return createHash28("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
40701
42127
|
}
|
|
40702
42128
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
40703
42129
|
if (mainOwnsIntegrations()) {
|
|
@@ -40976,7 +42402,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
40976
42402
|
}
|
|
40977
42403
|
|
|
40978
42404
|
// src/api/resend-control.ts
|
|
40979
|
-
import { createHash as
|
|
42405
|
+
import { createHash as createHash29 } from "crypto";
|
|
40980
42406
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
40981
42407
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
40982
42408
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -41235,7 +42661,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
41235
42661
|
return isRecord5(data) ? data.result ?? data : data;
|
|
41236
42662
|
}
|
|
41237
42663
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
41238
|
-
const requestId = `main-resend-action:${
|
|
42664
|
+
const requestId = `main-resend-action:${createHash29("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
41239
42665
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
41240
42666
|
method: "POST",
|
|
41241
42667
|
headers: { "x-request-id": requestId },
|
|
@@ -41362,9 +42788,9 @@ var auth4 = createMiddleware5(async (c, next) => {
|
|
|
41362
42788
|
return next();
|
|
41363
42789
|
});
|
|
41364
42790
|
var sessionAuth = createMiddleware5(async (c, next) => {
|
|
41365
|
-
const
|
|
41366
|
-
if (!
|
|
41367
|
-
const userId = verifySession(
|
|
42791
|
+
const token6 = getCookie2(c, "session");
|
|
42792
|
+
if (!token6) return c.json({ error: "Not authenticated" }, 401);
|
|
42793
|
+
const userId = verifySession(token6);
|
|
41368
42794
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
41369
42795
|
const user = await getUserById(userId);
|
|
41370
42796
|
if (!user) return c.json({ error: "User not found" }, 401);
|
|
@@ -41509,8 +42935,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
41509
42935
|
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
41510
42936
|
}
|
|
41511
42937
|
})();
|
|
41512
|
-
const
|
|
41513
|
-
setCookie2(c, "session",
|
|
42938
|
+
const token6 = signSession(user.id);
|
|
42939
|
+
setCookie2(c, "session", token6, sessionCookieOptions2);
|
|
41514
42940
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
41515
42941
|
} catch (err) {
|
|
41516
42942
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -41528,8 +42954,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
41528
42954
|
if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
|
|
41529
42955
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
41530
42956
|
}
|
|
41531
|
-
const
|
|
41532
|
-
setCookie2(c, "session",
|
|
42957
|
+
const token6 = signSession(user.id);
|
|
42958
|
+
setCookie2(c, "session", token6, sessionCookieOptions2);
|
|
41533
42959
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
41534
42960
|
});
|
|
41535
42961
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
@@ -41584,9 +43010,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
41584
43010
|
if (emailLimited) return emailLimited;
|
|
41585
43011
|
const user = await getUserByEmail(normalizedEmail);
|
|
41586
43012
|
if (!user) return c.json({ ok: true });
|
|
41587
|
-
const
|
|
43013
|
+
const token6 = await createPasswordResetToken(user.id);
|
|
41588
43014
|
const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
|
|
41589
|
-
const resetUrl = `${appUrl}/reset-password?token=${
|
|
43015
|
+
const resetUrl = `${appUrl}/reset-password?token=${token6}`;
|
|
41590
43016
|
if (process.env.RESEND_API_KEY) {
|
|
41591
43017
|
try {
|
|
41592
43018
|
const resend = new Resend3(process.env.RESEND_API_KEY);
|
|
@@ -41606,12 +43032,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
41606
43032
|
return c.json({ ok: true });
|
|
41607
43033
|
});
|
|
41608
43034
|
app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
|
|
41609
|
-
const { token:
|
|
41610
|
-
if (!
|
|
43035
|
+
const { token: token6, password } = await c.req.json();
|
|
43036
|
+
if (!token6 || !password) return c.json({ error: "Token and password required" }, 400);
|
|
41611
43037
|
if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
|
|
41612
43038
|
const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
|
|
41613
43039
|
if (limited) return limited;
|
|
41614
|
-
const userId = await consumePasswordResetToken(
|
|
43040
|
+
const userId = await consumePasswordResetToken(token6);
|
|
41615
43041
|
if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
|
|
41616
43042
|
await setPassword(userId, password);
|
|
41617
43043
|
return c.json({ ok: true });
|
|
@@ -41648,9 +43074,9 @@ function sessionBootstrapPayload(user) {
|
|
|
41648
43074
|
}
|
|
41649
43075
|
app.get("/me", async (c) => {
|
|
41650
43076
|
c.header("Cache-Control", "no-store");
|
|
41651
|
-
const
|
|
41652
|
-
if (!
|
|
41653
|
-
const userId = verifySession(
|
|
43077
|
+
const token6 = getCookie2(c, "session");
|
|
43078
|
+
if (!token6) return c.json({ authenticated: false });
|
|
43079
|
+
const userId = verifySession(token6);
|
|
41654
43080
|
if (!userId) {
|
|
41655
43081
|
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
41656
43082
|
return c.json({ authenticated: false });
|
|
@@ -43276,6 +44702,8 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43276
44702
|
}
|
|
43277
44703
|
const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
|
|
43278
44704
|
const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
|
|
44705
|
+
const maxMediaAssets = bodyResult.data.maxMediaAssets ?? 100;
|
|
44706
|
+
const maxInlineImages = bodyResult.data.maxInlineImages ?? 3;
|
|
43279
44707
|
if (!allowLocal) {
|
|
43280
44708
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
43281
44709
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -43310,7 +44738,14 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43310
44738
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
43311
44739
|
const [result, pageData] = await Promise.all([
|
|
43312
44740
|
extractKpo({ url: extractionUrl, kernelApiKey }),
|
|
43313
|
-
screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, {
|
|
44741
|
+
screenshot2 || extractBranding2 || preserveMedia ? capturePageData(canonicalUrl, {
|
|
44742
|
+
kernelApiKey,
|
|
44743
|
+
device,
|
|
44744
|
+
screenshot: !!screenshot2,
|
|
44745
|
+
branding: !!extractBranding2,
|
|
44746
|
+
media: preserveMedia,
|
|
44747
|
+
maxMediaAssets: Math.min(1e3, Math.max(250, maxMediaAssets * 4))
|
|
44748
|
+
}).catch((err) => {
|
|
43314
44749
|
console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
|
|
43315
44750
|
return null;
|
|
43316
44751
|
}) : null
|
|
@@ -43321,7 +44756,28 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43321
44756
|
if (screenshotBuf) {
|
|
43322
44757
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
43323
44758
|
}
|
|
43324
|
-
|
|
44759
|
+
let mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, {
|
|
44760
|
+
types: mediaTypes ?? ["image", "video", "audio"],
|
|
44761
|
+
outputDir: null,
|
|
44762
|
+
rendered: pageData?.renderedMedia ?? null,
|
|
44763
|
+
maxAssets: maxMediaAssets
|
|
44764
|
+
}) : null;
|
|
44765
|
+
if (mediaMeta) {
|
|
44766
|
+
try {
|
|
44767
|
+
const packaged = await packagePageMedia({
|
|
44768
|
+
ownerId: String(user.id),
|
|
44769
|
+
pageUrl: canonicalUrl,
|
|
44770
|
+
title: result.title ?? null,
|
|
44771
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
44772
|
+
media: mediaMeta,
|
|
44773
|
+
maxInlineImages
|
|
44774
|
+
});
|
|
44775
|
+
mediaMeta = packaged.media;
|
|
44776
|
+
mediaMeta.artifact = packaged.artifact;
|
|
44777
|
+
} catch (error) {
|
|
44778
|
+
mediaMeta.warnings.push(`Media packaging failed: ${error instanceof Error ? error.message : "media_packaging_failed"}`);
|
|
44779
|
+
}
|
|
44780
|
+
}
|
|
43325
44781
|
const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
|
|
43326
44782
|
html: result.bodyHtml,
|
|
43327
44783
|
meta: result.meta,
|
|
@@ -43504,7 +44960,7 @@ app.post("/archive/read", auth4, async (c) => {
|
|
|
43504
44960
|
try {
|
|
43505
44961
|
let ownedArchive = null;
|
|
43506
44962
|
if (artifactId) {
|
|
43507
|
-
const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
44963
|
+
const buffer = artifactId.startsWith(MAPS_MEDIA_ARTIFACT_PREFIX) ? await readOwnedMapsMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : artifactId.startsWith(PAGE_MEDIA_ARTIFACT_PREFIX) ? await readOwnedPageMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
43508
44964
|
if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
|
|
43509
44965
|
ownedArchive = { buffer, ref: artifactId };
|
|
43510
44966
|
}
|
|
@@ -44314,7 +45770,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
44314
45770
|
return c.json({ error: "Unauthorized" }, 401);
|
|
44315
45771
|
}
|
|
44316
45772
|
const startedAt = Date.now();
|
|
44317
|
-
const { drainQueue } = await import("./worker-
|
|
45773
|
+
const { drainQueue } = await import("./worker-JL4TG6IF.js");
|
|
44318
45774
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
44319
45775
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
44320
45776
|
const unfinished = [];
|
|
@@ -44385,7 +45841,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
44385
45841
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
44386
45842
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
44387
45843
|
} = await import("./site-extract-repository-3VQAALNR.js");
|
|
44388
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
45844
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
|
|
44389
45845
|
const existing = await getExtractJob2(jobId2);
|
|
44390
45846
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
44391
45847
|
if (existing.status !== "failed") {
|
|
@@ -44597,4 +46053,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
44597
46053
|
export {
|
|
44598
46054
|
app
|
|
44599
46055
|
};
|
|
44600
|
-
//# sourceMappingURL=server-
|
|
46056
|
+
//# sourceMappingURL=server-7EXEDKAU.js.map
|