mcp-scraper 0.51.1 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/bin/api-server.cjs +2912 -904
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +5 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +384 -43
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +4 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-ZSJHRZY5.js → chunk-J7TH5KU7.js} +2 -2
- package/dist/{chunk-2K4RCBYY.js → chunk-OT2AF7SH.js} +385 -44
- package/dist/chunk-OT2AF7SH.js.map +1 -0
- package/dist/chunk-SG2PEHR3.js +562 -0
- package/dist/chunk-SG2PEHR3.js.map +1 -0
- package/dist/{chunk-GOZIG6HD.js → chunk-TVZD37SE.js} +2 -2
- package/dist/{chunk-3ZUBQQPQ.js → chunk-V7CEOUBF.js} +5 -1
- package/dist/chunk-V7CEOUBF.js.map +1 -0
- package/dist/chunk-VIAXQTZK.js +7 -0
- package/dist/chunk-VIAXQTZK.js.map +1 -0
- package/dist/{extract-bundle-RCTNANCH.js → extract-bundle-GUEUCTAE.js} +2 -2
- package/dist/index.cjs +4 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{server-JR5XPTXS.js → server-7EXEDKAU.js} +1920 -420
- package/dist/server-7EXEDKAU.js.map +1 -0
- package/dist/{worker-KQN673JF.js → worker-JL4TG6IF.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-27FMOD6S.js +0 -430
- package/dist/chunk-27FMOD6S.js.map +0 -1
- package/dist/chunk-2K4RCBYY.js.map +0 -1
- package/dist/chunk-3ZUBQQPQ.js.map +0 -1
- package/dist/chunk-XLUITZUM.js +0 -7
- package/dist/chunk-XLUITZUM.js.map +0 -1
- package/dist/server-JR5XPTXS.js.map +0 -1
- /package/dist/{chunk-ZSJHRZY5.js.map → chunk-J7TH5KU7.js.map} +0 -0
- /package/dist/{chunk-GOZIG6HD.js.map → chunk-TVZD37SE.js.map} +0 -0
- /package/dist/{extract-bundle-RCTNANCH.js.map → extract-bundle-GUEUCTAE.js.map} +0 -0
- /package/dist/{worker-KQN673JF.js.map → worker-JL4TG6IF.js.map} +0 -0
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
3
3
|
cleanupExpiredSiteExtractArtifacts,
|
|
4
|
+
discoverStaticMedia,
|
|
4
5
|
harvestPageMedia,
|
|
5
6
|
readOwnedSiteExtractArtifactBuffer,
|
|
6
|
-
renewSiteExtractArtifactDownload
|
|
7
|
-
|
|
7
|
+
renewSiteExtractArtifactDownload,
|
|
8
|
+
typeFromMime
|
|
9
|
+
} from "./chunk-SG2PEHR3.js";
|
|
8
10
|
import {
|
|
9
11
|
REQUIRED_LOCATION_STATE_CODES,
|
|
10
12
|
importHostedCensusPlacesCsv,
|
|
@@ -100,7 +102,7 @@ import {
|
|
|
100
102
|
resolveKernelProxyId,
|
|
101
103
|
serpArrivalUrl,
|
|
102
104
|
withRuntimeUserAgent
|
|
103
|
-
} from "./chunk-
|
|
105
|
+
} from "./chunk-TVZD37SE.js";
|
|
104
106
|
import {
|
|
105
107
|
ApiErrorResponseSchema,
|
|
106
108
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -158,7 +160,7 @@ import {
|
|
|
158
160
|
resolveDeploymentProfile,
|
|
159
161
|
resolveLocalSourcebookSchemaType,
|
|
160
162
|
transcribeMediaUrl
|
|
161
|
-
} from "./chunk-
|
|
163
|
+
} from "./chunk-OT2AF7SH.js";
|
|
162
164
|
import {
|
|
163
165
|
auditImageUrls,
|
|
164
166
|
auditImages,
|
|
@@ -196,7 +198,7 @@ import {
|
|
|
196
198
|
workflowOutputBaseDir,
|
|
197
199
|
workflowStepCount,
|
|
198
200
|
workflowSupportsSteps
|
|
199
|
-
} from "./chunk-
|
|
201
|
+
} from "./chunk-J7TH5KU7.js";
|
|
200
202
|
import {
|
|
201
203
|
csvRecords,
|
|
202
204
|
parseCsv,
|
|
@@ -211,10 +213,10 @@ import {
|
|
|
211
213
|
RawMapsHoursRowSchema,
|
|
212
214
|
RawMapsOverviewSchema,
|
|
213
215
|
RawMapsReviewStatsSchema
|
|
214
|
-
} from "./chunk-
|
|
216
|
+
} from "./chunk-V7CEOUBF.js";
|
|
215
217
|
import {
|
|
216
218
|
PACKAGE_VERSION
|
|
217
|
-
} from "./chunk-
|
|
219
|
+
} from "./chunk-VIAXQTZK.js";
|
|
218
220
|
import {
|
|
219
221
|
abandonExtractSettlement,
|
|
220
222
|
countSuccessfulPages,
|
|
@@ -3813,40 +3815,136 @@ async function readResponseTextWithLimit(response, maxBytes, label) {
|
|
|
3813
3815
|
}
|
|
3814
3816
|
|
|
3815
3817
|
// src/api/kernel-fetch.ts
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3818
|
+
var CHALLENGE_PATTERNS = [
|
|
3819
|
+
{ reason: "cloudflare_title", pattern: /<title[^>]*>\s*just a moment(?:\.\.\.)?\s*<\/title>/i },
|
|
3820
|
+
{ reason: "cloudflare_challenge_platform", pattern: /(?:cdn-cgi\/challenge-platform|challenge-platform\/h\/|cf-chl-|cf_chl_)/i },
|
|
3821
|
+
{ reason: "cloudflare_browser_verification", pattern: /(?:cf-browser-verification|checking your browser before accessing)/i },
|
|
3822
|
+
{ reason: "cloudflare_cookie_message", pattern: /enable javascript and cookies to continue/i },
|
|
3823
|
+
{ reason: "generic_security_check", pattern: /performing security verification|verify you are human/i }
|
|
3824
|
+
];
|
|
3825
|
+
function detectBrowserChallenge(input) {
|
|
3826
|
+
if (input.status === 403 || input.status === 429) {
|
|
3827
|
+
return { detected: true, reason: `http_${input.status}` };
|
|
3828
|
+
}
|
|
3829
|
+
const sample = `${input.title ?? ""}
|
|
3830
|
+
${input.html ?? ""}`.slice(0, 25e4);
|
|
3831
|
+
for (const candidate of CHALLENGE_PATTERNS) {
|
|
3832
|
+
if (candidate.pattern.test(sample)) return { detected: true, reason: candidate.reason };
|
|
3833
|
+
}
|
|
3834
|
+
return { detected: false, reason: null };
|
|
3835
|
+
}
|
|
3836
|
+
async function runKernelAttempt(client2, auditedUrl, mode) {
|
|
3823
3837
|
const openedAtMs = Date.now();
|
|
3824
|
-
const
|
|
3825
|
-
const kb = await client2.browsers.create({
|
|
3838
|
+
const headlessSent = mode === "stealth_headless" ? true : null;
|
|
3839
|
+
const kb = await client2.browsers.create({
|
|
3840
|
+
stealth: true,
|
|
3841
|
+
...headlessSent ? { headless: true } : {},
|
|
3842
|
+
timeout_seconds: 60
|
|
3843
|
+
});
|
|
3826
3844
|
const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
|
|
3845
|
+
let attemptError = null;
|
|
3827
3846
|
try {
|
|
3828
3847
|
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
3829
|
-
const context = await browser.newContext(withRuntimeUserAgent({
|
|
3830
|
-
serviceWorkers: "block"
|
|
3831
|
-
}, runtimeUserAgent));
|
|
3848
|
+
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
3832
3849
|
const page = context.pages()[0] ?? await context.newPage();
|
|
3833
3850
|
const guard = await installBrowserRequestGuard(page, auditedUrl);
|
|
3834
|
-
await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
3851
|
+
const response = await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
3835
3852
|
if (guard.blockedReason && page.url() === "about:blank") throw new Error(guard.blockedReason);
|
|
3836
3853
|
await page.waitForTimeout(2500);
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3854
|
+
let status = response?.status() ?? null;
|
|
3855
|
+
let title = await page.title().catch(() => "");
|
|
3856
|
+
let html = await page.content();
|
|
3857
|
+
let challenge = detectBrowserChallenge({ status, title, html });
|
|
3858
|
+
if (challenge.detected && mode === "stealth_headful") {
|
|
3859
|
+
const deadline = Date.now() + 2e4;
|
|
3860
|
+
while (challenge.detected && Date.now() < deadline) {
|
|
3861
|
+
await page.waitForTimeout(1e3);
|
|
3862
|
+
title = await page.title().catch(() => "");
|
|
3863
|
+
html = await page.content();
|
|
3864
|
+
challenge = detectBrowserChallenge({ status, title, html });
|
|
3865
|
+
}
|
|
3866
|
+
status = response?.status() ?? status;
|
|
3867
|
+
}
|
|
3868
|
+
const byteLength2 = new TextEncoder().encode(html).byteLength;
|
|
3869
|
+
if (byteLength2 > MAX_PAGE_HTML_BYTES) throw new Error(`Browser HTML exceeds ${MAX_PAGE_HTML_BYTES} byte limit`);
|
|
3870
|
+
return { html, status, challenge, durationMs: Date.now() - openedAtMs };
|
|
3871
|
+
} catch (error) {
|
|
3872
|
+
attemptError = error instanceof Error ? error.message : String(error);
|
|
3873
|
+
throw error;
|
|
3842
3874
|
} finally {
|
|
3843
3875
|
await browser.close().catch(() => {
|
|
3844
3876
|
});
|
|
3845
3877
|
await client2.browsers.deleteByID(kb.session_id).catch(() => {
|
|
3846
3878
|
});
|
|
3847
|
-
void recordKernelSession({
|
|
3879
|
+
void recordKernelSession({
|
|
3880
|
+
kernelSessionId: kb.session_id,
|
|
3881
|
+
source: "kernel_fetch",
|
|
3882
|
+
stealth: typeof kb.stealth === "boolean" ? kb.stealth : null,
|
|
3883
|
+
headlessSent,
|
|
3884
|
+
proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null,
|
|
3885
|
+
fallback: true,
|
|
3886
|
+
openedAtMs,
|
|
3887
|
+
closedAtMs: Date.now(),
|
|
3888
|
+
error: attemptError
|
|
3889
|
+
});
|
|
3848
3890
|
}
|
|
3849
3891
|
}
|
|
3892
|
+
async function fetchWithKernelDetailed(url) {
|
|
3893
|
+
const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
|
|
3894
|
+
if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
|
|
3895
|
+
const auditedUrl = checked.parsed.href;
|
|
3896
|
+
const apiKey = browserServiceApiKey();
|
|
3897
|
+
if (!apiKey) throw new Error("Browser backend API key not set");
|
|
3898
|
+
const client2 = new Kernel({ apiKey });
|
|
3899
|
+
const startsHeadful = currentCostContext()?.forceHeadful === true;
|
|
3900
|
+
const modes = startsHeadful ? ["stealth_headful"] : ["stealth_headless", "stealth_headful"];
|
|
3901
|
+
const attempts = [];
|
|
3902
|
+
for (const mode of modes) {
|
|
3903
|
+
const result = await runKernelAttempt(client2, auditedUrl, mode);
|
|
3904
|
+
if (result.challenge.detected) {
|
|
3905
|
+
const completion = mode === "stealth_headful" ? "challenge_unresolved" : "challenge_detected";
|
|
3906
|
+
attempts.push({
|
|
3907
|
+
mode,
|
|
3908
|
+
status: result.status,
|
|
3909
|
+
challengeDetected: true,
|
|
3910
|
+
challengeReason: result.challenge.reason,
|
|
3911
|
+
completion,
|
|
3912
|
+
durationMs: result.durationMs
|
|
3913
|
+
});
|
|
3914
|
+
if (mode === "stealth_headful") {
|
|
3915
|
+
const error = new Error(`browser_challenge_unresolved: ${result.challenge.reason ?? "challenge page remained after bounded wait"}`);
|
|
3916
|
+
error.code = "browser_challenge_unresolved";
|
|
3917
|
+
error.attempts = attempts;
|
|
3918
|
+
throw error;
|
|
3919
|
+
}
|
|
3920
|
+
continue;
|
|
3921
|
+
}
|
|
3922
|
+
if (result.status != null && (result.status < 200 || result.status >= 400)) {
|
|
3923
|
+
attempts.push({
|
|
3924
|
+
mode,
|
|
3925
|
+
status: result.status,
|
|
3926
|
+
challengeDetected: false,
|
|
3927
|
+
challengeReason: null,
|
|
3928
|
+
completion: "http_error",
|
|
3929
|
+
durationMs: result.durationMs
|
|
3930
|
+
});
|
|
3931
|
+
throw new Error(`browser_http_${result.status}: Browser fetch returned HTTP ${result.status}`);
|
|
3932
|
+
}
|
|
3933
|
+
attempts.push({
|
|
3934
|
+
mode,
|
|
3935
|
+
status: result.status,
|
|
3936
|
+
challengeDetected: false,
|
|
3937
|
+
challengeReason: null,
|
|
3938
|
+
completion: "usable_page",
|
|
3939
|
+
durationMs: result.durationMs
|
|
3940
|
+
});
|
|
3941
|
+
return { html: result.html, mode, attempts };
|
|
3942
|
+
}
|
|
3943
|
+
throw new Error("browser_challenge_unresolved: no usable browser response");
|
|
3944
|
+
}
|
|
3945
|
+
async function fetchWithKernel(url) {
|
|
3946
|
+
return (await fetchWithKernelDetailed(url)).html;
|
|
3947
|
+
}
|
|
3850
3948
|
|
|
3851
3949
|
// src/lib/http-user-agent.ts
|
|
3852
3950
|
var MCP_SCRAPER_CRAWLER_USER_AGENT = "MCPScraperBot/1.0 (+https://mcpscraper.dev)";
|
|
@@ -3875,6 +3973,8 @@ async function extractKpo(opts) {
|
|
|
3875
3973
|
}
|
|
3876
3974
|
if (!res.ok) return null;
|
|
3877
3975
|
const text2 = await res.text();
|
|
3976
|
+
const title2 = text2.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? null;
|
|
3977
|
+
if (detectBrowserChallenge({ status: res.status, title: title2, html: text2 }).detected) return null;
|
|
3878
3978
|
if (text2.length < 2e3) return null;
|
|
3879
3979
|
const stripped = text2.replace(/<(script|style|noscript|svg|iframe|canvas)[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
3880
3980
|
const contentBlock = stripped.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? stripped.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? "";
|
|
@@ -3889,13 +3989,16 @@ async function extractKpo(opts) {
|
|
|
3889
3989
|
};
|
|
3890
3990
|
let html;
|
|
3891
3991
|
let fetchedVia = "fetch";
|
|
3992
|
+
let browserFetch;
|
|
3892
3993
|
const plainFetchStart = Date.now();
|
|
3893
3994
|
const plainHtml = await tryPlainFetch();
|
|
3894
3995
|
const plainFetchMs = Date.now() - plainFetchStart;
|
|
3895
3996
|
if (plainHtml) {
|
|
3896
3997
|
html = plainHtml;
|
|
3897
3998
|
} else if (opts.kernelApiKey) {
|
|
3898
|
-
|
|
3999
|
+
const browserResult = await fetchWithKernelDetailed(parsed.href);
|
|
4000
|
+
html = browserResult.html;
|
|
4001
|
+
browserFetch = { mode: browserResult.mode, attempts: browserResult.attempts };
|
|
3899
4002
|
fetchedVia = "headless";
|
|
3900
4003
|
} else {
|
|
3901
4004
|
throw new Error("Fetch failed: site blocked plain HTTP and no headless browser backend is configured");
|
|
@@ -4195,6 +4298,7 @@ async function extractKpo(opts) {
|
|
|
4195
4298
|
return {
|
|
4196
4299
|
url: parsed.href,
|
|
4197
4300
|
fetchedVia,
|
|
4301
|
+
browserFetch,
|
|
4198
4302
|
durationMs,
|
|
4199
4303
|
plainFetchMs,
|
|
4200
4304
|
title,
|
|
@@ -4212,13 +4316,47 @@ import { chromium as chromium2 } from "playwright";
|
|
|
4212
4316
|
import Kernel2 from "@onkernel/sdk";
|
|
4213
4317
|
|
|
4214
4318
|
// src/lib/branding-extractor.ts
|
|
4215
|
-
function
|
|
4216
|
-
const
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4319
|
+
function selectRankedLogoCandidates(candidates) {
|
|
4320
|
+
const ranked = candidates.filter((candidate) => {
|
|
4321
|
+
if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.region === "body") return false;
|
|
4322
|
+
if (candidate.region === "json_ld" || candidate.region === "favicon") return true;
|
|
4323
|
+
return candidate.evidence.includes("links_to_homepage") || candidate.evidence.includes("site_identity_marker");
|
|
4324
|
+
}).map((candidate) => ({
|
|
4325
|
+
...candidate,
|
|
4326
|
+
confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
|
|
4327
|
+
evidence: [...new Set(candidate.evidence)]
|
|
4328
|
+
})).sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
|
|
4329
|
+
const primary = ranked.find((candidate) => candidate.score >= 55 && candidate.region !== "favicon") ?? null;
|
|
4330
|
+
return {
|
|
4331
|
+
primary,
|
|
4332
|
+
variants: primary ? ranked.filter((candidate) => candidate.url !== primary.url && candidate.score >= 55 && candidate.region !== "favicon" && logoFamilyKey(candidate.url) === logoFamilyKey(primary.url)).slice(0, 5).map((candidate) => candidate.url) : [],
|
|
4333
|
+
candidates: ranked,
|
|
4334
|
+
selectionReason: primary?.evidence.join(", ") ?? null
|
|
4335
|
+
};
|
|
4336
|
+
}
|
|
4337
|
+
function logoFamilyKey(value) {
|
|
4338
|
+
try {
|
|
4339
|
+
const url = new URL(value);
|
|
4340
|
+
const host2 = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
4341
|
+
const path5 = decodeURIComponent(url.pathname).toLowerCase().replace(/-\d+x\d+(?=\.[a-z0-9]+$)/, "");
|
|
4342
|
+
return `${host2}${path5}`;
|
|
4343
|
+
} catch {
|
|
4344
|
+
return value.toLowerCase();
|
|
4345
|
+
}
|
|
4346
|
+
}
|
|
4347
|
+
function selectProofImages(candidates) {
|
|
4348
|
+
const byUrl = /* @__PURE__ */ new Map();
|
|
4349
|
+
for (const candidate of candidates) {
|
|
4350
|
+
if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.score < 45) continue;
|
|
4351
|
+
const normalized = {
|
|
4352
|
+
...candidate,
|
|
4353
|
+
confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
|
|
4354
|
+
evidence: [...new Set(candidate.evidence)]
|
|
4355
|
+
};
|
|
4356
|
+
const existing = byUrl.get(normalized.url);
|
|
4357
|
+
if (!existing || normalized.score > existing.score) byUrl.set(normalized.url, normalized);
|
|
4358
|
+
}
|
|
4359
|
+
return [...byUrl.values()].sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
|
|
4222
4360
|
}
|
|
4223
4361
|
function luminance(hex) {
|
|
4224
4362
|
const m = hex.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
|
|
@@ -4232,6 +4370,12 @@ function isTransparentOrWhite(hex) {
|
|
|
4232
4370
|
const lum = luminance(hex);
|
|
4233
4371
|
return lum > 240;
|
|
4234
4372
|
}
|
|
4373
|
+
function isChromatic(hex) {
|
|
4374
|
+
const match = hex?.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
|
|
4375
|
+
if (!match) return false;
|
|
4376
|
+
const channels = [parseInt(match[1], 16), parseInt(match[2], 16), parseInt(match[3], 16)];
|
|
4377
|
+
return Math.max(...channels) - Math.min(...channels) >= 24 && luminance(hex) > 18;
|
|
4378
|
+
}
|
|
4235
4379
|
function firstFont(fontFamily) {
|
|
4236
4380
|
if (!fontFamily) return null;
|
|
4237
4381
|
const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
|
|
@@ -4240,14 +4384,34 @@ function firstFont(fontFamily) {
|
|
|
4240
4384
|
function dominantColor(freq) {
|
|
4241
4385
|
return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
|
4242
4386
|
}
|
|
4387
|
+
function dominantChromaticColor(freq) {
|
|
4388
|
+
return Object.entries(freq).filter(([hex]) => isChromatic(hex)).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
|
4389
|
+
}
|
|
4243
4390
|
async function extractBrandingFromPage(page) {
|
|
4391
|
+
const challenge = detectBrowserChallenge({
|
|
4392
|
+
title: await page.title().catch(() => ""),
|
|
4393
|
+
html: await page.content().catch(() => "")
|
|
4394
|
+
});
|
|
4395
|
+
if (challenge.detected) {
|
|
4396
|
+
const error = new Error(`browser_challenge_unresolved: ${challenge.reason ?? "challenge page is not branding evidence"}`);
|
|
4397
|
+
error.code = "browser_challenge_unresolved";
|
|
4398
|
+
throw error;
|
|
4399
|
+
}
|
|
4244
4400
|
const evalScript = `
|
|
4245
4401
|
(function() {
|
|
4246
4402
|
function cs(el) { return el ? window.getComputedStyle(el) : null; }
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4403
|
+
var colorCanvas = document.createElement('canvas');
|
|
4404
|
+
colorCanvas.width = 1; colorCanvas.height = 1;
|
|
4405
|
+
var colorContext = colorCanvas.getContext('2d', { willReadFrequently: true });
|
|
4406
|
+
function toHex(value) {
|
|
4407
|
+
if (!value || !colorContext) return null;
|
|
4408
|
+
colorContext.clearRect(0, 0, 1, 1);
|
|
4409
|
+
colorContext.fillStyle = 'rgba(1, 2, 3, 0.001)';
|
|
4410
|
+
colorContext.fillStyle = value;
|
|
4411
|
+
colorContext.fillRect(0, 0, 1, 1);
|
|
4412
|
+
var data = colorContext.getImageData(0, 0, 1, 1).data;
|
|
4413
|
+
if (data[3] < 64) return null;
|
|
4414
|
+
return '#' + [data[0], data[1], data[2]].map(function(v){ return ('0'+v.toString(16)).slice(-2); }).join('');
|
|
4251
4415
|
}
|
|
4252
4416
|
function isUsable(hex) {
|
|
4253
4417
|
if (!hex) return false;
|
|
@@ -4256,30 +4420,63 @@ async function extractBrandingFromPage(page) {
|
|
|
4256
4420
|
return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
|
|
4257
4421
|
}
|
|
4258
4422
|
|
|
4423
|
+
function isChromatic(hex) {
|
|
4424
|
+
if (!hex) return false;
|
|
4425
|
+
var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
|
|
4426
|
+
return Math.max(r,g,b) - Math.min(r,g,b) >= 24 && (0.2126*r + 0.7152*g + 0.0722*b) > 18;
|
|
4427
|
+
}
|
|
4428
|
+
|
|
4429
|
+
function visibleRect(el) {
|
|
4430
|
+
if (!el) return null;
|
|
4431
|
+
var style = cs(el);
|
|
4432
|
+
var rect = el.getBoundingClientRect();
|
|
4433
|
+
if (!style || style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') <= 0 || rect.width < 2 || rect.height < 2) return null;
|
|
4434
|
+
return rect;
|
|
4435
|
+
}
|
|
4436
|
+
|
|
4259
4437
|
var navEl = document.querySelector('nav, header, [role="banner"]');
|
|
4260
4438
|
var bodyEl = document.body;
|
|
4261
4439
|
var h1El = document.querySelector('h1');
|
|
4262
|
-
var btnEl
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4440
|
+
var btnEl = null;
|
|
4441
|
+
var btnScore = -Infinity;
|
|
4442
|
+
var interactive = document.querySelectorAll('a[href], button, [role="button"]');
|
|
4443
|
+
for (var bi = 0; bi < interactive.length; bi++) {
|
|
4444
|
+
var candidateButton = interactive[bi];
|
|
4445
|
+
var buttonRect = visibleRect(candidateButton);
|
|
4446
|
+
if (!buttonRect || buttonRect.width < 44 || buttonRect.height < 20) continue;
|
|
4447
|
+
if (buttonRect.bottom < 0 || buttonRect.top > window.innerHeight) continue;
|
|
4448
|
+
var buttonStyle = cs(candidateButton);
|
|
4449
|
+
var buttonBg = buttonStyle ? toHex(buttonStyle.backgroundColor) : null;
|
|
4450
|
+
if (!buttonBg) continue;
|
|
4451
|
+
var buttonDescriptor = [candidateButton.textContent, candidateButton.id, candidateButton.className, candidateButton.getAttribute('aria-label')].join(' ');
|
|
4452
|
+
if (/\\b(?:skip|screen-reader|sr-only|visually-hidden)\\b/i.test(buttonDescriptor)) continue;
|
|
4453
|
+
var buttonText = String(candidateButton.textContent || candidateButton.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim();
|
|
4454
|
+
var explicitButtonEvidence = /\\b(?:btn[-_ ]?primary|button[-_ ]?primary|cta|call[-_ ]?to[-_ ]?action)\\b/i.test(buttonDescriptor);
|
|
4455
|
+
var actionTextEvidence = /\\b(?:get started|start|try|book|contact|schedule|request|sign up|join|buy|shop|call|quote|estimate|verify)\\b/i.test(buttonDescriptor);
|
|
4456
|
+
if (!explicitButtonEvidence && !actionTextEvidence && (buttonRect.height > 100 || buttonText.length > 80)) continue;
|
|
4457
|
+
var score = 0;
|
|
4458
|
+
if (isChromatic(buttonBg)) score += 35;
|
|
4459
|
+
if (explicitButtonEvidence) score += 30;
|
|
4460
|
+
if (actionTextEvidence) score += 20;
|
|
4461
|
+
if (candidateButton.closest('main, [role="main"], [class*="hero"]')) score += 12;
|
|
4462
|
+
if (candidateButton.closest('header, nav, [role="banner"]')) score += 4;
|
|
4463
|
+
score += Math.min(10, Math.round((buttonRect.width * buttonRect.height) / 5000));
|
|
4464
|
+
if (score > btnScore) { btnEl = candidateButton; btnScore = score; }
|
|
4465
|
+
}
|
|
4272
4466
|
var navStyle = cs(navEl);
|
|
4273
4467
|
var bodyStyle = cs(bodyEl);
|
|
4274
4468
|
var h1Style = cs(h1El);
|
|
4275
4469
|
var btnStyle = cs(btnEl);
|
|
4276
4470
|
|
|
4277
4471
|
var svgFreq = {};
|
|
4278
|
-
var svgScope =
|
|
4472
|
+
var svgScope = document;
|
|
4279
4473
|
if (svgScope) {
|
|
4280
|
-
var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
|
|
4474
|
+
var svgEls = svgScope.querySelectorAll('svg *, svg[fill], svg path, svg circle, svg rect, svg polygon, svg polyline');
|
|
4281
4475
|
for (var si = 0; si < svgEls.length; si++) {
|
|
4282
4476
|
var svgEl = svgEls[si];
|
|
4477
|
+
var owningSvg = svgEl.closest('svg');
|
|
4478
|
+
var owningRect = owningSvg ? owningSvg.getBoundingClientRect() : null;
|
|
4479
|
+
if (!owningRect || owningRect.bottom < 0 || owningRect.top > 300 || owningRect.width < 12 || owningRect.height < 12) continue;
|
|
4283
4480
|
var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
|
|
4284
4481
|
var fillAttr = svgEl.getAttribute('fill');
|
|
4285
4482
|
var fillHex = null;
|
|
@@ -4301,55 +4498,251 @@ async function extractBrandingFromPage(page) {
|
|
|
4301
4498
|
}
|
|
4302
4499
|
}
|
|
4303
4500
|
|
|
4501
|
+
var cssPrimaryCandidates = [];
|
|
4502
|
+
var cssAccentCandidates = [];
|
|
4503
|
+
var rootStyle = cs(document.documentElement);
|
|
4504
|
+
if (rootStyle) {
|
|
4505
|
+
for (var ci = 0; ci < rootStyle.length; ci++) {
|
|
4506
|
+
var propertyName = rootStyle[ci];
|
|
4507
|
+
if (!propertyName || propertyName.indexOf('--') !== 0) continue;
|
|
4508
|
+
var normalizedName = propertyName.toLowerCase();
|
|
4509
|
+
if (!/(?:^|[-_])(?:brand|primary|accent)(?:$|[-_])/.test(normalizedName)) continue;
|
|
4510
|
+
if (/(?:on[-_]?primary|primary[-_]?(?:text|foreground)|accent[-_]?(?:text|foreground))/.test(normalizedName)) continue;
|
|
4511
|
+
var propertyValue = rootStyle.getPropertyValue(propertyName).trim();
|
|
4512
|
+
var propertyHex = toHex(propertyValue);
|
|
4513
|
+
if (!propertyHex || !isChromatic(propertyHex)) continue;
|
|
4514
|
+
var item = { hex: propertyHex, name: propertyName };
|
|
4515
|
+
if (/(?:^|[-_])primary(?:$|[-_])|brand/.test(normalizedName)) cssPrimaryCandidates.push(item);
|
|
4516
|
+
if (/(?:^|[-_])accent(?:$|[-_])/.test(normalizedName)) cssAccentCandidates.push(item);
|
|
4517
|
+
}
|
|
4518
|
+
}
|
|
4519
|
+
|
|
4520
|
+
var themeColorEl = document.querySelector('meta[name="theme-color"]');
|
|
4521
|
+
var themeColor = themeColorEl ? toHex(themeColorEl.getAttribute('content')) : null;
|
|
4522
|
+
|
|
4523
|
+
var visibleBgFreq = {};
|
|
4524
|
+
var topBgFreq = {};
|
|
4525
|
+
var visibleElements = document.querySelectorAll('body *');
|
|
4526
|
+
for (var vi = 0; vi < visibleElements.length && vi < 1500; vi++) {
|
|
4527
|
+
var visibleElement = visibleElements[vi];
|
|
4528
|
+
var visibleElementRect = visibleRect(visibleElement);
|
|
4529
|
+
if (!visibleElementRect || visibleElementRect.width < 20 || visibleElementRect.height < 12) continue;
|
|
4530
|
+
var visibleStyle = cs(visibleElement);
|
|
4531
|
+
var visibleHex = visibleStyle ? toHex(visibleStyle.backgroundColor) : null;
|
|
4532
|
+
if (!visibleHex || !isChromatic(visibleHex)) continue;
|
|
4533
|
+
var areaWeight = Math.max(1, Math.min(20, Math.round(Math.sqrt(visibleElementRect.width * visibleElementRect.height) / 40)));
|
|
4534
|
+
visibleBgFreq[visibleHex] = (visibleBgFreq[visibleHex] || 0) + areaWeight;
|
|
4535
|
+
if (visibleElementRect.top < 300 && visibleElementRect.bottom > 0 && visibleElementRect.width >= window.innerWidth * 0.5) {
|
|
4536
|
+
topBgFreq[visibleHex] = (topBgFreq[visibleHex] || 0) + areaWeight;
|
|
4537
|
+
}
|
|
4538
|
+
}
|
|
4539
|
+
|
|
4304
4540
|
var pageHost = window.location.hostname.replace(/^www\\./, '');
|
|
4305
4541
|
function isSameDomain(src) {
|
|
4306
|
-
try {
|
|
4542
|
+
try {
|
|
4543
|
+
var host = new URL(src).hostname.replace(/^www\\./, '');
|
|
4544
|
+
return host === pageHost || host.endsWith('.' + pageHost);
|
|
4545
|
+
} catch { return false; }
|
|
4307
4546
|
}
|
|
4308
|
-
var
|
|
4309
|
-
|
|
4310
|
-
'
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
var
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4547
|
+
var logoCandidates = {};
|
|
4548
|
+
function addLogo(raw, region, evidence, score, alt, width, height) {
|
|
4549
|
+
if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
|
|
4550
|
+
var url;
|
|
4551
|
+
try { url = new URL(raw, document.baseURI); } catch (_) { return; }
|
|
4552
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
|
|
4553
|
+
url.hash = '';
|
|
4554
|
+
var key = url.href;
|
|
4555
|
+
var existing = logoCandidates[key] || {
|
|
4556
|
+
url: key, score: 0, region: region, evidence: [], alt: null, width: null, height: null
|
|
4557
|
+
};
|
|
4558
|
+
existing.score = Math.max(existing.score, score);
|
|
4559
|
+
existing.evidence = Array.from(new Set(existing.evidence.concat(evidence)));
|
|
4560
|
+
if (region === 'json_ld' || (existing.region === 'body' && region !== 'body')) existing.region = region;
|
|
4561
|
+
if (alt && !existing.alt) existing.alt = alt.replace(/\\s+/g, ' ').trim().slice(0, 300) || null;
|
|
4562
|
+
if ((width || 0) * (height || 0) > (existing.width || 0) * (existing.height || 0)) {
|
|
4563
|
+
existing.width = width || null; existing.height = height || null;
|
|
4564
|
+
}
|
|
4565
|
+
logoCandidates[key] = existing;
|
|
4566
|
+
}
|
|
4567
|
+
|
|
4568
|
+
function jsonLogo(value) {
|
|
4569
|
+
if (typeof value === 'string') return value;
|
|
4570
|
+
if (!value || typeof value !== 'object') return null;
|
|
4571
|
+
return typeof value.url === 'string' ? value.url : typeof value.contentUrl === 'string' ? value.contentUrl : null;
|
|
4572
|
+
}
|
|
4573
|
+
var jsonScripts = document.querySelectorAll('script[type="application/ld+json"]');
|
|
4574
|
+
for (var ji = 0; ji < jsonScripts.length; ji++) {
|
|
4575
|
+
try {
|
|
4576
|
+
var parsed = JSON.parse(jsonScripts[ji].textContent || 'null');
|
|
4577
|
+
var stack = Array.isArray(parsed) ? parsed.slice() : [parsed];
|
|
4578
|
+
var visited = 0;
|
|
4579
|
+
while (stack.length && visited++ < 1000) {
|
|
4580
|
+
var node = stack.pop();
|
|
4581
|
+
if (!node || typeof node !== 'object') continue;
|
|
4582
|
+
var types = Array.isArray(node['@type']) ? node['@type'] : [node['@type']];
|
|
4583
|
+
var entity = types.some(function(t) { return /^(Organization|Corporation|LocalBusiness|WebSite|Brand)$/i.test(String(t || '')); });
|
|
4584
|
+
var logo = entity ? jsonLogo(node.logo) : null;
|
|
4585
|
+
if (logo) addLogo(logo, 'json_ld', ['json_ld_logo'], 100, null, null, null);
|
|
4586
|
+
Object.keys(node).forEach(function(key) {
|
|
4587
|
+
if (key === '@context') return;
|
|
4588
|
+
var child = node[key];
|
|
4589
|
+
if (Array.isArray(child)) stack = stack.concat(child.filter(function(value) { return value && typeof value === 'object'; }));
|
|
4590
|
+
else if (child && typeof child === 'object') stack.push(child);
|
|
4591
|
+
});
|
|
4592
|
+
}
|
|
4593
|
+
} catch (_) {}
|
|
4594
|
+
}
|
|
4595
|
+
|
|
4596
|
+
var proofCandidates = {};
|
|
4597
|
+
var pageHeadings = Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6'));
|
|
4598
|
+
function nearestHeadingText(element) {
|
|
4599
|
+
var nearest = null;
|
|
4600
|
+
for (var hi = 0; hi < pageHeadings.length; hi++) {
|
|
4601
|
+
if (pageHeadings[hi].compareDocumentPosition(element) & 4) nearest = pageHeadings[hi];
|
|
4602
|
+
else break;
|
|
4603
|
+
}
|
|
4604
|
+
return nearest && nearest.textContent ? nearest.textContent.replace(/\\s+/g, ' ').trim().slice(0, 300) : null;
|
|
4605
|
+
}
|
|
4606
|
+
function proofTypeFor(descriptor, context) {
|
|
4607
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(descriptor)) return 'accreditation';
|
|
4608
|
+
if (/\\b(?:certifi(?:ed|cation)|compliance|legitscript|hipaa)\\b/i.test(descriptor)) return 'certification';
|
|
4609
|
+
if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(descriptor)) return 'award';
|
|
4610
|
+
if (/\\b(?:member|membership|association|professionals?)\\b/i.test(descriptor)) return 'membership';
|
|
4611
|
+
if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(descriptor)) return 'press_mention';
|
|
4612
|
+
if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(descriptor)) return 'partner_or_customer';
|
|
4613
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(context || '')) return 'accreditation';
|
|
4614
|
+
if (/\\b(?:certifi(?:ed|cation)|compliance)\\b/i.test(context || '')) return 'certification';
|
|
4615
|
+
if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(context || '')) return 'award';
|
|
4616
|
+
if (/\\b(?:member|membership|association)\\b/i.test(context || '')) return 'membership';
|
|
4617
|
+
if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(context || '')) return 'press_mention';
|
|
4618
|
+
if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(context || '')) return 'partner_or_customer';
|
|
4619
|
+
return 'trust_mark';
|
|
4620
|
+
}
|
|
4621
|
+
function addProof(raw, proofType, score, evidence, alt, context, width, height) {
|
|
4622
|
+
if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
|
|
4623
|
+
var url;
|
|
4624
|
+
try { url = new URL(raw, document.baseURI); } catch (_) { return; }
|
|
4625
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
|
|
4626
|
+
url.hash = '';
|
|
4627
|
+
var key = url.href;
|
|
4628
|
+
var existing = proofCandidates[key];
|
|
4629
|
+
if (!existing || score > existing.score) {
|
|
4630
|
+
proofCandidates[key] = {
|
|
4631
|
+
url: key, proofType: proofType, score: score, confidence: 'low',
|
|
4632
|
+
evidence: Array.from(new Set(evidence)), alt: alt || null, context: context || null,
|
|
4633
|
+
width: width || null, height: height || null
|
|
4634
|
+
};
|
|
4635
|
+
}
|
|
4636
|
+
}
|
|
4637
|
+
|
|
4638
|
+
var imageEls = document.querySelectorAll('img');
|
|
4639
|
+
for (var li = 0; li < imageEls.length; li++) {
|
|
4640
|
+
var image = imageEls[li];
|
|
4641
|
+
var src = image.currentSrc || image.src || image.getAttribute('data-src') || image.getAttribute('data-lazy-src');
|
|
4642
|
+
if (!src) continue;
|
|
4643
|
+
var header = image.closest('header,[role="banner"]');
|
|
4644
|
+
var nav = image.closest('nav,[role="navigation"]');
|
|
4645
|
+
var footer = image.closest('footer,[role="contentinfo"]');
|
|
4646
|
+
var region = header ? 'header' : nav ? 'nav' : footer ? 'footer' : 'body';
|
|
4647
|
+
var evidence = [];
|
|
4648
|
+
var score = 0;
|
|
4649
|
+
if (header) { score += 35; evidence.push('header_or_banner'); }
|
|
4650
|
+
else if (nav) { score += 28; evidence.push('navigation'); }
|
|
4651
|
+
else if (footer) { score += 18; evidence.push('footer_or_contentinfo'); }
|
|
4652
|
+
var descriptor = [src, image.id, image.className, image.alt, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
|
|
4653
|
+
var identityDescriptor = [image.id, image.className, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
|
|
4654
|
+
if (/\\b(?:site[-_ ]?logo|brand[-_ ]?logo|custom-logo|navbar[-_ ]?brand|site[-_ ]?branding|site[-_ ]?identity|logo[-_ ]?link)\\b/i.test(identityDescriptor)) { score += 30; evidence.push('site_identity_marker'); }
|
|
4655
|
+
else if (/\\blogo\\b/i.test(descriptor)) { score += 12; evidence.push('generic_logo_marker'); }
|
|
4656
|
+
var anchor = image.closest('a[href]');
|
|
4657
|
+
if (anchor) {
|
|
4658
|
+
try {
|
|
4659
|
+
var destination = new URL(anchor.href, document.baseURI);
|
|
4660
|
+
if (destination.origin === location.origin && destination.pathname.replace(/\\/+$/, '') === '') {
|
|
4661
|
+
score += 22; evidence.push('links_to_homepage');
|
|
4662
|
+
}
|
|
4663
|
+
} catch (_) {}
|
|
4664
|
+
}
|
|
4665
|
+
if (isSameDomain(src)) { score += 8; evidence.push('same_domain'); }
|
|
4666
|
+
var style = cs(image);
|
|
4667
|
+
var visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity || '1') > 0;
|
|
4668
|
+
if (visible) { score += 5; evidence.push('visible'); }
|
|
4669
|
+
var width = image.naturalWidth || image.width || null;
|
|
4670
|
+
var height = image.naturalHeight || image.height || null;
|
|
4671
|
+
if (width && height && width >= 80 && height >= 24) { score += 5; evidence.push('plausible_logo_dimensions'); }
|
|
4672
|
+
if ((width && width < 32) || (height && height < 16)) { score -= 45; evidence.push('tiny_asset_penalty'); }
|
|
4673
|
+
if (/\\b(?:icon|avatar|profile|search|menu|hamburger|social|badge)\\b/i.test(descriptor)) { score -= 35; evidence.push('icon_or_avatar_penalty'); }
|
|
4674
|
+
addLogo(src, region, evidence, score, image.alt || null, width, height);
|
|
4675
|
+
|
|
4676
|
+
if (region === 'body' && visible && width && height && width >= 80 && height >= 40) {
|
|
4677
|
+
var context = nearestHeadingText(image);
|
|
4678
|
+
var proofDescriptor = [src, image.alt, image.getAttribute('title')].filter(Boolean).join(' ');
|
|
4679
|
+
var proofEvidence = [];
|
|
4680
|
+
var proofScore = 0;
|
|
4681
|
+
var commercialContext = !!context && /\\b(?:insurances?|payment|financing|we accept|accepted health)\\b/i.test(context);
|
|
4682
|
+
if (/\\b(?:logo|seal|trust[-_ ]?mark|certification[-_ ]?mark)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_asset_marker'); }
|
|
4683
|
+
if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation)|certifi(?:ed|cation)|compliance|legitscript|hipaa|award|winner|best of|top[-_ ]?rated|member|membership|association|professionals?|press|featured|as seen|mentioned|partner|customer|client|trusted by|used by|works with)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_descriptor'); }
|
|
4684
|
+
if (context && /\\b(?:recognized|accredit(?:ed|ation)|certifi(?:ed|cation)|award|winner|member|membership|association|featured|as seen|mentioned|partner|customer|client|trusted|trust)\\b/i.test(context)) { proofScore += 25; proofEvidence.push('proof_section_context'); }
|
|
4685
|
+
if (commercialContext) { proofScore -= 100; proofEvidence.push('commercial_acceptance_context_exclusion'); }
|
|
4686
|
+
proofScore += 10; proofEvidence.push('proof_dimensions');
|
|
4687
|
+
proofScore += 5; proofEvidence.push('visible');
|
|
4688
|
+
if (isSameDomain(src)) { proofScore += 5; proofEvidence.push('same_domain'); }
|
|
4689
|
+
else { proofScore += 3; proofEvidence.push('external_asset_domain'); }
|
|
4690
|
+
if (!commercialContext && proofScore >= 45) addProof(src, proofTypeFor(proofDescriptor, context), proofScore, proofEvidence, image.alt || null, context, width, height);
|
|
4323
4691
|
}
|
|
4324
4692
|
}
|
|
4325
4693
|
var faviconEl = document.querySelector(
|
|
4326
4694
|
'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
|
|
4327
4695
|
);
|
|
4696
|
+
if (faviconEl && faviconEl.href) addLogo(faviconEl.href, 'favicon', ['favicon_fallback_only'], 10, null, null, null);
|
|
4697
|
+
var rankedLogos = Object.values(logoCandidates).filter(function(candidate) { return candidate.region !== 'body'; }).sort(function(a, b) { return b.score - a.score || a.url.localeCompare(b.url); }).slice(0, 20);
|
|
4698
|
+
for (var ri = 0; ri < rankedLogos.length; ri++) {
|
|
4699
|
+
rankedLogos[ri].confidence = rankedLogos[ri].score >= 85 ? 'high' : rankedLogos[ri].score >= 60 ? 'medium' : 'low';
|
|
4700
|
+
}
|
|
4701
|
+
var primaryLogo = rankedLogos.length && rankedLogos[0].score >= 55 && rankedLogos[0].region !== 'favicon' ? rankedLogos[0] : null;
|
|
4328
4702
|
return {
|
|
4329
|
-
navBg: navStyle ? navStyle.backgroundColor : null,
|
|
4330
|
-
bodyBg: bodyStyle ? bodyStyle.backgroundColor : null,
|
|
4331
|
-
bodyColor: bodyStyle ? bodyStyle.color : null,
|
|
4332
|
-
h1Color: h1Style ? h1Style.color : null,
|
|
4333
|
-
btnBg: btnStyle ? btnStyle.backgroundColor : null,
|
|
4703
|
+
navBg: navStyle ? toHex(navStyle.backgroundColor) : null,
|
|
4704
|
+
bodyBg: (bodyStyle ? toHex(bodyStyle.backgroundColor) : null) || (cs(document.documentElement) ? toHex(cs(document.documentElement).backgroundColor) : null) || '#ffffff',
|
|
4705
|
+
bodyColor: bodyStyle ? toHex(bodyStyle.color) : null,
|
|
4706
|
+
h1Color: h1Style ? toHex(h1Style.color) : null,
|
|
4707
|
+
btnBg: btnStyle ? toHex(btnStyle.backgroundColor) : null,
|
|
4708
|
+
btnDetail: btnEl ? String(btnEl.textContent || btnEl.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim().slice(0, 120) : null,
|
|
4334
4709
|
bodyFont: bodyStyle ? bodyStyle.fontFamily : null,
|
|
4335
4710
|
h1Font: h1Style ? h1Style.fontFamily : null,
|
|
4336
|
-
logoSrc:
|
|
4711
|
+
logoSrc: primaryLogo ? primaryLogo.url : null,
|
|
4712
|
+
logoConfidence: primaryLogo ? primaryLogo.confidence : null,
|
|
4713
|
+
logoSelectionReason: primaryLogo ? primaryLogo.evidence.join(', ') : null,
|
|
4714
|
+
logoCandidates: rankedLogos,
|
|
4715
|
+
proofCandidates: Object.values(proofCandidates),
|
|
4337
4716
|
faviconHref: faviconEl ? faviconEl.href : null,
|
|
4338
4717
|
svgFreq: svgFreq,
|
|
4339
4718
|
navChildBgFreq: navChildBgFreq,
|
|
4719
|
+
visibleBgFreq: visibleBgFreq,
|
|
4720
|
+
topBgFreq: topBgFreq,
|
|
4721
|
+
cssPrimaryCandidates: cssPrimaryCandidates,
|
|
4722
|
+
cssAccentCandidates: cssAccentCandidates,
|
|
4723
|
+
themeColor: themeColor,
|
|
4340
4724
|
};
|
|
4341
4725
|
})()
|
|
4342
4726
|
`;
|
|
4343
4727
|
const raw = await page.evaluate(evalScript);
|
|
4344
|
-
const
|
|
4345
|
-
const
|
|
4728
|
+
const logoSelection = selectRankedLogoCandidates(raw.logoCandidates ?? []);
|
|
4729
|
+
const navBgHex = raw.navBg;
|
|
4730
|
+
const bodyBgHex = raw.bodyBg;
|
|
4346
4731
|
const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
|
|
4347
|
-
const svgPrimary =
|
|
4732
|
+
const svgPrimary = dominantChromaticColor(raw.svgFreq ?? {});
|
|
4348
4733
|
const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
|
|
4349
|
-
const
|
|
4350
|
-
const
|
|
4351
|
-
const
|
|
4352
|
-
const
|
|
4734
|
+
const visibleBg = dominantColor(raw.visibleBgFreq ?? {});
|
|
4735
|
+
const topBg = dominantColor(raw.topBgFreq ?? {});
|
|
4736
|
+
const cssPrimary = raw.cssPrimaryCandidates?.[0] ?? null;
|
|
4737
|
+
const cssAccent = raw.cssAccentCandidates?.[0] ?? null;
|
|
4738
|
+
const themeColor = isChromatic(raw.themeColor) ? raw.themeColor : null;
|
|
4739
|
+
const ctaColor = isChromatic(raw.btnBg) ? raw.btnBg : null;
|
|
4740
|
+
const primaryChoice = cssPrimary ? { value: cssPrimary.hex, source: "css_variable", confidence: "high", detail: cssPrimary.name } : themeColor ? { value: themeColor, source: "theme_color", confidence: "high", detail: 'meta[name="theme-color"]' } : navBgUsable && isChromatic(navBgUsable) ? { value: navBgUsable, source: "navigation_background", confidence: "medium", detail: "nav, header, or banner computed background" } : topBg && isChromatic(topBg) ? { value: topBg, source: "navigation_background", confidence: "medium", detail: "dominant wide rendered background near the top of the page" } : svgPrimary && isChromatic(svgPrimary) ? { value: svgPrimary, source: "header_svg", confidence: "medium", detail: "dominant rendered identity SVG fill near the top of the page" } : navChildBg && isChromatic(navChildBg) ? { value: navChildBg, source: "navigation_background", confidence: "medium", detail: "dominant rendered navigation child background" } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : visibleBg && isChromatic(visibleBg) ? { value: visibleBg, source: "visible_background", confidence: "low", detail: "area-weighted rendered element background" } : bodyBgHex && !isTransparentOrWhite(bodyBgHex) ? { value: bodyBgHex, source: "body_background", confidence: "low", detail: "computed body background fallback" } : null;
|
|
4741
|
+
const accentChoice = cssAccent ? { value: cssAccent.hex, source: "css_variable", confidence: "high", detail: cssAccent.name } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : null;
|
|
4742
|
+
const primary = primaryChoice?.value ?? null;
|
|
4743
|
+
const accent = accentChoice?.value ?? null;
|
|
4744
|
+
const text2 = raw.bodyColor;
|
|
4745
|
+
const heading = raw.h1Color;
|
|
4353
4746
|
const background = bodyBgHex;
|
|
4354
4747
|
const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
|
|
4355
4748
|
return {
|
|
@@ -4361,14 +4754,200 @@ async function extractBrandingFromPage(page) {
|
|
|
4361
4754
|
text: text2 ?? null,
|
|
4362
4755
|
heading: heading ?? null
|
|
4363
4756
|
},
|
|
4757
|
+
colorEvidence: {
|
|
4758
|
+
primary: primaryChoice,
|
|
4759
|
+
accent: accentChoice,
|
|
4760
|
+
background: background ? { value: background, source: "body_background", confidence: "high", detail: "computed body background" } : null,
|
|
4761
|
+
text: text2 ? { value: text2, source: "body_text", confidence: "high", detail: "computed body text color" } : null,
|
|
4762
|
+
heading: heading ? { value: heading, source: "heading_text", confidence: "high", detail: "computed first H1 text color" } : null
|
|
4763
|
+
},
|
|
4364
4764
|
fonts: {
|
|
4365
4765
|
body: firstFont(raw.bodyFont ?? ""),
|
|
4366
4766
|
heading: firstFont(raw.h1Font ?? "")
|
|
4367
4767
|
},
|
|
4368
4768
|
assets: {
|
|
4369
|
-
logo:
|
|
4370
|
-
favicon: raw.faviconHref || null
|
|
4769
|
+
logo: logoSelection.primary?.url ?? null,
|
|
4770
|
+
favicon: raw.faviconHref || null,
|
|
4771
|
+
logoConfidence: logoSelection.primary?.confidence ?? null,
|
|
4772
|
+
logoSelectionReason: logoSelection.selectionReason,
|
|
4773
|
+
logoVariants: logoSelection.variants,
|
|
4774
|
+
logoCandidates: logoSelection.candidates,
|
|
4775
|
+
proofImages: selectProofImages(raw.proofCandidates ?? [])
|
|
4776
|
+
}
|
|
4777
|
+
};
|
|
4778
|
+
}
|
|
4779
|
+
|
|
4780
|
+
// src/lib/rendered-media.ts
|
|
4781
|
+
var SETTLE_MS = 450;
|
|
4782
|
+
var STABLE_CHECKS_REQUIRED = 3;
|
|
4783
|
+
var MAX_SCROLL_ROUNDS = 40;
|
|
4784
|
+
var MAX_DOM_NODES = 15e3;
|
|
4785
|
+
function mergeAsset(target, incoming) {
|
|
4786
|
+
const existing = target.get(incoming.url);
|
|
4787
|
+
if (!existing) {
|
|
4788
|
+
target.set(incoming.url, incoming);
|
|
4789
|
+
return;
|
|
4790
|
+
}
|
|
4791
|
+
existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
|
|
4792
|
+
existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
|
|
4793
|
+
existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
|
|
4794
|
+
if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
|
|
4795
|
+
existing.width = incoming.width;
|
|
4796
|
+
existing.height = incoming.height;
|
|
4797
|
+
}
|
|
4798
|
+
}
|
|
4799
|
+
async function readRenderedSnapshot(page) {
|
|
4800
|
+
await page.evaluate("globalThis.__name = globalThis.__name || ((target) => target)");
|
|
4801
|
+
return page.evaluate(({ maxDomNodes }) => {
|
|
4802
|
+
const rows = /* @__PURE__ */ new Map();
|
|
4803
|
+
const resolve2 = (raw) => {
|
|
4804
|
+
if (!raw || raw.startsWith("data:") || raw.startsWith("blob:")) return null;
|
|
4805
|
+
try {
|
|
4806
|
+
const parsed = new URL(raw, document.baseURI);
|
|
4807
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
4808
|
+
parsed.hash = "";
|
|
4809
|
+
return parsed.href;
|
|
4810
|
+
} catch {
|
|
4811
|
+
return null;
|
|
4812
|
+
}
|
|
4813
|
+
};
|
|
4814
|
+
const contextFor = (element) => {
|
|
4815
|
+
const contexts = [];
|
|
4816
|
+
const section2 = element.closest('section,article,main,header,footer,aside,[role="region"]');
|
|
4817
|
+
if (section2?.id) contexts.push(`#${section2.id}`);
|
|
4818
|
+
const labelled = section2?.getAttribute("aria-label")?.trim();
|
|
4819
|
+
if (labelled) contexts.push(labelled.slice(0, 200));
|
|
4820
|
+
const heading = section2?.querySelector("h1,h2,h3,h4,h5,h6")?.textContent?.replace(/\s+/g, " ").trim();
|
|
4821
|
+
if (heading) contexts.push(heading.slice(0, 200));
|
|
4822
|
+
return contexts;
|
|
4823
|
+
};
|
|
4824
|
+
const add = (raw, type, method, element, dimensions, alt) => {
|
|
4825
|
+
const url = resolve2(raw);
|
|
4826
|
+
if (!url) return;
|
|
4827
|
+
const existing = rows.get(url) ?? {
|
|
4828
|
+
url,
|
|
4829
|
+
type,
|
|
4830
|
+
discoveryMethods: [],
|
|
4831
|
+
altTexts: [],
|
|
4832
|
+
contexts: [],
|
|
4833
|
+
width: null,
|
|
4834
|
+
height: null
|
|
4835
|
+
};
|
|
4836
|
+
existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, method])];
|
|
4837
|
+
const cleanAlt = alt?.replace(/\s+/g, " ").trim();
|
|
4838
|
+
if (cleanAlt) existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, cleanAlt.slice(0, 500)])];
|
|
4839
|
+
existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...contextFor(element)])];
|
|
4840
|
+
if (dimensions && dimensions.width > 0 && dimensions.height > 0) {
|
|
4841
|
+
const currentArea = (existing.width ?? 0) * (existing.height ?? 0);
|
|
4842
|
+
if (dimensions.width * dimensions.height > currentArea) {
|
|
4843
|
+
existing.width = dimensions.width;
|
|
4844
|
+
existing.height = dimensions.height;
|
|
4845
|
+
}
|
|
4846
|
+
}
|
|
4847
|
+
rows.set(url, existing);
|
|
4848
|
+
};
|
|
4849
|
+
const srcsetUrls = (value) => (value ?? "").split(",").map((part) => part.trim().split(/\s+/)[0] ?? "").filter(Boolean);
|
|
4850
|
+
for (const img of Array.from(document.querySelectorAll("img"))) {
|
|
4851
|
+
const image = img;
|
|
4852
|
+
const dimensions = { width: image.naturalWidth, height: image.naturalHeight };
|
|
4853
|
+
add(image.currentSrc, "image", "rendered-current-src", image, dimensions, image.alt);
|
|
4854
|
+
add(image.getAttribute("src"), "image", "rendered-src", image, dimensions, image.alt);
|
|
4855
|
+
for (const attr of ["data-src", "data-lazy-src", "data-original", "data-bg", "data-background"]) {
|
|
4856
|
+
add(image.getAttribute(attr), "image", `rendered-${attr}`, image, dimensions, image.alt);
|
|
4857
|
+
}
|
|
4858
|
+
for (const url of srcsetUrls(image.getAttribute("srcset"))) add(url, "image", "rendered-srcset", image, dimensions, image.alt);
|
|
4859
|
+
const anchor = image.closest("a[href]");
|
|
4860
|
+
const anchorAttrs = anchor ? Array.from(anchor.attributes).map((attribute) => `${attribute.name}=${attribute.value}`).join(" ") : "";
|
|
4861
|
+
if (anchor && (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(anchor.href))) add(anchor.href, "image", "rendered-lightbox-target", image, void 0, image.alt);
|
|
4862
|
+
}
|
|
4863
|
+
for (const source of Array.from(document.querySelectorAll("source"))) {
|
|
4864
|
+
const parent = source.closest("picture,video,audio");
|
|
4865
|
+
const type = parent?.tagName === "VIDEO" ? "video" : parent?.tagName === "AUDIO" ? "audio" : "image";
|
|
4866
|
+
add(source.getAttribute("src"), type, "rendered-source", source);
|
|
4867
|
+
for (const url of srcsetUrls(source.getAttribute("srcset"))) add(url, type, "rendered-source-srcset", source);
|
|
4868
|
+
}
|
|
4869
|
+
for (const video of Array.from(document.querySelectorAll("video"))) {
|
|
4870
|
+
const media = video;
|
|
4871
|
+
add(media.currentSrc || media.src, "video", "rendered-video", media);
|
|
4872
|
+
add(media.poster, "image", "rendered-video-poster", media, { width: media.videoWidth, height: media.videoHeight });
|
|
4873
|
+
}
|
|
4874
|
+
for (const audio of Array.from(document.querySelectorAll("audio"))) {
|
|
4875
|
+
const media = audio;
|
|
4876
|
+
add(media.currentSrc || media.src, "audio", "rendered-audio", media);
|
|
4877
|
+
}
|
|
4878
|
+
for (const svgImage of Array.from(document.querySelectorAll("svg image"))) {
|
|
4879
|
+
add(svgImage.getAttribute("href") || svgImage.getAttribute("xlink:href"), "image", "rendered-svg-image", svgImage);
|
|
4880
|
+
}
|
|
4881
|
+
const all = Array.from(document.querySelectorAll("*")).slice(0, maxDomNodes);
|
|
4882
|
+
for (const element of all) {
|
|
4883
|
+
const background = getComputedStyle(element).backgroundImage;
|
|
4884
|
+
if (!background || background === "none") continue;
|
|
4885
|
+
for (const match of background.matchAll(/url\(["']?(.+?)["']?\)/gi)) {
|
|
4886
|
+
add(match[1], "image", "rendered-computed-background", element);
|
|
4887
|
+
}
|
|
4371
4888
|
}
|
|
4889
|
+
const documentHeight = Math.max(document.body?.scrollHeight ?? 0, document.documentElement?.scrollHeight ?? 0);
|
|
4890
|
+
const viewportHeight = window.innerHeight;
|
|
4891
|
+
return {
|
|
4892
|
+
assets: [...rows.values()],
|
|
4893
|
+
documentHeight,
|
|
4894
|
+
scrollY: window.scrollY,
|
|
4895
|
+
viewportHeight,
|
|
4896
|
+
atBottom: window.scrollY + viewportHeight >= documentHeight - 5
|
|
4897
|
+
};
|
|
4898
|
+
}, { maxDomNodes: MAX_DOM_NODES });
|
|
4899
|
+
}
|
|
4900
|
+
async function collectRenderedPageMedia(page, maxAssets = 1e3) {
|
|
4901
|
+
const boundedMax = Math.max(1, Math.min(maxAssets, 2e3));
|
|
4902
|
+
await page.evaluate(() => {
|
|
4903
|
+
for (const image of Array.from(document.querySelectorAll('img[loading="lazy"]'))) image.loading = "eager";
|
|
4904
|
+
}).catch(() => void 0);
|
|
4905
|
+
const assets = /* @__PURE__ */ new Map();
|
|
4906
|
+
let stableChecks = 0;
|
|
4907
|
+
let previousHeight = -1;
|
|
4908
|
+
let rounds = 0;
|
|
4909
|
+
let documentHeight = 0;
|
|
4910
|
+
while (rounds < MAX_SCROLL_ROUNDS) {
|
|
4911
|
+
rounds += 1;
|
|
4912
|
+
const before = assets.size;
|
|
4913
|
+
const snapshot = await readRenderedSnapshot(page);
|
|
4914
|
+
documentHeight = snapshot.documentHeight;
|
|
4915
|
+
for (const asset of snapshot.assets) mergeAsset(assets, asset);
|
|
4916
|
+
if (assets.size >= boundedMax) {
|
|
4917
|
+
return {
|
|
4918
|
+
assets: [...assets.values()].slice(0, boundedMax),
|
|
4919
|
+
exhausted: false,
|
|
4920
|
+
stopReason: "asset_limit",
|
|
4921
|
+
scrollRounds: rounds,
|
|
4922
|
+
documentHeight
|
|
4923
|
+
};
|
|
4924
|
+
}
|
|
4925
|
+
const unchanged = snapshot.atBottom && before === assets.size && previousHeight === snapshot.documentHeight;
|
|
4926
|
+
stableChecks = unchanged ? stableChecks + 1 : 0;
|
|
4927
|
+
previousHeight = snapshot.documentHeight;
|
|
4928
|
+
if (stableChecks >= STABLE_CHECKS_REQUIRED) {
|
|
4929
|
+
await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
|
|
4930
|
+
return {
|
|
4931
|
+
assets: [...assets.values()],
|
|
4932
|
+
exhausted: true,
|
|
4933
|
+
stopReason: "page_exhausted",
|
|
4934
|
+
scrollRounds: rounds,
|
|
4935
|
+
documentHeight
|
|
4936
|
+
};
|
|
4937
|
+
}
|
|
4938
|
+
await page.evaluate(({ viewportHeight, documentHeight: height }) => {
|
|
4939
|
+
const next = Math.min(Math.max(0, height - viewportHeight), window.scrollY + Math.max(800, Math.floor(viewportHeight * 0.9)));
|
|
4940
|
+
window.scrollTo(0, next);
|
|
4941
|
+
}, { viewportHeight: snapshot.viewportHeight, documentHeight: snapshot.documentHeight });
|
|
4942
|
+
await page.waitForTimeout(SETTLE_MS);
|
|
4943
|
+
}
|
|
4944
|
+
await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
|
|
4945
|
+
return {
|
|
4946
|
+
assets: [...assets.values()],
|
|
4947
|
+
exhausted: false,
|
|
4948
|
+
stopReason: "scroll_round_limit",
|
|
4949
|
+
scrollRounds: rounds,
|
|
4950
|
+
documentHeight
|
|
4372
4951
|
};
|
|
4373
4952
|
}
|
|
4374
4953
|
|
|
@@ -4376,7 +4955,7 @@ async function extractBrandingFromPage(page) {
|
|
|
4376
4955
|
var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
|
|
4377
4956
|
var MOBILE_VIEWPORT = { width: 390, height: 844 };
|
|
4378
4957
|
var SCROLL_STEP = 900;
|
|
4379
|
-
var
|
|
4958
|
+
var SETTLE_MS2 = 400;
|
|
4380
4959
|
async function scrollAndStitch(page, device = "desktop") {
|
|
4381
4960
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
4382
4961
|
await page.setViewportSize(vp);
|
|
@@ -4392,7 +4971,7 @@ async function scrollAndStitch(page, device = "desktop") {
|
|
|
4392
4971
|
let scrollY = 0;
|
|
4393
4972
|
while (scrollY < totalH) {
|
|
4394
4973
|
await page.evaluate((y) => window.scrollTo(0, y), scrollY);
|
|
4395
|
-
await page.waitForTimeout(
|
|
4974
|
+
await page.waitForTimeout(SETTLE_MS2);
|
|
4396
4975
|
const remaining = totalH - scrollY;
|
|
4397
4976
|
const stripH = Math.min(SCROLL_STEP, remaining);
|
|
4398
4977
|
const buf = await page.screenshot({
|
|
@@ -4423,7 +5002,7 @@ async function scrollAndStitch(page, device = "desktop") {
|
|
|
4423
5002
|
);
|
|
4424
5003
|
return Buffer.from(base64, "base64");
|
|
4425
5004
|
}
|
|
4426
|
-
async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
5005
|
+
async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
4427
5006
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
4428
5007
|
const browser = await chromium2.launch({ headless: true });
|
|
4429
5008
|
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
@@ -4434,17 +5013,18 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
|
4434
5013
|
}, runtimeUserAgent));
|
|
4435
5014
|
const page = await context.newPage();
|
|
4436
5015
|
try {
|
|
4437
|
-
await page.goto(url, { waitUntil: "
|
|
5016
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
4438
5017
|
await page.waitForTimeout(1500);
|
|
4439
5018
|
const result = {};
|
|
4440
5019
|
if (wantBranding) result.branding = await extractBrandingFromPage(page);
|
|
5020
|
+
if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
|
|
4441
5021
|
if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
|
|
4442
5022
|
return result;
|
|
4443
5023
|
} finally {
|
|
4444
5024
|
await browser.close();
|
|
4445
5025
|
}
|
|
4446
5026
|
}
|
|
4447
|
-
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
|
|
5027
|
+
async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
|
|
4448
5028
|
const kernel = new Kernel2({ apiKey: kernelApiKey });
|
|
4449
5029
|
const openedAtMs = Date.now();
|
|
4450
5030
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
@@ -4457,10 +5037,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
4457
5037
|
}, runtimeUserAgent));
|
|
4458
5038
|
const page = context.pages()[0] ?? await context.newPage();
|
|
4459
5039
|
try {
|
|
4460
|
-
await page.goto(url, { waitUntil: "
|
|
5040
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
4461
5041
|
await page.waitForTimeout(1500);
|
|
4462
5042
|
const result = {};
|
|
4463
5043
|
if (wantBranding) result.branding = await extractBrandingFromPage(page);
|
|
5044
|
+
if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
|
|
4464
5045
|
if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
|
|
4465
5046
|
return result;
|
|
4466
5047
|
} finally {
|
|
@@ -4480,12 +5061,14 @@ async function capturePageData(url, opts = {}) {
|
|
|
4480
5061
|
const device = opts.device ?? "desktop";
|
|
4481
5062
|
const wantShot = opts.screenshot ?? false;
|
|
4482
5063
|
const wantBranding = opts.branding ?? false;
|
|
5064
|
+
const wantMedia = opts.media ?? false;
|
|
5065
|
+
const maxMediaAssets = opts.maxMediaAssets ?? 1e3;
|
|
4483
5066
|
try {
|
|
4484
|
-
return await captureLocal(url, device, wantShot, wantBranding);
|
|
5067
|
+
return await captureLocal(url, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
4485
5068
|
} catch (localErr) {
|
|
4486
5069
|
const msg = localErr instanceof Error ? localErr.message : String(localErr);
|
|
4487
5070
|
if (!opts.kernelApiKey) throw new Error(`Local capture failed (${msg}) and no headless browser backend is configured`);
|
|
4488
|
-
return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
|
|
5071
|
+
return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
|
|
4489
5072
|
}
|
|
4490
5073
|
}
|
|
4491
5074
|
|
|
@@ -4890,7 +5473,7 @@ function withTimeout(p, ms) {
|
|
|
4890
5473
|
async function createBrowserWithRetry(client2, headless) {
|
|
4891
5474
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
4892
5475
|
try {
|
|
4893
|
-
return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
|
|
5476
|
+
return await client2.browsers.create({ stealth: true, ...headless ? { headless: true } : {}, timeout_seconds: 120 });
|
|
4894
5477
|
} catch (err) {
|
|
4895
5478
|
if (attempt === 2) throw err;
|
|
4896
5479
|
await new Promise((r) => setTimeout(r, 5e3));
|
|
@@ -4899,7 +5482,7 @@ async function createBrowserWithRetry(client2, headless) {
|
|
|
4899
5482
|
throw new Error("unreachable");
|
|
4900
5483
|
}
|
|
4901
5484
|
var PAGE_HARD_TIMEOUT_MS = 35e3;
|
|
4902
|
-
async function crawlPage(context, url) {
|
|
5485
|
+
async function crawlPage(context, url, headless) {
|
|
4903
5486
|
const startedAt = Date.now();
|
|
4904
5487
|
let page = null;
|
|
4905
5488
|
let guardState = null;
|
|
@@ -4908,13 +5491,38 @@ async function crawlPage(context, url) {
|
|
|
4908
5491
|
guardState = await installBrowserRequestGuard(page, url);
|
|
4909
5492
|
const work = (async (p) => {
|
|
4910
5493
|
const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
4911
|
-
|
|
5494
|
+
let html = await p.evaluate((maxBytes) => {
|
|
4912
5495
|
const value = document.documentElement?.outerHTML ?? "";
|
|
4913
5496
|
if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
|
|
4914
5497
|
return value;
|
|
4915
5498
|
}, MAX_PAGE_HTML_BYTES);
|
|
4916
5499
|
const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
|
|
4917
5500
|
const finalUrl = p.url();
|
|
5501
|
+
let challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
|
|
5502
|
+
if (challenge.detected && !headless) {
|
|
5503
|
+
const deadline = Date.now() + 2e4;
|
|
5504
|
+
while (challenge.detected && Date.now() < deadline) {
|
|
5505
|
+
await p.waitForTimeout(1e3);
|
|
5506
|
+
html = await p.content();
|
|
5507
|
+
challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
|
|
5508
|
+
}
|
|
5509
|
+
}
|
|
5510
|
+
if (challenge.detected) {
|
|
5511
|
+
return {
|
|
5512
|
+
url,
|
|
5513
|
+
html: null,
|
|
5514
|
+
status: resp?.status() ?? 0,
|
|
5515
|
+
via: "browser",
|
|
5516
|
+
headers,
|
|
5517
|
+
responseTimeMs: Date.now() - startedAt,
|
|
5518
|
+
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
|
|
5519
|
+
errorCode: "browser_challenge_unresolved",
|
|
5520
|
+
error: `Browser challenge remained unresolved (${challenge.reason ?? "unknown challenge"}).`,
|
|
5521
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5522
|
+
challengeDetected: true,
|
|
5523
|
+
stopReason: "challenge_unresolved"
|
|
5524
|
+
};
|
|
5525
|
+
}
|
|
4918
5526
|
return {
|
|
4919
5527
|
url,
|
|
4920
5528
|
html,
|
|
@@ -4922,7 +5530,10 @@ async function crawlPage(context, url) {
|
|
|
4922
5530
|
via: "browser",
|
|
4923
5531
|
headers,
|
|
4924
5532
|
responseTimeMs: Date.now() - startedAt,
|
|
4925
|
-
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null
|
|
5533
|
+
redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
|
|
5534
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5535
|
+
challengeDetected: false,
|
|
5536
|
+
stopReason: resp && (resp.status() < 200 || resp.status() >= 300) ? "http_error" : "usable_page"
|
|
4926
5537
|
};
|
|
4927
5538
|
})(page);
|
|
4928
5539
|
const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
|
|
@@ -4937,7 +5548,10 @@ async function crawlPage(context, url) {
|
|
|
4937
5548
|
via: "browser",
|
|
4938
5549
|
responseTimeMs: Date.now() - startedAt,
|
|
4939
5550
|
errorCode,
|
|
4940
|
-
error: message.slice(0, 500)
|
|
5551
|
+
error: message.slice(0, 500),
|
|
5552
|
+
browserMode: headless ? "stealth_headless" : "stealth_headful",
|
|
5553
|
+
challengeDetected: false,
|
|
5554
|
+
stopReason: "navigation_failed"
|
|
4941
5555
|
};
|
|
4942
5556
|
} finally {
|
|
4943
5557
|
if (page) await page.close().catch(() => {
|
|
@@ -4951,7 +5565,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4951
5565
|
const headless = opts.headless ?? DEFAULT_HEADLESS;
|
|
4952
5566
|
const uniqueUrls = [...new Set(urls)];
|
|
4953
5567
|
const byUrl = /* @__PURE__ */ new Map();
|
|
4954
|
-
const runPass = async (targets, phase) => {
|
|
5568
|
+
const runPass = async (targets, phase, passHeadless) => {
|
|
4955
5569
|
if (targets.length === 0) return;
|
|
4956
5570
|
let next = 0;
|
|
4957
5571
|
let done = 0;
|
|
@@ -4974,7 +5588,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4974
5588
|
const closedAtMs = Date.now();
|
|
4975
5589
|
await client2.browsers.deleteByID(k.session_id).catch(() => {
|
|
4976
5590
|
});
|
|
4977
|
-
await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent:
|
|
5591
|
+
await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: passHeadless ? true : null, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, fallback: phase === "retry", openedAtMs, closedAtMs });
|
|
4978
5592
|
}
|
|
4979
5593
|
};
|
|
4980
5594
|
try {
|
|
@@ -4986,7 +5600,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4986
5600
|
try {
|
|
4987
5601
|
if (!browser || inBrowser >= urlsPerBrowser) {
|
|
4988
5602
|
await retire();
|
|
4989
|
-
kb = await createBrowserWithRetry(client2,
|
|
5603
|
+
kb = await createBrowserWithRetry(client2, passHeadless);
|
|
4990
5604
|
kbOpenedAtMs = Date.now();
|
|
4991
5605
|
browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
|
|
4992
5606
|
runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
@@ -4994,7 +5608,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
4994
5608
|
}
|
|
4995
5609
|
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
4996
5610
|
try {
|
|
4997
|
-
r = await crawlPage(context, url);
|
|
5611
|
+
r = await crawlPage(context, url, passHeadless);
|
|
4998
5612
|
} finally {
|
|
4999
5613
|
await context.close().catch(() => {
|
|
5000
5614
|
});
|
|
@@ -5008,7 +5622,10 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5008
5622
|
status: 0,
|
|
5009
5623
|
via: "browser",
|
|
5010
5624
|
errorCode: "browser_session_failed",
|
|
5011
|
-
error: message.slice(0, 500)
|
|
5625
|
+
error: message.slice(0, 500),
|
|
5626
|
+
browserMode: passHeadless ? "stealth_headless" : "stealth_headful",
|
|
5627
|
+
challengeDetected: false,
|
|
5628
|
+
stopReason: "navigation_failed"
|
|
5012
5629
|
};
|
|
5013
5630
|
await retire();
|
|
5014
5631
|
}
|
|
@@ -5032,12 +5649,12 @@ async function crawlWithRotation(urls, opts) {
|
|
|
5032
5649
|
};
|
|
5033
5650
|
await Promise.all(Array.from({ length: workers }, () => worker()));
|
|
5034
5651
|
};
|
|
5035
|
-
await runPass(uniqueUrls, "main");
|
|
5652
|
+
await runPass(uniqueUrls, "main", headless);
|
|
5036
5653
|
const failed = uniqueUrls.filter((u) => {
|
|
5037
5654
|
const r = byUrl.get(u);
|
|
5038
5655
|
return !r || !r.html || r.status < 200 || r.status >= 300;
|
|
5039
5656
|
});
|
|
5040
|
-
if (failed.length > 0) await runPass(failed, "retry");
|
|
5657
|
+
if (failed.length > 0) await runPass(failed, "retry", headless ? false : headless);
|
|
5041
5658
|
return urls.map((u) => byUrl.get(u) ?? {
|
|
5042
5659
|
url: u,
|
|
5043
5660
|
html: null,
|
|
@@ -5749,8 +6366,6 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5749
6366
|
if (text2) headings.push({ level: Number(m[1]), text: text2.slice(0, 200) });
|
|
5750
6367
|
}
|
|
5751
6368
|
const h1 = headings.find((h) => h.level === 1)?.text ?? null;
|
|
5752
|
-
const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
5753
|
-
const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
|
|
5754
6369
|
const schemaTypes = [];
|
|
5755
6370
|
let schemaTypeBytes = 0;
|
|
5756
6371
|
const addSchemaType = (value) => {
|
|
@@ -5807,11 +6422,14 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5807
6422
|
}
|
|
5808
6423
|
}
|
|
5809
6424
|
const mainHtml = (html.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? html).replace(/<(script|style|nav|header|footer|aside|noscript|svg|iframe)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
6425
|
+
const mainText = mainHtml.replace(/<[^>]+>/g, " ").replace(/&(?:nbsp|#160);/gi, " ").replace(/&(?:amp|#38);/gi, "&").replace(/&#(?:x[0-9a-f]+|\d+);/gi, " ").replace(/\s+/g, " ").trim();
|
|
6426
|
+
const mainWords = mainText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
|
|
6427
|
+
const wordCount2 = mainWords.filter((word) => word.length > 2).length;
|
|
5810
6428
|
let bodyMarkdown = "";
|
|
5811
6429
|
try {
|
|
5812
6430
|
bodyMarkdown = turndown.turndown(mainHtml).replace(/\n{3,}/g, "\n\n").trim().slice(0, MAX_PAGE_MARKDOWN);
|
|
5813
6431
|
} catch {
|
|
5814
|
-
bodyMarkdown =
|
|
6432
|
+
bodyMarkdown = mainText.slice(0, MAX_PAGE_MARKDOWN);
|
|
5815
6433
|
}
|
|
5816
6434
|
const outlinks = [];
|
|
5817
6435
|
const discoveryLinkSet = /* @__PURE__ */ new Set();
|
|
@@ -5839,10 +6457,11 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5839
6457
|
const internal = abs.origin === origin;
|
|
5840
6458
|
if (internal) {
|
|
5841
6459
|
internalLinks++;
|
|
5842
|
-
|
|
5843
|
-
|
|
6460
|
+
const crawlable = normalizeUrl(abs.href, documentUrl);
|
|
6461
|
+
if (crawlable && !discoveryLinkSet.has(crawlable)) {
|
|
6462
|
+
const bytes2 = Buffer.byteLength(crawlable);
|
|
5844
6463
|
if (discoveryLinkSet.size < PAGE_DATA_LIMITS.discoveryLinks && discoveryBytes + bytes2 <= PAGE_DATA_LIMITS.discoveryBytes) {
|
|
5845
|
-
discoveryLinkSet.add(
|
|
6464
|
+
discoveryLinkSet.add(crawlable);
|
|
5846
6465
|
discoveryBytes += bytes2;
|
|
5847
6466
|
} else truncatedFields.add("discoveryLinks");
|
|
5848
6467
|
}
|
|
@@ -5864,8 +6483,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5864
6483
|
const h1List = headings.filter((h) => h.level === 1);
|
|
5865
6484
|
const h1_2 = h1List[1]?.text ?? null;
|
|
5866
6485
|
const h2Count = headings.filter((h) => h.level === 2).length;
|
|
5867
|
-
const textRatio = Number((
|
|
5868
|
-
const contentHash = createHash("sha1").update(
|
|
6486
|
+
const textRatio = Number((mainText.length / Math.max(1, html.length)).toFixed(3));
|
|
6487
|
+
const contentHash = createHash("sha1").update(mainText).digest("hex");
|
|
5869
6488
|
const metaRobots = boundedText(html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1], 512)?.toLowerCase() ?? null;
|
|
5870
6489
|
const relNext = boundedUrl(html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]);
|
|
5871
6490
|
const relPrev = boundedUrl(html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]);
|
|
@@ -5919,11 +6538,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5919
6538
|
}
|
|
5920
6539
|
}
|
|
5921
6540
|
}
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
addImageLink(new URL(og.image, documentUrl).href);
|
|
5925
|
-
} catch {
|
|
5926
|
-
}
|
|
6541
|
+
for (const asset of discoverStaticMedia(html, documentUrl)) {
|
|
6542
|
+
if (asset.type === "image") addImageLink(asset.url);
|
|
5927
6543
|
}
|
|
5928
6544
|
const imageLinks = [...imageLinkSet];
|
|
5929
6545
|
const canonNorm = (u) => {
|
|
@@ -5950,12 +6566,19 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
5950
6566
|
indexable = false;
|
|
5951
6567
|
indexabilityReason = "canonicalised";
|
|
5952
6568
|
}
|
|
5953
|
-
const
|
|
6569
|
+
const bodyClass = html.match(/<body\b[^>]*\bclass\s*=\s*["']([^"']*)["']/i)?.[1] ?? "";
|
|
6570
|
+
const wordpressAttachment = /(?:^|\s)(?:attachment|attachment-template-default)(?:\s|$)/i.test(bodyClass);
|
|
6571
|
+
const meaningfulText = mainText.replace(/https?:\/\/\S+/gi, " ").replace(/\s+/g, " ").trim();
|
|
6572
|
+
const meaningfulWords = meaningfulText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
|
|
6573
|
+
const hasMeaningfulText = meaningfulText.length >= 24 && meaningfulWords.length >= 4;
|
|
6574
|
+
const extractionSucceeded = !wordpressAttachment && bodyMarkdown.trim().length > 0 && hasMeaningfulText;
|
|
6575
|
+
const failureCode = wordpressAttachment ? "wordpress_attachment_page" : bodyMarkdown.trim().length === 0 ? "empty_rendered_content" : "insufficient_text_content";
|
|
6576
|
+
const failureReason = wordpressAttachment ? "The HTML is a WordPress media attachment page, not a text-content page." : bodyMarkdown.trim().length === 0 ? "The HTML response produced no usable page content." : `The main content contained only ${meaningfulWords.length} word(s) and ${meaningfulText.length} text character(s).`;
|
|
5954
6577
|
if (!extractionSucceeded) {
|
|
5955
6578
|
indexable = false;
|
|
5956
6579
|
indexabilityReason = "extraction-failed";
|
|
5957
6580
|
}
|
|
5958
|
-
const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode:
|
|
6581
|
+
const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: failureCode, error: failureReason } : attempt);
|
|
5959
6582
|
return {
|
|
5960
6583
|
url,
|
|
5961
6584
|
status,
|
|
@@ -6002,9 +6625,12 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
|
|
|
6002
6625
|
discoveryLinks,
|
|
6003
6626
|
bodyMarkdown,
|
|
6004
6627
|
schema,
|
|
6628
|
+
contentKind: extractionSucceeded ? "text_page" : "insufficient_text",
|
|
6629
|
+
mainTextChars: meaningfulText.length,
|
|
6630
|
+
mainWordCount: meaningfulWords.length,
|
|
6005
6631
|
extractionStatus: extractionSucceeded ? "successful" : "failed",
|
|
6006
|
-
failureCode: extractionSucceeded ? null :
|
|
6007
|
-
failureReason: extractionSucceeded ? null :
|
|
6632
|
+
failureCode: extractionSucceeded ? null : failureCode,
|
|
6633
|
+
failureReason: extractionSucceeded ? null : failureReason,
|
|
6008
6634
|
fetchAttempts: normalizedAttempts,
|
|
6009
6635
|
truncatedFields: [...truncatedFields]
|
|
6010
6636
|
};
|
|
@@ -6092,6 +6718,7 @@ async function fetchPagePlain(url) {
|
|
|
6092
6718
|
const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "non_html_response", `Plain fetch returned ${contentType}, not HTML.`);
|
|
6093
6719
|
const page = emptyPageData(url, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
|
|
6094
6720
|
page.contentType = contentType;
|
|
6721
|
+
page.contentKind = "non_html";
|
|
6095
6722
|
return page;
|
|
6096
6723
|
}
|
|
6097
6724
|
const html = await readResponseTextWithLimit(res, MAX_PAGE_HTML_BYTES, "Page HTML");
|
|
@@ -6134,9 +6761,24 @@ async function mapWithConcurrency(items, concurrency, fn) {
|
|
|
6134
6761
|
}
|
|
6135
6762
|
async function extractPagesRotating(urls, opts) {
|
|
6136
6763
|
const uniqueUrls = [...new Set(urls)];
|
|
6137
|
-
const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY,
|
|
6764
|
+
const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
|
|
6765
|
+
const checked = await validatePublicHttpUrl(url, { field: "page URL" });
|
|
6766
|
+
if (checked.error || !checked.parsed) return fetchPagePlain(url);
|
|
6767
|
+
if (normalizeUrl(checked.parsed.href, checked.parsed.href)) return fetchPagePlain(url);
|
|
6768
|
+
const attempt = failedAttempt("fetch", null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
|
|
6769
|
+
const page = emptyPageData(url, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
|
|
6770
|
+
page.contentKind = "excluded_url";
|
|
6771
|
+
return page;
|
|
6772
|
+
});
|
|
6138
6773
|
const byUrl = new Map(plainPages.map((page) => [page.url, page]));
|
|
6139
|
-
const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
|
|
6774
|
+
const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
|
|
6775
|
+
"unsafe_url",
|
|
6776
|
+
"unsafe_redirect",
|
|
6777
|
+
"cross_site_redirect",
|
|
6778
|
+
"non_html_response",
|
|
6779
|
+
"non_page_url",
|
|
6780
|
+
"wordpress_attachment_page"
|
|
6781
|
+
].includes(page.failureCode ?? "")).map((page) => page.url);
|
|
6140
6782
|
if (opts.kernelApiKey && browserTargets.length > 0) {
|
|
6141
6783
|
const fetched = await crawlWithRotation(browserTargets, {
|
|
6142
6784
|
apiKey: opts.kernelApiKey,
|
|
@@ -8740,7 +9382,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
8740
9382
|
});
|
|
8741
9383
|
}) : null;
|
|
8742
9384
|
const finalized = await step.run("finalize", async () => {
|
|
8743
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9385
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
|
|
8744
9386
|
const current = await getExtractJob(jobId2);
|
|
8745
9387
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
8746
9388
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10071,8 +10713,8 @@ async function saveDirectoryCsvLocal(result) {
|
|
|
10071
10713
|
const outDir = join3(outputBaseDir(), "directory-workflows");
|
|
10072
10714
|
await mkdir(outDir, { recursive: true });
|
|
10073
10715
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
10074
|
-
const
|
|
10075
|
-
const path5 = join3(outDir, `${stamp}-${
|
|
10716
|
+
const slug4 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
10717
|
+
const path5 = join3(outDir, `${stamp}-${slug4}-directory-workflow.csv`);
|
|
10076
10718
|
await writeFile(path5, renderDirectoryWorkflowCsv(result), "utf8");
|
|
10077
10719
|
return path5;
|
|
10078
10720
|
}
|
|
@@ -12214,9 +12856,9 @@ async function listAllPublicLocalSourcebook(category) {
|
|
|
12214
12856
|
}) : await getDb().execute(`SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') ORDER BY s.category, s.state, s.business_name`);
|
|
12215
12857
|
return result.rows.map((row) => parseJson2(row.payload_json));
|
|
12216
12858
|
}
|
|
12217
|
-
async function getPublicLocalSourcebook(category, state,
|
|
12859
|
+
async function getPublicLocalSourcebook(category, state, slug4) {
|
|
12218
12860
|
await ensureLocalSourcebookSchema();
|
|
12219
|
-
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state,
|
|
12861
|
+
const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
|
|
12220
12862
|
return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
|
|
12221
12863
|
}
|
|
12222
12864
|
|
|
@@ -12520,6 +13162,235 @@ var MapsReviewCollector = class {
|
|
|
12520
13162
|
}
|
|
12521
13163
|
};
|
|
12522
13164
|
|
|
13165
|
+
// src/extractor/MapsImageCollector.ts
|
|
13166
|
+
var QUIESCENCE_MS = 700;
|
|
13167
|
+
var STABLE_CHECKS_REQUIRED2 = 3;
|
|
13168
|
+
var MAX_SCROLL_ROUNDS2 = 80;
|
|
13169
|
+
var OWNER_DISCOVERY_LIMIT = 500;
|
|
13170
|
+
function mapsImageIdentity(rawUrl) {
|
|
13171
|
+
try {
|
|
13172
|
+
const url = new URL(rawUrl);
|
|
13173
|
+
const path5 = url.hostname.toLowerCase().endsWith("googleusercontent.com") ? url.pathname.replace(/=[^/]+$/, "") : url.pathname;
|
|
13174
|
+
return `${url.hostname.toLowerCase()}${path5}`;
|
|
13175
|
+
} catch {
|
|
13176
|
+
return rawUrl.split("=")[0] ?? rawUrl;
|
|
13177
|
+
}
|
|
13178
|
+
}
|
|
13179
|
+
function isMapsPhotoUrl(rawUrl) {
|
|
13180
|
+
try {
|
|
13181
|
+
const host2 = new URL(rawUrl).hostname.toLowerCase().replace(/\.$/, "");
|
|
13182
|
+
return host2 === "googleusercontent.com" || host2.endsWith(".googleusercontent.com") || host2 === "streetviewpixels-pa.googleapis.com";
|
|
13183
|
+
} catch {
|
|
13184
|
+
return false;
|
|
13185
|
+
}
|
|
13186
|
+
}
|
|
13187
|
+
async function openPhotoGallery(page) {
|
|
13188
|
+
const seePhotos = page.getByRole("button", { name: /^see photos$/i }).first();
|
|
13189
|
+
const exact = await seePhotos.count() > 0 ? seePhotos : page.locator('[aria-label^="See photos" i], button[aria-label^="Photo of" i]').first();
|
|
13190
|
+
if (await exact.count() === 0) {
|
|
13191
|
+
return false;
|
|
13192
|
+
}
|
|
13193
|
+
const clicked = await exact.click({ timeout: 5e3 }).then(() => true, () => false);
|
|
13194
|
+
if (!clicked) return false;
|
|
13195
|
+
const galleryControl = page.locator('button[aria-label="By owner" i], [role="tab"][aria-label*="owner" i]').first();
|
|
13196
|
+
await galleryControl.waitFor({ state: "visible", timeout: 5e3 }).catch(() => void 0);
|
|
13197
|
+
return galleryControl.isVisible().catch(() => false);
|
|
13198
|
+
}
|
|
13199
|
+
async function selectGalleryTab(page, scope) {
|
|
13200
|
+
const matcher = scope === "owner" ? /by owner/i : /^all(?:\s|$)/i;
|
|
13201
|
+
const direct = page.locator(scope === "owner" ? 'button[aria-label="By owner" i]' : 'button[aria-label="All" i]');
|
|
13202
|
+
const roleTab = page.getByRole("tab", { name: matcher });
|
|
13203
|
+
const tabs = await direct.count() > 0 ? direct : roleTab;
|
|
13204
|
+
if (await tabs.count() === 0) return false;
|
|
13205
|
+
const visible = tabs.filter({ visible: true }).first();
|
|
13206
|
+
if (await visible.count() === 0) return false;
|
|
13207
|
+
const clicked = await visible.click({ timeout: 5e3 }).then(() => true, () => false);
|
|
13208
|
+
if (!clicked) return false;
|
|
13209
|
+
await page.waitForTimeout(QUIESCENCE_MS);
|
|
13210
|
+
return true;
|
|
13211
|
+
}
|
|
13212
|
+
async function readAndAdvanceGallery(page) {
|
|
13213
|
+
return page.evaluate(() => {
|
|
13214
|
+
const mediaNodes = Array.from(document.querySelectorAll('[aria-label^="Photo " i], [aria-label^="Video " i]'));
|
|
13215
|
+
const imageUrl = (root) => {
|
|
13216
|
+
const candidates = [root, ...Array.from(root.querySelectorAll("*"))];
|
|
13217
|
+
for (const node of candidates) {
|
|
13218
|
+
if (node instanceof HTMLImageElement) {
|
|
13219
|
+
const value = node.currentSrc || node.src;
|
|
13220
|
+
if (value) return value;
|
|
13221
|
+
}
|
|
13222
|
+
const inline = node.style.backgroundImage;
|
|
13223
|
+
const computed = inline || window.getComputedStyle(node).backgroundImage;
|
|
13224
|
+
const match = computed?.match(/url\(["']?(.+?)["']?\)/i);
|
|
13225
|
+
if (match?.[1]) return match[1];
|
|
13226
|
+
}
|
|
13227
|
+
return null;
|
|
13228
|
+
};
|
|
13229
|
+
const items = mediaNodes.map((node) => {
|
|
13230
|
+
const label = node.getAttribute("aria-label") ?? "";
|
|
13231
|
+
const match = label.match(/^(Photo|Video)\s+(\d+)/i);
|
|
13232
|
+
return {
|
|
13233
|
+
kind: match?.[1]?.toLowerCase() === "video" ? "video" : "photo",
|
|
13234
|
+
position: match?.[2] ? Number(match[2]) : null,
|
|
13235
|
+
url: imageUrl(node)
|
|
13236
|
+
};
|
|
13237
|
+
});
|
|
13238
|
+
const first = mediaNodes[0] ?? null;
|
|
13239
|
+
const ancestors = [];
|
|
13240
|
+
let current = first?.parentElement ?? null;
|
|
13241
|
+
while (current) {
|
|
13242
|
+
if (current.scrollHeight > current.clientHeight + 40) ancestors.push(current);
|
|
13243
|
+
current = current.parentElement;
|
|
13244
|
+
}
|
|
13245
|
+
const fallback = Array.from(document.querySelectorAll("div")).filter((node) => {
|
|
13246
|
+
const style = window.getComputedStyle(node);
|
|
13247
|
+
return /(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight + 40 && mediaNodes.some((media) => node.contains(media));
|
|
13248
|
+
});
|
|
13249
|
+
const preferred = Array.from(document.querySelectorAll("div.m6QErb.DxyBCb.kA9KIf.dS8AEf")).find((node) => mediaNodes.some((media) => node.contains(media)) && node.scrollHeight > node.clientHeight + 40);
|
|
13250
|
+
const scroller = preferred ?? [...ancestors, ...fallback].sort((a, b) => b.scrollHeight - a.scrollHeight)[0] ?? null;
|
|
13251
|
+
const atBottom = scroller ? scroller.scrollTop + scroller.clientHeight >= scroller.scrollHeight - 5 : true;
|
|
13252
|
+
const metrics = {
|
|
13253
|
+
scrollHeight: scroller?.scrollHeight ?? 0,
|
|
13254
|
+
scrollTop: scroller?.scrollTop ?? 0,
|
|
13255
|
+
clientHeight: scroller?.clientHeight ?? 0,
|
|
13256
|
+
atBottom
|
|
13257
|
+
};
|
|
13258
|
+
if (scroller && !atBottom) {
|
|
13259
|
+
scroller.scrollTop = Math.min(
|
|
13260
|
+
scroller.scrollHeight - scroller.clientHeight,
|
|
13261
|
+
scroller.scrollTop + Math.max(1200, Math.floor(scroller.clientHeight * 1.75))
|
|
13262
|
+
);
|
|
13263
|
+
}
|
|
13264
|
+
return { items, ...metrics };
|
|
13265
|
+
});
|
|
13266
|
+
}
|
|
13267
|
+
async function collectGalleryPass(page, limit) {
|
|
13268
|
+
const photos = /* @__PURE__ */ new Map();
|
|
13269
|
+
const videos = /* @__PURE__ */ new Set();
|
|
13270
|
+
let stableChecks = 0;
|
|
13271
|
+
let previousHeight = -1;
|
|
13272
|
+
let rounds = 0;
|
|
13273
|
+
while (rounds < MAX_SCROLL_ROUNDS2) {
|
|
13274
|
+
rounds += 1;
|
|
13275
|
+
const before = photos.size + videos.size;
|
|
13276
|
+
const snapshot = await readAndAdvanceGallery(page);
|
|
13277
|
+
for (const item of snapshot.items) {
|
|
13278
|
+
if (item.kind === "video") {
|
|
13279
|
+
videos.add(item.url ? mapsImageIdentity(item.url) : `position:${item.position ?? videos.size}`);
|
|
13280
|
+
continue;
|
|
13281
|
+
}
|
|
13282
|
+
if (!item.url || !isMapsPhotoUrl(item.url)) continue;
|
|
13283
|
+
const key = mapsImageIdentity(item.url);
|
|
13284
|
+
const existing = photos.get(key);
|
|
13285
|
+
if (!existing || (item.position ?? Infinity) < (existing.position ?? Infinity)) photos.set(key, item);
|
|
13286
|
+
}
|
|
13287
|
+
if (photos.size >= limit) {
|
|
13288
|
+
return {
|
|
13289
|
+
items: [...photos.values()].slice(0, limit),
|
|
13290
|
+
exhausted: false,
|
|
13291
|
+
stopReason: "limit_reached",
|
|
13292
|
+
photosDiscovered: photos.size
|
|
13293
|
+
};
|
|
13294
|
+
}
|
|
13295
|
+
await page.waitForTimeout(QUIESCENCE_MS);
|
|
13296
|
+
const unchanged = snapshot.atBottom && before === photos.size + videos.size && previousHeight === snapshot.scrollHeight;
|
|
13297
|
+
stableChecks = unchanged ? stableChecks + 1 : 0;
|
|
13298
|
+
previousHeight = snapshot.scrollHeight;
|
|
13299
|
+
if (stableChecks >= STABLE_CHECKS_REQUIRED2) {
|
|
13300
|
+
return {
|
|
13301
|
+
items: [...photos.values()],
|
|
13302
|
+
exhausted: true,
|
|
13303
|
+
stopReason: "gallery_exhausted",
|
|
13304
|
+
photosDiscovered: photos.size
|
|
13305
|
+
};
|
|
13306
|
+
}
|
|
13307
|
+
}
|
|
13308
|
+
return {
|
|
13309
|
+
items: [...photos.values()],
|
|
13310
|
+
exhausted: false,
|
|
13311
|
+
stopReason: "scroll_round_limit",
|
|
13312
|
+
photosDiscovered: photos.size
|
|
13313
|
+
};
|
|
13314
|
+
}
|
|
13315
|
+
var MapsImageCollector = class {
|
|
13316
|
+
constructor(page) {
|
|
13317
|
+
this.page = page;
|
|
13318
|
+
}
|
|
13319
|
+
page;
|
|
13320
|
+
async collect(scope, maxImages) {
|
|
13321
|
+
if (!await openPhotoGallery(this.page)) {
|
|
13322
|
+
return {
|
|
13323
|
+
images: [],
|
|
13324
|
+
scope,
|
|
13325
|
+
exhausted: false,
|
|
13326
|
+
stopReason: "gallery_unavailable",
|
|
13327
|
+
ownerGalleryAvailable: false,
|
|
13328
|
+
ownerGalleryExhausted: false,
|
|
13329
|
+
ownerPhotosDiscovered: 0,
|
|
13330
|
+
allPhotosDiscovered: 0
|
|
13331
|
+
};
|
|
13332
|
+
}
|
|
13333
|
+
const ownerGalleryAvailable = await selectGalleryTab(this.page, "owner");
|
|
13334
|
+
const ownerPass = ownerGalleryAvailable ? await collectGalleryPass(this.page, scope === "owner" ? maxImages : OWNER_DISCOVERY_LIMIT) : { items: [], exhausted: false, stopReason: "gallery_unavailable", photosDiscovered: 0 };
|
|
13335
|
+
const ownerKeys = new Set(ownerPass.items.flatMap((item) => item.url ? [mapsImageIdentity(item.url)] : []));
|
|
13336
|
+
if (scope === "owner") {
|
|
13337
|
+
return {
|
|
13338
|
+
images: ownerPass.items.flatMap((item, index) => item.url ? [{
|
|
13339
|
+
index: index + 1,
|
|
13340
|
+
galleryPosition: item.position,
|
|
13341
|
+
sourceUrl: item.url,
|
|
13342
|
+
mediaKey: mapsImageIdentity(item.url),
|
|
13343
|
+
origin: "owner",
|
|
13344
|
+
originConfidence: "owner_gallery"
|
|
13345
|
+
}] : []),
|
|
13346
|
+
scope,
|
|
13347
|
+
exhausted: ownerPass.exhausted,
|
|
13348
|
+
stopReason: ownerGalleryAvailable ? ownerPass.stopReason : "gallery_unavailable",
|
|
13349
|
+
ownerGalleryAvailable,
|
|
13350
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13351
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13352
|
+
allPhotosDiscovered: ownerPass.photosDiscovered
|
|
13353
|
+
};
|
|
13354
|
+
}
|
|
13355
|
+
if (!await selectGalleryTab(this.page, "all")) {
|
|
13356
|
+
return {
|
|
13357
|
+
images: [],
|
|
13358
|
+
scope,
|
|
13359
|
+
exhausted: false,
|
|
13360
|
+
stopReason: "gallery_unavailable",
|
|
13361
|
+
ownerGalleryAvailable,
|
|
13362
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13363
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13364
|
+
allPhotosDiscovered: 0
|
|
13365
|
+
};
|
|
13366
|
+
}
|
|
13367
|
+
const allPass = await collectGalleryPass(this.page, maxImages);
|
|
13368
|
+
const images = allPass.items.flatMap((item, index) => {
|
|
13369
|
+
if (!item.url) return [];
|
|
13370
|
+
const mediaKey = mapsImageIdentity(item.url);
|
|
13371
|
+
const isOwner = ownerKeys.has(mediaKey);
|
|
13372
|
+
return [{
|
|
13373
|
+
index: index + 1,
|
|
13374
|
+
galleryPosition: item.position,
|
|
13375
|
+
sourceUrl: item.url,
|
|
13376
|
+
mediaKey,
|
|
13377
|
+
origin: isOwner ? "owner" : ownerPass.exhausted ? "other" : "unknown",
|
|
13378
|
+
originConfidence: isOwner ? "owner_gallery_match" : ownerPass.exhausted ? "not_in_exhausted_owner_gallery" : "owner_gallery_incomplete"
|
|
13379
|
+
}];
|
|
13380
|
+
});
|
|
13381
|
+
return {
|
|
13382
|
+
images,
|
|
13383
|
+
scope,
|
|
13384
|
+
exhausted: allPass.exhausted,
|
|
13385
|
+
stopReason: allPass.stopReason,
|
|
13386
|
+
ownerGalleryAvailable,
|
|
13387
|
+
ownerGalleryExhausted: ownerPass.exhausted,
|
|
13388
|
+
ownerPhotosDiscovered: ownerPass.photosDiscovered,
|
|
13389
|
+
allPhotosDiscovered: allPass.photosDiscovered
|
|
13390
|
+
};
|
|
13391
|
+
}
|
|
13392
|
+
};
|
|
13393
|
+
|
|
12523
13394
|
// src/extractor/MapsExtractor.ts
|
|
12524
13395
|
import { z as z11 } from "zod";
|
|
12525
13396
|
var MapsExtractor = class {
|
|
@@ -12544,6 +13415,48 @@ var MapsExtractor = class {
|
|
|
12544
13415
|
const identifiers = this.parseUrlIdentifiers(placeUrl);
|
|
12545
13416
|
const overview = await this.extractOverview(page);
|
|
12546
13417
|
const hoursTable = await this.extractFullHours(page);
|
|
13418
|
+
let media = {
|
|
13419
|
+
status: "not_requested",
|
|
13420
|
+
scope: options.imageScope,
|
|
13421
|
+
requestedMaxImages: options.maxImages,
|
|
13422
|
+
images: [],
|
|
13423
|
+
imagesCollected: 0,
|
|
13424
|
+
imagesDownloaded: 0,
|
|
13425
|
+
ownerImagesCollected: 0,
|
|
13426
|
+
otherImagesCollected: 0,
|
|
13427
|
+
unknownOriginImagesCollected: 0,
|
|
13428
|
+
ownerGalleryAvailable: false,
|
|
13429
|
+
ownerGalleryExhausted: false,
|
|
13430
|
+
ownerPhotosDiscovered: 0,
|
|
13431
|
+
allPhotosDiscovered: 0,
|
|
13432
|
+
exhausted: false,
|
|
13433
|
+
stopReason: "not_requested",
|
|
13434
|
+
artifact: null,
|
|
13435
|
+
warnings: []
|
|
13436
|
+
};
|
|
13437
|
+
if (options.includeImages) {
|
|
13438
|
+
try {
|
|
13439
|
+
await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
|
|
13440
|
+
await page.waitForTimeout(500);
|
|
13441
|
+
const collected = await new MapsImageCollector(page).collect(options.imageScope, options.maxImages);
|
|
13442
|
+
const ownerImagesCollected = collected.images.filter((image) => image.origin === "owner").length;
|
|
13443
|
+
const otherImagesCollected = collected.images.filter((image) => image.origin === "other").length;
|
|
13444
|
+
const unknownOriginImagesCollected = collected.images.filter((image) => image.origin === "unknown").length;
|
|
13445
|
+
media = {
|
|
13446
|
+
...media,
|
|
13447
|
+
...collected,
|
|
13448
|
+
status: collected.images.length > 0 ? "collected" : collected.stopReason === "gallery_unavailable" ? "unavailable" : "none_exist",
|
|
13449
|
+
imagesCollected: collected.images.length,
|
|
13450
|
+
ownerImagesCollected,
|
|
13451
|
+
otherImagesCollected,
|
|
13452
|
+
unknownOriginImagesCollected
|
|
13453
|
+
};
|
|
13454
|
+
await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 }).catch(() => void 0);
|
|
13455
|
+
} catch (err) {
|
|
13456
|
+
console.warn("[MapsExtractor] image gallery extraction failed:", err instanceof Error ? err.message : String(err));
|
|
13457
|
+
media = { ...media, status: "unavailable", stopReason: "gallery_unavailable", warnings: ["Google Maps image gallery could not be read in this run."] };
|
|
13458
|
+
}
|
|
13459
|
+
}
|
|
12547
13460
|
let reviewStats = { reviewHistogram: [], reviewTopics: [] };
|
|
12548
13461
|
let reviews = [];
|
|
12549
13462
|
let reviewsStatus = "not_requested";
|
|
@@ -12631,6 +13544,7 @@ var MapsExtractor = class {
|
|
|
12631
13544
|
services,
|
|
12632
13545
|
areasServed,
|
|
12633
13546
|
servicesStatus,
|
|
13547
|
+
media,
|
|
12634
13548
|
durationMs: Date.now() - startMs
|
|
12635
13549
|
};
|
|
12636
13550
|
} finally {
|
|
@@ -12913,11 +13827,11 @@ function rootDomain() {
|
|
|
12913
13827
|
const configured = process.env.LOCAL_SOURCEBOOK_PUBLIC_ROOT_DOMAIN?.trim() || DEFAULT_ROOT_DOMAIN;
|
|
12914
13828
|
return configured.replace(/^https?:\/\//i, "").replace(/^www\./i, "").replace(/\/.*$/, "").replace(/\.+$/, "").toLowerCase();
|
|
12915
13829
|
}
|
|
12916
|
-
function localSourcebookPublicUrls(category, state,
|
|
13830
|
+
function localSourcebookPublicUrls(category, state, slug4) {
|
|
12917
13831
|
const categoryUrl = `https://${CATEGORY_PUBLIC_SLUGS[category]}.${rootDomain()}`;
|
|
12918
13832
|
const statePath = `/${state.toLowerCase()}`;
|
|
12919
|
-
const profilePath = `${statePath}/${
|
|
12920
|
-
const reviewsPath = `${statePath}/${
|
|
13833
|
+
const profilePath = `${statePath}/${slug4}/`;
|
|
13834
|
+
const reviewsPath = `${statePath}/${slug4}/reviews`;
|
|
12921
13835
|
return {
|
|
12922
13836
|
categoryUrl,
|
|
12923
13837
|
nationwideDirectoryUrl: `${categoryUrl}/directory`,
|
|
@@ -13465,13 +14379,13 @@ function host(value) {
|
|
|
13465
14379
|
}
|
|
13466
14380
|
}
|
|
13467
14381
|
function tokens(value) {
|
|
13468
|
-
return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((
|
|
14382
|
+
return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token6) => token6.length > 1 && !BUSINESS_SUFFIXES.has(token6)));
|
|
13469
14383
|
}
|
|
13470
14384
|
function nameOverlap(left, right) {
|
|
13471
14385
|
const a = tokens(left);
|
|
13472
14386
|
const b = tokens(right);
|
|
13473
14387
|
if (!a.size || !b.size) return 0;
|
|
13474
|
-
const intersection = [...a].filter((
|
|
14388
|
+
const intersection = [...a].filter((token6) => b.has(token6)).length;
|
|
13475
14389
|
return intersection / Math.max(a.size, b.size);
|
|
13476
14390
|
}
|
|
13477
14391
|
function validateLocalSourcebookMapsIdentity(submission, maps) {
|
|
@@ -13520,6 +14434,10 @@ function createDefaultLocalSourcebookProvider() {
|
|
|
13520
14434
|
includeReviews: true,
|
|
13521
14435
|
maxReviews,
|
|
13522
14436
|
includeServices: true,
|
|
14437
|
+
includeImages: false,
|
|
14438
|
+
imageScope: "all",
|
|
14439
|
+
maxImages: 100,
|
|
14440
|
+
maxInlineImages: 3,
|
|
13523
14441
|
kernelApiKey,
|
|
13524
14442
|
headless: true
|
|
13525
14443
|
});
|
|
@@ -14405,12 +15323,12 @@ function normalizeIdempotencyKey(value) {
|
|
|
14405
15323
|
function tokenFor(id, idempotencyKey3) {
|
|
14406
15324
|
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
|
|
14407
15325
|
}
|
|
14408
|
-
function tokenHash(
|
|
14409
|
-
return createHash4("sha256").update(
|
|
15326
|
+
function tokenHash(token6) {
|
|
15327
|
+
return createHash4("sha256").update(token6).digest("hex");
|
|
14410
15328
|
}
|
|
14411
|
-
function verifyToken(row,
|
|
15329
|
+
function verifyToken(row, token6) {
|
|
14412
15330
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
14413
|
-
const actual = Buffer.from(tokenHash(
|
|
15331
|
+
const actual = Buffer.from(tokenHash(token6), "hex");
|
|
14414
15332
|
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
14415
15333
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
14416
15334
|
}
|
|
@@ -14527,11 +15445,11 @@ async function authorizeScheduledRun(args) {
|
|
|
14527
15445
|
);
|
|
14528
15446
|
}
|
|
14529
15447
|
const id = randomUUID4();
|
|
14530
|
-
const
|
|
15448
|
+
const token6 = tokenFor(id, idempotencyKey3);
|
|
14531
15449
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
14532
15450
|
const inserted = await getDb().execute({
|
|
14533
15451
|
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
14534
|
-
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(
|
|
15452
|
+
args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token6), safeMetadata2(args.metadata), expiresAt]
|
|
14535
15453
|
});
|
|
14536
15454
|
if (inserted.rowsAffected === 0) {
|
|
14537
15455
|
const raced = await getDb().execute({
|
|
@@ -14547,7 +15465,7 @@ async function authorizeScheduledRun(args) {
|
|
|
14547
15465
|
}
|
|
14548
15466
|
return {
|
|
14549
15467
|
id,
|
|
14550
|
-
token:
|
|
15468
|
+
token: token6,
|
|
14551
15469
|
status: "authorized",
|
|
14552
15470
|
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
14553
15471
|
maximumMc,
|
|
@@ -16157,9 +17075,9 @@ var YouTubeExtractor = class {
|
|
|
16157
17075
|
}
|
|
16158
17076
|
async fetchContinuation(page, endpoint, contToken, context) {
|
|
16159
17077
|
const result = await page.evaluate(
|
|
16160
|
-
async ({ ep, token:
|
|
17078
|
+
async ({ ep, token: token6, ctx }) => {
|
|
16161
17079
|
const body = {
|
|
16162
|
-
continuation:
|
|
17080
|
+
continuation: token6,
|
|
16163
17081
|
context: {
|
|
16164
17082
|
client: {
|
|
16165
17083
|
clientName: ctx.clientName,
|
|
@@ -16403,23 +17321,23 @@ async function extractOnce(options) {
|
|
|
16403
17321
|
}
|
|
16404
17322
|
async function writeOutputs(result, outputDir) {
|
|
16405
17323
|
await fs.mkdir(outputDir, { recursive: true });
|
|
16406
|
-
const
|
|
17324
|
+
const slug4 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16407
17325
|
const ts = Date.now();
|
|
16408
17326
|
await fs.writeFile(
|
|
16409
|
-
path4.join(outputDir, `${
|
|
17327
|
+
path4.join(outputDir, `${slug4}-${ts}.json`),
|
|
16410
17328
|
JSON.stringify(result, null, 2),
|
|
16411
17329
|
"utf8"
|
|
16412
17330
|
);
|
|
16413
17331
|
if (result.videos.length > 0) {
|
|
16414
17332
|
await fs.writeFile(
|
|
16415
|
-
path4.join(outputDir, `${
|
|
17333
|
+
path4.join(outputDir, `${slug4}-videos-${ts}.csv`),
|
|
16416
17334
|
Papa.unparse(result.videos, { header: true }),
|
|
16417
17335
|
"utf8"
|
|
16418
17336
|
);
|
|
16419
17337
|
}
|
|
16420
17338
|
if (result.downloads.length > 0) {
|
|
16421
17339
|
await fs.writeFile(
|
|
16422
|
-
path4.join(outputDir, `${
|
|
17340
|
+
path4.join(outputDir, `${slug4}-downloads-${ts}.csv`),
|
|
16423
17341
|
Papa.unparse(result.downloads, { header: true }),
|
|
16424
17342
|
"utf8"
|
|
16425
17343
|
);
|
|
@@ -16783,6 +17701,8 @@ var ExtractUrlBodyShape = {
|
|
|
16783
17701
|
includeFeaturedImage: z16.boolean().optional(),
|
|
16784
17702
|
downloadMedia: z16.boolean().optional(),
|
|
16785
17703
|
mediaTypes: z16.array(z16.enum(["image", "video", "audio"])).optional(),
|
|
17704
|
+
maxMediaAssets: z16.number().int().min(1).max(250).optional(),
|
|
17705
|
+
maxInlineImages: z16.number().int().min(0).max(5).optional(),
|
|
16786
17706
|
delivery: z16.enum(["auto", "inline", "artifact", "memory"]).optional(),
|
|
16787
17707
|
preserveMedia: z16.boolean().optional(),
|
|
16788
17708
|
depositToVault: z16.boolean().optional(),
|
|
@@ -18059,7 +18979,7 @@ function advertiserTokenOverlap(query, candidate) {
|
|
|
18059
18979
|
const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
|
|
18060
18980
|
if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
|
|
18061
18981
|
let overlap = 0;
|
|
18062
|
-
for (const
|
|
18982
|
+
for (const token6 of queryTokens) if (candidateTokens.has(token6)) overlap++;
|
|
18063
18983
|
return overlap / queryTokens.size;
|
|
18064
18984
|
}
|
|
18065
18985
|
function assessPageIntelMatch(body, result) {
|
|
@@ -19294,8 +20214,8 @@ function getBlobStore() {
|
|
|
19294
20214
|
return cached;
|
|
19295
20215
|
}
|
|
19296
20216
|
var VercelBlobStore = class {
|
|
19297
|
-
constructor(
|
|
19298
|
-
this.token =
|
|
20217
|
+
constructor(token6) {
|
|
20218
|
+
this.token = token6;
|
|
19299
20219
|
}
|
|
19300
20220
|
token;
|
|
19301
20221
|
kind = "vercel-blob";
|
|
@@ -19751,8 +20671,8 @@ function safeFilePart(input) {
|
|
|
19751
20671
|
}
|
|
19752
20672
|
function blobKeyPrefix(shortcode, sourceUrl) {
|
|
19753
20673
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
19754
|
-
const
|
|
19755
|
-
return `instagram/${stamp}-${
|
|
20674
|
+
const slug4 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
20675
|
+
return `instagram/${stamp}-${slug4}`;
|
|
19756
20676
|
}
|
|
19757
20677
|
async function fetchMediaBytes(url, referer, runtimeUserAgent) {
|
|
19758
20678
|
const res = await fetch(url, {
|
|
@@ -20664,6 +21584,212 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
|
|
|
20664
21584
|
|
|
20665
21585
|
// src/api/maps-routes.ts
|
|
20666
21586
|
import { Hono as Hono13 } from "hono";
|
|
21587
|
+
|
|
21588
|
+
// src/api/maps-media-artifacts.ts
|
|
21589
|
+
import { homedir as homedir2 } from "os";
|
|
21590
|
+
import { join as join6 } from "path";
|
|
21591
|
+
import { randomBytes as randomBytes2 } from "crypto";
|
|
21592
|
+
import { ZipFile } from "yazl";
|
|
21593
|
+
import pLimit4 from "p-limit";
|
|
21594
|
+
var MAPS_MEDIA_ARTIFACT_PREFIX = "maps-media/";
|
|
21595
|
+
var MAPS_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
21596
|
+
var MAPS_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
21597
|
+
var MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
|
21598
|
+
var MAX_ARCHIVE_IMAGE_BYTES = 40 * 1024 * 1024;
|
|
21599
|
+
var MAX_INLINE_IMAGE_BYTES = 15e5;
|
|
21600
|
+
var MAX_INLINE_TOTAL_BYTES = 25e5;
|
|
21601
|
+
var DOWNLOAD_CONCURRENCY = 6;
|
|
21602
|
+
var MAX_REDIRECTS = 4;
|
|
21603
|
+
function token() {
|
|
21604
|
+
return process.env.MAPS_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
21605
|
+
}
|
|
21606
|
+
function policy() {
|
|
21607
|
+
return {
|
|
21608
|
+
prefix: MAPS_MEDIA_ARTIFACT_PREFIX,
|
|
21609
|
+
artifactTtlMs: MAPS_MEDIA_ARTIFACT_TTL_MS,
|
|
21610
|
+
downloadTtlMs: MAPS_MEDIA_DOWNLOAD_TTL_MS,
|
|
21611
|
+
token: token()
|
|
21612
|
+
};
|
|
21613
|
+
}
|
|
21614
|
+
function allowedMapsImageUrl(raw) {
|
|
21615
|
+
try {
|
|
21616
|
+
const url = new URL(raw);
|
|
21617
|
+
if (url.protocol !== "https:") return null;
|
|
21618
|
+
const host2 = url.hostname.toLowerCase().replace(/\.$/, "");
|
|
21619
|
+
if (host2 !== "googleusercontent.com" && !host2.endsWith(".googleusercontent.com") && host2 !== "streetviewpixels-pa.googleapis.com") return null;
|
|
21620
|
+
return url;
|
|
21621
|
+
} catch {
|
|
21622
|
+
return null;
|
|
21623
|
+
}
|
|
21624
|
+
}
|
|
21625
|
+
function preferredMapsImageRendition(rawUrl) {
|
|
21626
|
+
const url = allowedMapsImageUrl(rawUrl);
|
|
21627
|
+
if (!url) return rawUrl;
|
|
21628
|
+
if (url.hostname.toLowerCase().endsWith("googleusercontent.com") && /=[^/]+$/.test(url.pathname)) {
|
|
21629
|
+
url.pathname = url.pathname.replace(/=[^/]+$/, "=s2048");
|
|
21630
|
+
}
|
|
21631
|
+
return url.href;
|
|
21632
|
+
}
|
|
21633
|
+
function imageMime(bytes) {
|
|
21634
|
+
if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
|
|
21635
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
|
|
21636
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
|
|
21637
|
+
if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
|
|
21638
|
+
return null;
|
|
21639
|
+
}
|
|
21640
|
+
async function readBounded(response) {
|
|
21641
|
+
const declared = Number(response.headers.get("content-length") ?? "");
|
|
21642
|
+
if (Number.isFinite(declared) && declared > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21643
|
+
if (!response.body) {
|
|
21644
|
+
const bytes2 = new Uint8Array(await response.arrayBuffer());
|
|
21645
|
+
if (bytes2.byteLength > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21646
|
+
return bytes2;
|
|
21647
|
+
}
|
|
21648
|
+
const reader = response.body.getReader();
|
|
21649
|
+
const chunks = [];
|
|
21650
|
+
let total = 0;
|
|
21651
|
+
try {
|
|
21652
|
+
while (true) {
|
|
21653
|
+
const { done, value } = await reader.read();
|
|
21654
|
+
if (done) break;
|
|
21655
|
+
if (!value) continue;
|
|
21656
|
+
total += value.byteLength;
|
|
21657
|
+
if (total > MAX_IMAGE_BYTES) throw new Error("image_too_large");
|
|
21658
|
+
chunks.push(value);
|
|
21659
|
+
}
|
|
21660
|
+
} finally {
|
|
21661
|
+
reader.releaseLock();
|
|
21662
|
+
}
|
|
21663
|
+
const bytes = new Uint8Array(total);
|
|
21664
|
+
let offset = 0;
|
|
21665
|
+
for (const chunk of chunks) {
|
|
21666
|
+
bytes.set(chunk, offset);
|
|
21667
|
+
offset += chunk.byteLength;
|
|
21668
|
+
}
|
|
21669
|
+
return bytes;
|
|
21670
|
+
}
|
|
21671
|
+
async function fetchMapsImage(rawUrl) {
|
|
21672
|
+
let url = allowedMapsImageUrl(preferredMapsImageRendition(rawUrl));
|
|
21673
|
+
if (!url) throw new Error("image_host_not_allowed");
|
|
21674
|
+
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
|
|
21675
|
+
const checked = await validatePublicHttpUrl(url.href, { field: "Maps image URL" });
|
|
21676
|
+
if (checked.error || !checked.parsed) throw new Error("image_url_rejected");
|
|
21677
|
+
const response = await fetch(checked.parsed.href, {
|
|
21678
|
+
redirect: "manual",
|
|
21679
|
+
headers: { Accept: "image/jpeg,image/png,image/webp,image/gif;q=0.8" },
|
|
21680
|
+
signal: AbortSignal.timeout(15e3)
|
|
21681
|
+
});
|
|
21682
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
21683
|
+
const location2 = response.headers.get("location");
|
|
21684
|
+
if (!location2 || redirect === MAX_REDIRECTS) throw new Error("image_redirect_rejected");
|
|
21685
|
+
url = allowedMapsImageUrl(new URL(location2, url).href);
|
|
21686
|
+
if (!url) throw new Error("image_redirect_host_not_allowed");
|
|
21687
|
+
continue;
|
|
21688
|
+
}
|
|
21689
|
+
if (!response.ok) throw new Error(`image_fetch_${response.status}`);
|
|
21690
|
+
const body = await readBounded(response);
|
|
21691
|
+
const detected = imageMime(body);
|
|
21692
|
+
if (!detected) throw new Error("image_content_invalid");
|
|
21693
|
+
return { bytes: Buffer.from(body), ...detected, finalUrl: url.href };
|
|
21694
|
+
}
|
|
21695
|
+
throw new Error("image_redirect_rejected");
|
|
21696
|
+
}
|
|
21697
|
+
function zipBuffer(files) {
|
|
21698
|
+
return new Promise((resolve2, reject) => {
|
|
21699
|
+
const zip = new ZipFile();
|
|
21700
|
+
const chunks = [];
|
|
21701
|
+
zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
|
21702
|
+
zip.outputStream.once("error", reject);
|
|
21703
|
+
zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
|
|
21704
|
+
for (const file of files) zip.addBuffer(file.content, file.path);
|
|
21705
|
+
zip.end();
|
|
21706
|
+
});
|
|
21707
|
+
}
|
|
21708
|
+
function slug(value) {
|
|
21709
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "google-maps-place";
|
|
21710
|
+
}
|
|
21711
|
+
function mapsMediaArtifactOwnerId(artifactId) {
|
|
21712
|
+
return privateArtifactOwnerId(artifactId, MAPS_MEDIA_ARTIFACT_PREFIX);
|
|
21713
|
+
}
|
|
21714
|
+
async function readOwnedMapsMediaArtifactBuffer(args) {
|
|
21715
|
+
if (mapsMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
21716
|
+
return readPrivateArtifactBuffer({ policy: policy(), artifactId: args.artifactId, maxBytes: 50 * 1024 * 1024 });
|
|
21717
|
+
}
|
|
21718
|
+
async function packageMapsMedia(args) {
|
|
21719
|
+
const limit = pLimit4(DOWNLOAD_CONCURRENCY);
|
|
21720
|
+
const files = [];
|
|
21721
|
+
let storedBytes = 0;
|
|
21722
|
+
let inlineBytes = 0;
|
|
21723
|
+
let inlineCount = 0;
|
|
21724
|
+
await Promise.all(args.media.images.map((image) => limit(async () => {
|
|
21725
|
+
try {
|
|
21726
|
+
const fetched = await fetchMapsImage(image.sourceUrl);
|
|
21727
|
+
if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_IMAGE_BYTES) throw new Error("archive_byte_limit_reached");
|
|
21728
|
+
storedBytes += fetched.bytes.length;
|
|
21729
|
+
const filename = `images/${String(image.index).padStart(4, "0")}-${image.origin}.${fetched.extension}`;
|
|
21730
|
+
files.push({ path: filename, content: fetched.bytes });
|
|
21731
|
+
image.sourceUrl = fetched.finalUrl;
|
|
21732
|
+
image.filename = filename;
|
|
21733
|
+
image.mimeType = fetched.mimeType;
|
|
21734
|
+
image.bytes = fetched.bytes.length;
|
|
21735
|
+
image.downloadStatus = "downloaded";
|
|
21736
|
+
image.downloadError = null;
|
|
21737
|
+
if (inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES) {
|
|
21738
|
+
image.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
|
|
21739
|
+
inlineBytes += fetched.bytes.length;
|
|
21740
|
+
inlineCount += 1;
|
|
21741
|
+
}
|
|
21742
|
+
} catch (error) {
|
|
21743
|
+
image.filename = null;
|
|
21744
|
+
image.mimeType = null;
|
|
21745
|
+
image.bytes = null;
|
|
21746
|
+
image.downloadStatus = "failed";
|
|
21747
|
+
image.downloadError = error instanceof Error ? error.message : "image_download_failed";
|
|
21748
|
+
image.inlinePreview = null;
|
|
21749
|
+
}
|
|
21750
|
+
})));
|
|
21751
|
+
args.media.imagesDownloaded = args.media.images.filter((image) => image.downloadStatus === "downloaded").length;
|
|
21752
|
+
const failed = args.media.images.length - args.media.imagesDownloaded;
|
|
21753
|
+
if (failed > 0) args.media.warnings.push(`${failed} image(s) could not be downloaded; their manifest rows include a bounded error code.`);
|
|
21754
|
+
if (args.media.imagesDownloaded === 0) return { media: args.media, artifact: null };
|
|
21755
|
+
const cleanImages = args.media.images.map(({ inlinePreview: _preview, ...image }) => image);
|
|
21756
|
+
const summary = {
|
|
21757
|
+
businessName: args.businessName,
|
|
21758
|
+
placeUrl: args.placeUrl,
|
|
21759
|
+
extractedAt: args.extractedAt,
|
|
21760
|
+
scope: args.media.scope,
|
|
21761
|
+
imagesCollected: args.media.imagesCollected,
|
|
21762
|
+
imagesDownloaded: args.media.imagesDownloaded,
|
|
21763
|
+
ownerImagesCollected: args.media.ownerImagesCollected,
|
|
21764
|
+
otherImagesCollected: args.media.otherImagesCollected,
|
|
21765
|
+
unknownOriginImagesCollected: args.media.unknownOriginImagesCollected,
|
|
21766
|
+
ownerGalleryExhausted: args.media.ownerGalleryExhausted,
|
|
21767
|
+
galleryExhausted: args.media.exhausted,
|
|
21768
|
+
stopReason: args.media.stopReason,
|
|
21769
|
+
notes: [
|
|
21770
|
+
"origin=owner is backed by the Google Maps By owner gallery or an exact media-key match to it.",
|
|
21771
|
+
"origin=other means absent from an exhausted owner gallery; it can include customer, review, Street View, Google, or unattributed media.",
|
|
21772
|
+
"Google-hosted photo files use a bounded s2048 rendition when that CDN form is available; each file is still subject to byte and signature validation."
|
|
21773
|
+
]
|
|
21774
|
+
};
|
|
21775
|
+
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
21776
|
+
files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
|
|
21777
|
+
const archive = await zipBuffer(files);
|
|
21778
|
+
const id = randomBytes2(6).toString("hex");
|
|
21779
|
+
const pointer = await createPrivateArtifact({
|
|
21780
|
+
policy: policy(),
|
|
21781
|
+
ownerId: args.ownerId,
|
|
21782
|
+
artifactKey: `${id}.zip`,
|
|
21783
|
+
createdAt: new Date(args.extractedAt),
|
|
21784
|
+
filename: `${slug(args.businessName)}-maps-images.zip`,
|
|
21785
|
+
contentType: "application/zip",
|
|
21786
|
+
content: archive
|
|
21787
|
+
});
|
|
21788
|
+
const localPath = token() ? null : join6(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
21789
|
+
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
21790
|
+
}
|
|
21791
|
+
|
|
21792
|
+
// src/api/maps-routes.ts
|
|
20667
21793
|
function mapsErrorResponse(c, err, errorCode) {
|
|
20668
21794
|
const msg = err instanceof Error ? err.message : String(err);
|
|
20669
21795
|
const rotationError = err instanceof MapsSearchRotationError ? err : null;
|
|
@@ -20782,6 +21908,30 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
20782
21908
|
result.reviews = [];
|
|
20783
21909
|
}
|
|
20784
21910
|
}
|
|
21911
|
+
if (parsed.data.includeImages && result.media?.images.length > 0) {
|
|
21912
|
+
try {
|
|
21913
|
+
const packaged = await packageMapsMedia({
|
|
21914
|
+
ownerId: String(user.id),
|
|
21915
|
+
businessName: result.name ?? parsed.data.businessName,
|
|
21916
|
+
placeUrl: result.placeUrl,
|
|
21917
|
+
extractedAt: result.extractedAt,
|
|
21918
|
+
media: result.media,
|
|
21919
|
+
maxInlineImages: parsed.data.maxInlineImages
|
|
21920
|
+
});
|
|
21921
|
+
result.media = packaged.media;
|
|
21922
|
+
result.media.artifact = packaged.artifact;
|
|
21923
|
+
} catch (error) {
|
|
21924
|
+
console.warn("[maps/place] image artifact packaging failed:", error instanceof Error ? error.message : String(error));
|
|
21925
|
+
result.media.warnings.push("The image manifest was collected, but the downloadable ZIP artifact could not be created.");
|
|
21926
|
+
}
|
|
21927
|
+
}
|
|
21928
|
+
const logResult = result.media ? {
|
|
21929
|
+
...result,
|
|
21930
|
+
media: {
|
|
21931
|
+
...result.media,
|
|
21932
|
+
images: result.media.images.map(({ inlinePreview: _preview, ...image }) => image)
|
|
21933
|
+
}
|
|
21934
|
+
} : result;
|
|
20785
21935
|
await logRequestEvent({
|
|
20786
21936
|
userId: user.id,
|
|
20787
21937
|
source: "maps_place",
|
|
@@ -20789,7 +21939,7 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
20789
21939
|
query: `${parsed.data.businessName} ${parsed.data.location}`,
|
|
20790
21940
|
location: parsed.data.location,
|
|
20791
21941
|
resultCount: reviewCount,
|
|
20792
|
-
result
|
|
21942
|
+
result: logResult
|
|
20793
21943
|
});
|
|
20794
21944
|
return c.json(result);
|
|
20795
21945
|
} catch (err) {
|
|
@@ -21729,7 +22879,7 @@ function getLocalSourcebookContract(category) {
|
|
|
21729
22879
|
}
|
|
21730
22880
|
async function prepareLocalSourcebookWrite(input) {
|
|
21731
22881
|
const parsed = DirectoryIdentitySchema.parse(input);
|
|
21732
|
-
const
|
|
22882
|
+
const slug4 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
|
|
21733
22883
|
const categoryTag = CATEGORY_TAG[parsed.category];
|
|
21734
22884
|
const candidates = [
|
|
21735
22885
|
{ tag: categoryTag, central: true, reusable: true },
|
|
@@ -21737,12 +22887,12 @@ async function prepareLocalSourcebookWrite(input) {
|
|
|
21737
22887
|
...parsed.tags.filter((tag) => tag !== categoryTag).map((tag) => ({ tag }))
|
|
21738
22888
|
];
|
|
21739
22889
|
const tagResolutions = await resolveLocalSourcebookTags(candidates);
|
|
21740
|
-
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state,
|
|
22890
|
+
const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug4);
|
|
21741
22891
|
const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
|
|
21742
22892
|
return {
|
|
21743
22893
|
ok: true,
|
|
21744
|
-
route: { category: parsed.category, state: parsed.state, slug:
|
|
21745
|
-
proposedWrite: { ...parsed, schemaOrgType, slug:
|
|
22894
|
+
route: { category: parsed.category, state: parsed.state, slug: slug4, ...publicUrls },
|
|
22895
|
+
proposedWrite: { ...parsed, schemaOrgType, slug: slug4, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
|
|
21746
22896
|
contract: getLocalSourcebookContract(parsed.category),
|
|
21747
22897
|
tagResolutions,
|
|
21748
22898
|
instructions: [
|
|
@@ -21807,7 +22957,7 @@ async function persistLocalSourcebookTags(submissionId, userId, tags, decisions
|
|
|
21807
22957
|
}
|
|
21808
22958
|
|
|
21809
22959
|
// src/api/local-sourcebook-routes.ts
|
|
21810
|
-
var
|
|
22960
|
+
var slug2 = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
|
|
21811
22961
|
var SubmitSchema = z26.object({
|
|
21812
22962
|
category: z26.enum(LOCAL_SOURCEBOOK_CATEGORIES),
|
|
21813
22963
|
state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
@@ -21878,7 +23028,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
|
|
|
21878
23028
|
if (!idempotencyKey3) {
|
|
21879
23029
|
return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
|
|
21880
23030
|
}
|
|
21881
|
-
const listingSlug = identity.slug ??
|
|
23031
|
+
const listingSlug = identity.slug ?? slug2(identity.businessName);
|
|
21882
23032
|
const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
|
|
21883
23033
|
const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
|
|
21884
23034
|
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
@@ -21929,7 +23079,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
|
|
|
21929
23079
|
if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
|
|
21930
23080
|
const input = parsed.data;
|
|
21931
23081
|
if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
|
|
21932
|
-
const listingSlug = input.slug ??
|
|
23082
|
+
const listingSlug = input.slug ?? slug2(input.businessName);
|
|
21933
23083
|
const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
|
|
21934
23084
|
const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
|
|
21935
23085
|
let billing = null;
|
|
@@ -22986,7 +24136,7 @@ import { createHash as createHash8, randomUUID as randomUUID9 } from "crypto";
|
|
|
22986
24136
|
|
|
22987
24137
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
22988
24138
|
import { createHash as createHash7 } from "crypto";
|
|
22989
|
-
import
|
|
24139
|
+
import pLimit5 from "p-limit";
|
|
22990
24140
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
22991
24141
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
22992
24142
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
@@ -23267,7 +24417,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
23267
24417
|
async function capturePageSnapshots(targets, options = {}) {
|
|
23268
24418
|
const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
|
|
23269
24419
|
const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
|
|
23270
|
-
const limit =
|
|
24420
|
+
const limit = pLimit5(maxConcurrency);
|
|
23271
24421
|
const pageSnapshotArtifacts = await Promise.all(
|
|
23272
24422
|
targets.map((target) => limit(() => capturePageSnapshot(target, { ...options, timeoutMs })))
|
|
23273
24423
|
);
|
|
@@ -24010,7 +25160,50 @@ function folderInventory(result) {
|
|
|
24010
25160
|
}
|
|
24011
25161
|
function enrichMemoryResult(toolName, result) {
|
|
24012
25162
|
if (toolName !== "listTool" || result.ok !== true) return result;
|
|
24013
|
-
|
|
25163
|
+
if (result.scope === "account") return result;
|
|
25164
|
+
return { ...result, scope: "vault", folders: folderInventory(result) };
|
|
25165
|
+
}
|
|
25166
|
+
async function buildAccountMemoryInventory(args, listOne) {
|
|
25167
|
+
const filters = { ...args };
|
|
25168
|
+
delete filters.allVaults;
|
|
25169
|
+
delete filters.vault;
|
|
25170
|
+
const initial = enrichMemoryResult("listTool", await listOne(filters));
|
|
25171
|
+
if (initial.ok !== true) return initial;
|
|
25172
|
+
const vaults = Array.isArray(initial.vaults) ? initial.vaults.filter((vault) => typeof vault === "string") : [];
|
|
25173
|
+
const initialVault = typeof initial.vault === "string" ? initial.vault : null;
|
|
25174
|
+
if (!vaults.length && initialVault) vaults.push(initialVault);
|
|
25175
|
+
const vaultInventories = [];
|
|
25176
|
+
for (const vault of vaults) {
|
|
25177
|
+
const listed = vault === initialVault ? initial : enrichMemoryResult("listTool", await listOne({ ...filters, vault }));
|
|
25178
|
+
if (listed.ok !== true) {
|
|
25179
|
+
return {
|
|
25180
|
+
ok: false,
|
|
25181
|
+
scope: "account",
|
|
25182
|
+
partial: true,
|
|
25183
|
+
vaults,
|
|
25184
|
+
vaultInventories,
|
|
25185
|
+
error: typeof listed.error === "string" ? `Could not list vault ${vault}: ${listed.error}` : `Could not list vault ${vault}`
|
|
25186
|
+
};
|
|
25187
|
+
}
|
|
25188
|
+
const notes = Array.isArray(listed.notes) ? listed.notes : [];
|
|
25189
|
+
const folders = Array.isArray(listed.folders) ? listed.folders : [];
|
|
25190
|
+
vaultInventories.push({
|
|
25191
|
+
vault,
|
|
25192
|
+
noteCount: notes.length,
|
|
25193
|
+
folderCount: folders.length,
|
|
25194
|
+
notes,
|
|
25195
|
+
folders
|
|
25196
|
+
});
|
|
25197
|
+
}
|
|
25198
|
+
return {
|
|
25199
|
+
ok: true,
|
|
25200
|
+
scope: "account",
|
|
25201
|
+
partial: false,
|
|
25202
|
+
vaults,
|
|
25203
|
+
totalNotes: vaultInventories.reduce((sum, inventory) => sum + Number(inventory.noteCount), 0),
|
|
25204
|
+
totalFolders: vaultInventories.reduce((sum, inventory) => sum + Number(inventory.folderCount), 0),
|
|
25205
|
+
vaultInventories
|
|
25206
|
+
};
|
|
24014
25207
|
}
|
|
24015
25208
|
var MemoryMcpToolExecutor = class {
|
|
24016
25209
|
baseUrl;
|
|
@@ -24021,22 +25214,25 @@ var MemoryMcpToolExecutor = class {
|
|
|
24021
25214
|
}
|
|
24022
25215
|
async callMemoryTool(toolName, args) {
|
|
24023
25216
|
try {
|
|
24024
|
-
const
|
|
24025
|
-
|
|
24026
|
-
|
|
24027
|
-
|
|
24028
|
-
|
|
24029
|
-
|
|
24030
|
-
|
|
24031
|
-
|
|
24032
|
-
|
|
24033
|
-
|
|
24034
|
-
|
|
24035
|
-
|
|
24036
|
-
|
|
25217
|
+
const callOne = async (input) => {
|
|
25218
|
+
const res = await fetch(`${this.baseUrl}/memory/mcp-call`, {
|
|
25219
|
+
method: "POST",
|
|
25220
|
+
headers: {
|
|
25221
|
+
"content-type": "application/json",
|
|
25222
|
+
"x-api-key": this.apiKey
|
|
25223
|
+
},
|
|
25224
|
+
body: JSON.stringify({ toolName, args: input })
|
|
25225
|
+
});
|
|
25226
|
+
const data2 = await res.json().catch(() => null);
|
|
25227
|
+
if (!res.ok) {
|
|
25228
|
+
throw new Error(data2?.error ?? `memory ${toolName} failed (HTTP ${res.status})`);
|
|
25229
|
+
}
|
|
25230
|
+
return data2 ?? { ok: false, error: `memory ${toolName} returned no result` };
|
|
25231
|
+
};
|
|
25232
|
+
const data = toolName === "listTool" && args.allVaults === true ? await buildAccountMemoryInventory(args, callOne) : await callOne(args);
|
|
24037
25233
|
const result = enrichMemoryResult(
|
|
24038
25234
|
toolName,
|
|
24039
|
-
data
|
|
25235
|
+
data
|
|
24040
25236
|
);
|
|
24041
25237
|
return {
|
|
24042
25238
|
content: [{ type: "text", text: JSON.stringify(result) }],
|
|
@@ -24062,15 +25258,15 @@ var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
|
|
|
24062
25258
|
["image/webp", "webp"],
|
|
24063
25259
|
["image/gif", "gif"]
|
|
24064
25260
|
]);
|
|
24065
|
-
function
|
|
25261
|
+
function token2() {
|
|
24066
25262
|
return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
24067
25263
|
}
|
|
24068
|
-
function
|
|
25264
|
+
function policy2() {
|
|
24069
25265
|
return {
|
|
24070
25266
|
prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
|
|
24071
25267
|
artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
|
|
24072
25268
|
downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
|
|
24073
|
-
token:
|
|
25269
|
+
token: token2()
|
|
24074
25270
|
};
|
|
24075
25271
|
}
|
|
24076
25272
|
function imageSourceArtifactOwnerId(artifactId) {
|
|
@@ -24084,7 +25280,7 @@ async function createImageSourceArtifact(args) {
|
|
|
24084
25280
|
}
|
|
24085
25281
|
const id = randomUUID10().replaceAll("-", "");
|
|
24086
25282
|
return createPrivateArtifact({
|
|
24087
|
-
policy:
|
|
25283
|
+
policy: policy2(),
|
|
24088
25284
|
ownerId: args.ownerId,
|
|
24089
25285
|
artifactKey: `${id}.${extension}`,
|
|
24090
25286
|
createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
|
|
@@ -24096,7 +25292,7 @@ async function createImageSourceArtifact(args) {
|
|
|
24096
25292
|
async function readOwnedImageSourceArtifact(args) {
|
|
24097
25293
|
if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
24098
25294
|
return readPrivateArtifactBuffer({
|
|
24099
|
-
policy:
|
|
25295
|
+
policy: policy2(),
|
|
24100
25296
|
artifactId: args.artifactId,
|
|
24101
25297
|
maxBytes: IMAGE_SOURCE_MAX_BYTES
|
|
24102
25298
|
});
|
|
@@ -24253,10 +25449,8 @@ function registerReadCutovers() {
|
|
|
24253
25449
|
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
24254
25450
|
});
|
|
24255
25451
|
registerCutover("listTool", async (input, apiKey) => {
|
|
24256
|
-
const
|
|
24257
|
-
|
|
24258
|
-
await listTool.execute({ ...input, apiKey }, {}) ?? {}
|
|
24259
|
-
);
|
|
25452
|
+
const listOne = async (args) => await listTool.execute({ ...args, apiKey }, {}) ?? {};
|
|
25453
|
+
const result = input.allVaults === true ? await buildAccountMemoryInventory(input, listOne) : enrichMemoryResult("listTool", await listOne(input));
|
|
24260
25454
|
return { content: [{ type: "text", text: JSON.stringify(result) }], structuredContent: result, isError: false };
|
|
24261
25455
|
});
|
|
24262
25456
|
registerCutover("searchTool", async (input, apiKey) => {
|
|
@@ -24684,8 +25878,8 @@ function protectedResourceMetadata() {
|
|
|
24684
25878
|
function wwwAuthenticateValue() {
|
|
24685
25879
|
return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
|
|
24686
25880
|
}
|
|
24687
|
-
function isBearerJwt(
|
|
24688
|
-
return !
|
|
25881
|
+
function isBearerJwt(token6) {
|
|
25882
|
+
return !token6.startsWith("sk_") && token6.split(".").length === 3;
|
|
24689
25883
|
}
|
|
24690
25884
|
var jwksSingleton = null;
|
|
24691
25885
|
function getJwks() {
|
|
@@ -24693,8 +25887,8 @@ function getJwks() {
|
|
|
24693
25887
|
jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
|
|
24694
25888
|
return jwksSingleton;
|
|
24695
25889
|
}
|
|
24696
|
-
async function verifyScraperOAuthJwt(
|
|
24697
|
-
const { payload } = await jwtVerify(
|
|
25890
|
+
async function verifyScraperOAuthJwt(token6) {
|
|
25891
|
+
const { payload } = await jwtVerify(token6, getJwks(), {
|
|
24698
25892
|
issuer: oauthIssuer(),
|
|
24699
25893
|
audience: oauthResourceUrl()
|
|
24700
25894
|
});
|
|
@@ -27417,7 +28611,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
27417
28611
|
// src/api/oauth-routes.ts
|
|
27418
28612
|
import { Hono as Hono24 } from "hono";
|
|
27419
28613
|
import { getCookie, setCookie } from "hono/cookie";
|
|
27420
|
-
import { createHash as createHash9, randomBytes as
|
|
28614
|
+
import { createHash as createHash9, randomBytes as randomBytes3, randomUUID as randomUUID13 } from "crypto";
|
|
27421
28615
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
27422
28616
|
var ISSUER = "https://mcpscraper.dev";
|
|
27423
28617
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -27516,9 +28710,9 @@ function negotiateScope(requested, user, resource) {
|
|
|
27516
28710
|
if (operator && !unique3.includes("memory.admin")) unique3.push("memory.admin");
|
|
27517
28711
|
return unique3.join(" ");
|
|
27518
28712
|
}
|
|
27519
|
-
async function sessionUser(
|
|
27520
|
-
if (!
|
|
27521
|
-
const userId = verifySession(
|
|
28713
|
+
async function sessionUser(token6) {
|
|
28714
|
+
if (!token6) return null;
|
|
28715
|
+
const userId = verifySession(token6);
|
|
27522
28716
|
if (!userId) return null;
|
|
27523
28717
|
const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
|
|
27524
28718
|
return user ?? null;
|
|
@@ -27703,7 +28897,7 @@ oauthApp.post("/oauth/register", async (c) => {
|
|
|
27703
28897
|
}
|
|
27704
28898
|
}
|
|
27705
28899
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
27706
|
-
const clientId = `client_${
|
|
28900
|
+
const clientId = `client_${randomBytes3(16).toString("hex")}`;
|
|
27707
28901
|
await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
|
|
27708
28902
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
27709
28903
|
return c.json({
|
|
@@ -27778,7 +28972,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
27778
28972
|
if (action2 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
27779
28973
|
if (action2 !== "approve") return c.text("unsupported action", 400);
|
|
27780
28974
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
27781
|
-
const code = `code_${
|
|
28975
|
+
const code = `code_${randomBytes3(32).toString("base64url")}`;
|
|
27782
28976
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
27783
28977
|
await withOAuthDatabaseDeadline("put-authorization-code", putCode({
|
|
27784
28978
|
code,
|
|
@@ -27829,7 +29023,7 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27829
29023
|
const plan = user ? resolvePlan(user) : "free";
|
|
27830
29024
|
const audience = record.resource ?? RESOURCE();
|
|
27831
29025
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
27832
|
-
const refreshToken = `rt_${
|
|
29026
|
+
const refreshToken = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
27833
29027
|
await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
|
|
27834
29028
|
refresh_token: refreshToken,
|
|
27835
29029
|
client_id: clientId,
|
|
@@ -27859,7 +29053,7 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27859
29053
|
const plan = user ? resolvePlan(user) : "free";
|
|
27860
29054
|
const audience = record.resource ?? RESOURCE();
|
|
27861
29055
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
27862
|
-
const nextRefresh = `rt_${
|
|
29056
|
+
const nextRefresh = `rt_${randomBytes3(40).toString("base64url")}`;
|
|
27863
29057
|
await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
|
|
27864
29058
|
refresh_token: nextRefresh,
|
|
27865
29059
|
client_id: record.client_id,
|
|
@@ -27877,35 +29071,35 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
27877
29071
|
// src/api/chat-routes.ts
|
|
27878
29072
|
import { Hono as Hono25 } from "hono";
|
|
27879
29073
|
var chatApp = new Hono25();
|
|
27880
|
-
function tokenOk(
|
|
27881
|
-
return !!
|
|
29074
|
+
function tokenOk(token6) {
|
|
29075
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 10;
|
|
27882
29076
|
}
|
|
27883
29077
|
function jsonError(message, status = 400) {
|
|
27884
29078
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
27885
29079
|
}
|
|
27886
29080
|
chatApp.get("/api/channels", async (c) => {
|
|
27887
|
-
const
|
|
27888
|
-
if (!tokenOk(
|
|
27889
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
29081
|
+
const token6 = c.req.query("token");
|
|
29082
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29083
|
+
const r = await memoryCall("listVaultsTool", {}, token6);
|
|
27890
29084
|
if (!r.ok) return c.json(r);
|
|
27891
29085
|
const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
|
|
27892
29086
|
return c.json({ ok: true, channels: vaults });
|
|
27893
29087
|
});
|
|
27894
29088
|
chatApp.get("/api/messages", async (c) => {
|
|
27895
|
-
const
|
|
27896
|
-
if (!tokenOk(
|
|
29089
|
+
const token6 = c.req.query("token");
|
|
29090
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27897
29091
|
const vault = c.req.query("vault");
|
|
27898
29092
|
if (!vault) return jsonError("vault is required");
|
|
27899
29093
|
const parentMessageId = c.req.query("parentMessageId") || void 0;
|
|
27900
|
-
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId },
|
|
29094
|
+
const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token6);
|
|
27901
29095
|
return c.json(r);
|
|
27902
29096
|
});
|
|
27903
29097
|
chatApp.get("/api/poll", async (c) => {
|
|
27904
|
-
const
|
|
27905
|
-
if (!tokenOk(
|
|
29098
|
+
const token6 = c.req.query("token");
|
|
29099
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27906
29100
|
const vault = c.req.query("vault");
|
|
27907
29101
|
if (!vault) return jsonError("vault is required");
|
|
27908
|
-
const r = await memoryCall("pollChannelTool", { vault },
|
|
29102
|
+
const r = await memoryCall("pollChannelTool", { vault }, token6);
|
|
27909
29103
|
return c.json(r);
|
|
27910
29104
|
});
|
|
27911
29105
|
function parseAttachNote(v) {
|
|
@@ -27930,12 +29124,12 @@ chatApp.post("/api/reply", async (c) => {
|
|
|
27930
29124
|
return c.json(r);
|
|
27931
29125
|
});
|
|
27932
29126
|
chatApp.get("/api/notes-own", async (c) => {
|
|
27933
|
-
const
|
|
27934
|
-
if (!tokenOk(
|
|
27935
|
-
const vaultsRes = await memoryCall("listVaultsTool", {},
|
|
29127
|
+
const token6 = c.req.query("token");
|
|
29128
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29129
|
+
const vaultsRes = await memoryCall("listVaultsTool", {}, token6);
|
|
27936
29130
|
if (!vaultsRes.ok) return c.json(vaultsRes);
|
|
27937
29131
|
const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
|
|
27938
|
-
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle },
|
|
29132
|
+
const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token6)));
|
|
27939
29133
|
const notes = ownVaults.flatMap((v, i) => {
|
|
27940
29134
|
const r = lists[i];
|
|
27941
29135
|
return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
|
|
@@ -27943,12 +29137,12 @@ chatApp.get("/api/notes-own", async (c) => {
|
|
|
27943
29137
|
return c.json({ ok: true, notes });
|
|
27944
29138
|
});
|
|
27945
29139
|
chatApp.get("/api/message-note", async (c) => {
|
|
27946
|
-
const
|
|
27947
|
-
if (!tokenOk(
|
|
29140
|
+
const token6 = c.req.query("token");
|
|
29141
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27948
29142
|
const vault = c.req.query("vault");
|
|
27949
29143
|
const messageId = c.req.query("messageId");
|
|
27950
29144
|
if (!vault || !messageId) return jsonError("vault and messageId are required");
|
|
27951
|
-
const r = await memoryCall("getMessageNoteTool", { vault, messageId },
|
|
29145
|
+
const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token6);
|
|
27952
29146
|
return c.json(r);
|
|
27953
29147
|
});
|
|
27954
29148
|
chatApp.post("/api/react", async (c) => {
|
|
@@ -27959,17 +29153,17 @@ chatApp.post("/api/react", async (c) => {
|
|
|
27959
29153
|
return c.json(r);
|
|
27960
29154
|
});
|
|
27961
29155
|
chatApp.get("/api/mentions", async (c) => {
|
|
27962
|
-
const
|
|
27963
|
-
if (!tokenOk(
|
|
27964
|
-
const r = await memoryCall("myMentionsTool", {},
|
|
29156
|
+
const token6 = c.req.query("token");
|
|
29157
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
29158
|
+
const r = await memoryCall("myMentionsTool", {}, token6);
|
|
27965
29159
|
return c.json(r);
|
|
27966
29160
|
});
|
|
27967
29161
|
chatApp.get("/api/members", async (c) => {
|
|
27968
|
-
const
|
|
27969
|
-
if (!tokenOk(
|
|
29162
|
+
const token6 = c.req.query("token");
|
|
29163
|
+
if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
|
|
27970
29164
|
const vault = c.req.query("vault");
|
|
27971
29165
|
if (!vault) return jsonError("vault is required");
|
|
27972
|
-
const r = await memoryCall("listChannelMembersTool", { vault },
|
|
29166
|
+
const r = await memoryCall("listChannelMembersTool", { vault }, token6);
|
|
27973
29167
|
return c.json(r);
|
|
27974
29168
|
});
|
|
27975
29169
|
chatApp.post("/api/channel", async (c) => {
|
|
@@ -28005,13 +29199,13 @@ chatApp.post("/api/member-remove", async (c) => {
|
|
|
28005
29199
|
return c.json(r);
|
|
28006
29200
|
});
|
|
28007
29201
|
chatApp.get("/:token", (c) => {
|
|
28008
|
-
const
|
|
28009
|
-
if (!tokenOk(
|
|
29202
|
+
const token6 = c.req.param("token");
|
|
29203
|
+
if (!tokenOk(token6)) {
|
|
28010
29204
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
|
|
28011
29205
|
}
|
|
28012
|
-
return c.html(renderChatPage(
|
|
29206
|
+
return c.html(renderChatPage(token6));
|
|
28013
29207
|
});
|
|
28014
|
-
function renderChatPage(
|
|
29208
|
+
function renderChatPage(token6) {
|
|
28015
29209
|
return `<!doctype html>
|
|
28016
29210
|
<html>
|
|
28017
29211
|
<head>
|
|
@@ -28196,7 +29390,7 @@ function renderChatPage(token4) {
|
|
|
28196
29390
|
</div>
|
|
28197
29391
|
|
|
28198
29392
|
<script>
|
|
28199
|
-
const TOKEN = ${JSON.stringify(
|
|
29393
|
+
const TOKEN = ${JSON.stringify(token6)};
|
|
28200
29394
|
const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
|
|
28201
29395
|
|
|
28202
29396
|
function api(path, opts) {
|
|
@@ -28566,8 +29760,8 @@ var MAX_VAULT_RECORDS = 200;
|
|
|
28566
29760
|
var MAX_MEMORY_READ_CONCURRENCY = 4;
|
|
28567
29761
|
var MAX_MEMORY_READ_ATTEMPTS = 3;
|
|
28568
29762
|
var vaultApp = new Hono26();
|
|
28569
|
-
function validToken(
|
|
28570
|
-
return !!
|
|
29763
|
+
function validToken(token6) {
|
|
29764
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 20 && token6.length < 256;
|
|
28571
29765
|
}
|
|
28572
29766
|
function isVault(value) {
|
|
28573
29767
|
return typeof value === "string" && VAULTS.includes(value);
|
|
@@ -28693,11 +29887,11 @@ function shouldRetryMemoryRead(error) {
|
|
|
28693
29887
|
function sleep(ms) {
|
|
28694
29888
|
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
28695
29889
|
}
|
|
28696
|
-
async function readMemory(tool, args,
|
|
28697
|
-
let result = await memoryCall(tool, args,
|
|
29890
|
+
async function readMemory(tool, args, token6) {
|
|
29891
|
+
let result = await memoryCall(tool, args, token6);
|
|
28698
29892
|
for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
|
|
28699
29893
|
await sleep(attempt * 150);
|
|
28700
|
-
result = await memoryCall(tool, args,
|
|
29894
|
+
result = await memoryCall(tool, args, token6);
|
|
28701
29895
|
}
|
|
28702
29896
|
return result;
|
|
28703
29897
|
}
|
|
@@ -28714,27 +29908,27 @@ async function mapWithConcurrency3(items, limit, map) {
|
|
|
28714
29908
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
28715
29909
|
return values;
|
|
28716
29910
|
}
|
|
28717
|
-
async function readVault(vault,
|
|
28718
|
-
const list = await readMemory("listTool", { vault },
|
|
29911
|
+
async function readVault(vault, token6) {
|
|
29912
|
+
const list = await readMemory("listTool", { vault }, token6);
|
|
28719
29913
|
if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
|
|
28720
29914
|
const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
|
|
28721
29915
|
const notes = await mapWithConcurrency3(
|
|
28722
29916
|
noteRefs,
|
|
28723
29917
|
MAX_MEMORY_READ_CONCURRENCY,
|
|
28724
|
-
({ path: path5 }) => readMemory("getTool", { vault, path: path5 },
|
|
29918
|
+
({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token6)
|
|
28725
29919
|
);
|
|
28726
29920
|
const failed = notes.find((result) => !result.ok);
|
|
28727
29921
|
if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
|
|
28728
29922
|
return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
|
|
28729
29923
|
}
|
|
28730
29924
|
vaultApp.get("/api/workspace", async (c) => {
|
|
28731
|
-
const
|
|
28732
|
-
if (!validToken(
|
|
29925
|
+
const token6 = c.req.query("token");
|
|
29926
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28733
29927
|
try {
|
|
28734
29928
|
const requested = c.req.query("vault");
|
|
28735
29929
|
const vaults = isVault(requested) ? [requested] : VAULTS;
|
|
28736
29930
|
const results = [];
|
|
28737
|
-
for (const vault of vaults) results.push(await readVault(vault,
|
|
29931
|
+
for (const vault of vaults) results.push(await readVault(vault, token6));
|
|
28738
29932
|
const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
28739
29933
|
return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
|
|
28740
29934
|
} catch (error) {
|
|
@@ -28743,8 +29937,8 @@ vaultApp.get("/api/workspace", async (c) => {
|
|
|
28743
29937
|
});
|
|
28744
29938
|
vaultApp.post("/api/records", async (c) => {
|
|
28745
29939
|
const body = await c.req.json().catch(() => ({}));
|
|
28746
|
-
const
|
|
28747
|
-
if (!validToken(
|
|
29940
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29941
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28748
29942
|
const vault = body.vault;
|
|
28749
29943
|
const path5 = stringValue(body.path, 320);
|
|
28750
29944
|
const title = stringValue(body.title, 320);
|
|
@@ -28755,19 +29949,19 @@ vaultApp.post("/api/records", async (c) => {
|
|
|
28755
29949
|
const validationError2 = validateNewRecord(vault, normalizedProps);
|
|
28756
29950
|
if (validationError2) return c.json({ ok: false, error: validationError2 }, 400);
|
|
28757
29951
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28758
|
-
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } },
|
|
29952
|
+
const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token6);
|
|
28759
29953
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28760
29954
|
});
|
|
28761
29955
|
vaultApp.post("/api/records/transition", async (c) => {
|
|
28762
29956
|
const body = await c.req.json().catch(() => ({}));
|
|
28763
|
-
const
|
|
28764
|
-
if (!validToken(
|
|
29957
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29958
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28765
29959
|
const vault = typeof body.vault === "string" ? body.vault : "";
|
|
28766
29960
|
const path5 = typeof body.path === "string" ? body.path : "";
|
|
28767
29961
|
const status = stringValue(body.status, 80);
|
|
28768
29962
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28769
29963
|
if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
|
|
28770
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
29964
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28771
29965
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28772
29966
|
const result = await memoryCall("putTool", {
|
|
28773
29967
|
vault,
|
|
@@ -28776,13 +29970,13 @@ vaultApp.post("/api/records/transition", async (c) => {
|
|
|
28776
29970
|
content: current.note.content,
|
|
28777
29971
|
props: { ...current.note.props ?? {}, status },
|
|
28778
29972
|
baseRevision: baseRevision ?? current.note.revision
|
|
28779
|
-
},
|
|
29973
|
+
}, token6);
|
|
28780
29974
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28781
29975
|
});
|
|
28782
29976
|
vaultApp.post("/api/records/update", async (c) => {
|
|
28783
29977
|
const body = await c.req.json().catch(() => ({}));
|
|
28784
|
-
const
|
|
28785
|
-
if (!validToken(
|
|
29978
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
29979
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28786
29980
|
const vault = body.vault;
|
|
28787
29981
|
const path5 = stringValue(body.path, 320);
|
|
28788
29982
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -28792,7 +29986,7 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
28792
29986
|
if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
|
|
28793
29987
|
return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
|
|
28794
29988
|
}
|
|
28795
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
29989
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28796
29990
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28797
29991
|
const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
|
|
28798
29992
|
const validationError2 = validateNewRecord(vault, nextProps);
|
|
@@ -28804,18 +29998,18 @@ vaultApp.post("/api/records/update", async (c) => {
|
|
|
28804
29998
|
content: content ?? current.note.content,
|
|
28805
29999
|
props: nextProps,
|
|
28806
30000
|
baseRevision: baseRevision ?? current.note.revision
|
|
28807
|
-
},
|
|
30001
|
+
}, token6);
|
|
28808
30002
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28809
30003
|
});
|
|
28810
30004
|
vaultApp.delete("/api/records", async (c) => {
|
|
28811
30005
|
const body = await c.req.json().catch(() => ({}));
|
|
28812
|
-
const
|
|
28813
|
-
if (!validToken(
|
|
30006
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30007
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28814
30008
|
const vault = body.vault;
|
|
28815
30009
|
const path5 = stringValue(body.path, 320);
|
|
28816
30010
|
const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
|
|
28817
30011
|
if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
|
|
28818
|
-
const current = await readMemory("getTool", { vault, path: path5 },
|
|
30012
|
+
const current = await readMemory("getTool", { vault, path: path5 }, token6);
|
|
28819
30013
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28820
30014
|
if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
|
|
28821
30015
|
return c.json({ ok: false, error: "only task records can be deleted" }, 400);
|
|
@@ -28824,13 +30018,13 @@ vaultApp.delete("/api/records", async (c) => {
|
|
|
28824
30018
|
vault,
|
|
28825
30019
|
path: path5,
|
|
28826
30020
|
baseRevision: baseRevision ?? current.note.revision
|
|
28827
|
-
},
|
|
30021
|
+
}, token6);
|
|
28828
30022
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28829
30023
|
});
|
|
28830
30024
|
vaultApp.post("/api/communications/review", async (c) => {
|
|
28831
30025
|
const body = await c.req.json().catch(() => ({}));
|
|
28832
|
-
const
|
|
28833
|
-
if (!validToken(
|
|
30026
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30027
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28834
30028
|
const path5 = stringValue(body.path, 320);
|
|
28835
30029
|
const action2 = stringValue(body.action, 24);
|
|
28836
30030
|
const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
|
|
@@ -28841,7 +30035,7 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
28841
30035
|
if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
|
|
28842
30036
|
return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
|
|
28843
30037
|
}
|
|
28844
|
-
const current = await readMemory("getTool", { vault: "Communications", path: path5 },
|
|
30038
|
+
const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token6);
|
|
28845
30039
|
if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
|
|
28846
30040
|
if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
|
|
28847
30041
|
const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
|
|
@@ -28865,13 +30059,13 @@ vaultApp.post("/api/communications/review", async (c) => {
|
|
|
28865
30059
|
content: content ?? current.note.content,
|
|
28866
30060
|
props,
|
|
28867
30061
|
baseRevision: baseRevision ?? current.note.revision
|
|
28868
|
-
},
|
|
30062
|
+
}, token6);
|
|
28869
30063
|
return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
|
|
28870
30064
|
});
|
|
28871
30065
|
vaultApp.post("/api/activity", async (c) => {
|
|
28872
30066
|
const body = await c.req.json().catch(() => ({}));
|
|
28873
|
-
const
|
|
28874
|
-
if (!validToken(
|
|
30067
|
+
const token6 = typeof body.token === "string" ? body.token : void 0;
|
|
30068
|
+
if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
|
|
28875
30069
|
const title = stringValue(body.title, 320);
|
|
28876
30070
|
const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
|
|
28877
30071
|
const props = propsValue(body.props) ?? {};
|
|
@@ -28881,7 +30075,7 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
28881
30075
|
return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
|
|
28882
30076
|
}
|
|
28883
30077
|
for (const ref of personRefs) {
|
|
28884
|
-
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") },
|
|
30078
|
+
const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token6);
|
|
28885
30079
|
if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
|
|
28886
30080
|
return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
|
|
28887
30081
|
}
|
|
@@ -28909,37 +30103,37 @@ vaultApp.post("/api/activity", async (c) => {
|
|
|
28909
30103
|
...props,
|
|
28910
30104
|
person_refs: personRefs
|
|
28911
30105
|
}
|
|
28912
|
-
},
|
|
30106
|
+
}, token6);
|
|
28913
30107
|
return c.json(result, result.ok ? 200 : 502);
|
|
28914
30108
|
});
|
|
28915
30109
|
|
|
28916
30110
|
// src/api/schedule-routes.ts
|
|
28917
30111
|
import { Hono as Hono27 } from "hono";
|
|
28918
30112
|
var scheduleApp = new Hono27();
|
|
28919
|
-
function tokenOk2(
|
|
28920
|
-
return !!
|
|
30113
|
+
function tokenOk2(token6) {
|
|
30114
|
+
return !!token6 && token6.startsWith("mk_") && token6.length > 10;
|
|
28921
30115
|
}
|
|
28922
30116
|
function jsonError2(message, status = 400) {
|
|
28923
30117
|
return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
|
|
28924
30118
|
}
|
|
28925
30119
|
scheduleApp.get("/api/status", async (c) => {
|
|
28926
|
-
const
|
|
28927
|
-
if (!tokenOk2(
|
|
28928
|
-
const r = await memoryCall("getScheduleStatusTool", {},
|
|
30120
|
+
const token6 = c.req.query("token");
|
|
30121
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30122
|
+
const r = await memoryCall("getScheduleStatusTool", {}, token6);
|
|
28929
30123
|
return c.json(r);
|
|
28930
30124
|
});
|
|
28931
30125
|
scheduleApp.get("/api/vaults", async (c) => {
|
|
28932
|
-
const
|
|
28933
|
-
if (!tokenOk2(
|
|
28934
|
-
const r = await memoryCall("listVaultsTool", {},
|
|
30126
|
+
const token6 = c.req.query("token");
|
|
30127
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30128
|
+
const r = await memoryCall("listVaultsTool", {}, token6);
|
|
28935
30129
|
if (!r.ok) return c.json(r);
|
|
28936
30130
|
const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
|
|
28937
30131
|
return c.json({ ok: true, vaults });
|
|
28938
30132
|
});
|
|
28939
30133
|
scheduleApp.get("/api/actions", async (c) => {
|
|
28940
|
-
const
|
|
28941
|
-
if (!tokenOk2(
|
|
28942
|
-
const r = await memoryCall("listScheduledActionsTool", {},
|
|
30134
|
+
const token6 = c.req.query("token");
|
|
30135
|
+
if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
|
|
30136
|
+
const r = await memoryCall("listScheduledActionsTool", {}, token6);
|
|
28943
30137
|
return c.json(r);
|
|
28944
30138
|
});
|
|
28945
30139
|
scheduleApp.post("/api/propose", async (c) => {
|
|
@@ -28975,13 +30169,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
|
|
|
28975
30169
|
return c.json(r);
|
|
28976
30170
|
});
|
|
28977
30171
|
scheduleApp.get("/:token", (c) => {
|
|
28978
|
-
const
|
|
28979
|
-
if (!tokenOk2(
|
|
30172
|
+
const token6 = c.req.param("token");
|
|
30173
|
+
if (!tokenOk2(token6)) {
|
|
28980
30174
|
return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
|
|
28981
30175
|
}
|
|
28982
|
-
return c.html(renderSchedulePage(
|
|
30176
|
+
return c.html(renderSchedulePage(token6));
|
|
28983
30177
|
});
|
|
28984
|
-
function renderSchedulePage(
|
|
30178
|
+
function renderSchedulePage(token6) {
|
|
28985
30179
|
return `<!doctype html>
|
|
28986
30180
|
<html>
|
|
28987
30181
|
<head>
|
|
@@ -29079,7 +30273,7 @@ function renderSchedulePage(token4) {
|
|
|
29079
30273
|
</div>
|
|
29080
30274
|
|
|
29081
30275
|
<script>
|
|
29082
|
-
const TOKEN = ${JSON.stringify(
|
|
30276
|
+
const TOKEN = ${JSON.stringify(token6)};
|
|
29083
30277
|
const state = { actions: [], vaults: [] };
|
|
29084
30278
|
|
|
29085
30279
|
function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
|
|
@@ -29322,8 +30516,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
|
|
|
29322
30516
|
return shortHash(`${customerEmail.trim().toLowerCase()}
|
|
29323
30517
|
${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
|
|
29324
30518
|
}
|
|
29325
|
-
function supportThreadAddress(
|
|
29326
|
-
const safeToken =
|
|
30519
|
+
function supportThreadAddress(token6, receivingDomain) {
|
|
30520
|
+
const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
29327
30521
|
const domain = receivingDomain.trim().toLowerCase();
|
|
29328
30522
|
if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
|
|
29329
30523
|
return `support+${safeToken}@${domain}`;
|
|
@@ -29350,8 +30544,8 @@ function personPath(identity) {
|
|
|
29350
30544
|
const local = identity.email.split("@")[0];
|
|
29351
30545
|
return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
|
|
29352
30546
|
}
|
|
29353
|
-
function issuePath(
|
|
29354
|
-
return `support/threads/${
|
|
30547
|
+
function issuePath(token6) {
|
|
30548
|
+
return `support/threads/${token6}`;
|
|
29355
30549
|
}
|
|
29356
30550
|
function communicationPath(email) {
|
|
29357
30551
|
const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
|
|
@@ -29535,8 +30729,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
|
|
|
29535
30729
|
`- Person: ${noteRef("People", personPathValue)}`
|
|
29536
30730
|
].join("\n");
|
|
29537
30731
|
}
|
|
29538
|
-
async function findOrCreateIssue(memoryKey, email, customer, personPathValue,
|
|
29539
|
-
const path5 = issuePath(
|
|
30732
|
+
async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token6, threadAddressValue) {
|
|
30733
|
+
const path5 = issuePath(token6);
|
|
29540
30734
|
const existing = await getNote(memoryKey, "Issues", path5);
|
|
29541
30735
|
if (existing) return existing;
|
|
29542
30736
|
const subject = normalizeSupportSubject(email.subject);
|
|
@@ -29546,7 +30740,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29546
30740
|
path: path5,
|
|
29547
30741
|
title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
|
|
29548
30742
|
content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
|
|
29549
|
-
source: `resend:thread:${
|
|
30743
|
+
source: `resend:thread:${token6}`,
|
|
29550
30744
|
type: "user_friction",
|
|
29551
30745
|
tagCandidates: ISSUE_TAGS,
|
|
29552
30746
|
props: {
|
|
@@ -29555,7 +30749,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29555
30749
|
summary,
|
|
29556
30750
|
pinned: false,
|
|
29557
30751
|
source_type: "thread",
|
|
29558
|
-
source_ref: `resend:thread:${
|
|
30752
|
+
source_ref: `resend:thread:${token6}`,
|
|
29559
30753
|
related: [],
|
|
29560
30754
|
related_vault_notes: [noteRef("People", personPathValue)],
|
|
29561
30755
|
embed: true,
|
|
@@ -29570,7 +30764,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
|
|
|
29570
30764
|
customer_email: customer.email,
|
|
29571
30765
|
customer_name: customer.name,
|
|
29572
30766
|
normalized_subject: subject,
|
|
29573
|
-
thread_token:
|
|
30767
|
+
thread_token: token6,
|
|
29574
30768
|
thread_address: threadAddressValue,
|
|
29575
30769
|
communication_refs: [],
|
|
29576
30770
|
message_count: 0,
|
|
@@ -29613,7 +30807,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
|
|
|
29613
30807
|
...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
|
|
29614
30808
|
].join("\n");
|
|
29615
30809
|
}
|
|
29616
|
-
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue,
|
|
30810
|
+
async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token6, parentPath) {
|
|
29617
30811
|
const path5 = communicationPath(email);
|
|
29618
30812
|
const existing = await getNote(memoryKey, "Communication", path5);
|
|
29619
30813
|
if (existing) return existing;
|
|
@@ -29652,8 +30846,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
|
|
|
29652
30846
|
provider: "resend",
|
|
29653
30847
|
provider_connection_ref: "resend:updates.mcpscraper.dev",
|
|
29654
30848
|
provider_message_ref: providerMessageRef,
|
|
29655
|
-
provider_thread_ref:
|
|
29656
|
-
thread_ref:
|
|
30849
|
+
provider_thread_ref: token6,
|
|
30850
|
+
thread_ref: token6,
|
|
29657
30851
|
issue_ref: issuePathValue,
|
|
29658
30852
|
sender_email: supportSender(email)?.email ?? email.from,
|
|
29659
30853
|
recipient_emails: email.to ?? [],
|
|
@@ -29713,8 +30907,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
|
|
|
29713
30907
|
if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
|
|
29714
30908
|
}
|
|
29715
30909
|
}
|
|
29716
|
-
async function getSupportThreadByToken(memoryKey,
|
|
29717
|
-
const safeToken =
|
|
30910
|
+
async function getSupportThreadByToken(memoryKey, token6) {
|
|
30911
|
+
const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
|
|
29718
30912
|
if (safeToken.length !== 24) return null;
|
|
29719
30913
|
const path5 = issuePath(safeToken);
|
|
29720
30914
|
const issue = await getNote(memoryKey, "Issues", path5);
|
|
@@ -29742,8 +30936,8 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29742
30936
|
const customer = options.customer ?? supportSender(email);
|
|
29743
30937
|
if (!customer) throw new Error("support sender does not contain a usable email address");
|
|
29744
30938
|
const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
|
|
29745
|
-
const
|
|
29746
|
-
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(
|
|
30939
|
+
const token6 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
|
|
30940
|
+
const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token6, options.receivingDomain);
|
|
29747
30941
|
const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
|
|
29748
30942
|
if (!person) throw new Error("support person could not be resolved");
|
|
29749
30943
|
const issue = options.existingThread?.issue ?? await findOrCreateIssue(
|
|
@@ -29751,7 +30945,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29751
30945
|
email,
|
|
29752
30946
|
customer,
|
|
29753
30947
|
person.path,
|
|
29754
|
-
|
|
30948
|
+
token6,
|
|
29755
30949
|
threadAddressValue
|
|
29756
30950
|
);
|
|
29757
30951
|
const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
|
|
@@ -29762,7 +30956,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29762
30956
|
options.direction,
|
|
29763
30957
|
person.path,
|
|
29764
30958
|
issue.path,
|
|
29765
|
-
|
|
30959
|
+
token6,
|
|
29766
30960
|
parentPath
|
|
29767
30961
|
);
|
|
29768
30962
|
await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
|
|
@@ -29771,7 +30965,7 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
29771
30965
|
communicationPath: communication.path,
|
|
29772
30966
|
personPath: person.path,
|
|
29773
30967
|
parentCommunicationPath: parentPath,
|
|
29774
|
-
threadToken:
|
|
30968
|
+
threadToken: token6,
|
|
29775
30969
|
threadAddress: threadAddressValue,
|
|
29776
30970
|
customerEmail: customer.email,
|
|
29777
30971
|
normalizedSubject,
|
|
@@ -29944,8 +31138,8 @@ resendInboundApp.post("/webhooks", async (c) => {
|
|
|
29944
31138
|
threadAddress: captured.threadAddress
|
|
29945
31139
|
});
|
|
29946
31140
|
}
|
|
29947
|
-
const
|
|
29948
|
-
const thread =
|
|
31141
|
+
const token6 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
|
|
31142
|
+
const thread = token6 ? await getSupportThreadByToken(memoryKey, token6) : null;
|
|
29949
31143
|
if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
|
|
29950
31144
|
const operator = parseMailbox(destination);
|
|
29951
31145
|
if (operator && sender.email === operator.email) {
|
|
@@ -30231,8 +31425,8 @@ var workshopFieldNotes = [
|
|
|
30231
31425
|
function escapeHtml3(value) {
|
|
30232
31426
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
30233
31427
|
}
|
|
30234
|
-
function workshopFieldNoteUrl(
|
|
30235
|
-
return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(
|
|
31428
|
+
function workshopFieldNoteUrl(slug4) {
|
|
31429
|
+
return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug4)}`;
|
|
30236
31430
|
}
|
|
30237
31431
|
function renderWorkshopFieldNotesBody() {
|
|
30238
31432
|
let activeCategory = "";
|
|
@@ -30791,16 +31985,16 @@ import { z as z31 } from "zod";
|
|
|
30791
31985
|
// src/editorial-reading-room/render.ts
|
|
30792
31986
|
import { createHash as createHash11 } from "crypto";
|
|
30793
31987
|
import { readFileSync as readFileSync3 } from "fs";
|
|
30794
|
-
import { dirname as dirname3, join as
|
|
31988
|
+
import { dirname as dirname3, join as join7, resolve } from "path";
|
|
30795
31989
|
import { Marked, Renderer } from "marked";
|
|
30796
31990
|
var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
|
|
30797
31991
|
var assetCache = /* @__PURE__ */ new Map();
|
|
30798
31992
|
function assetPath(name) {
|
|
30799
31993
|
const candidates = [
|
|
30800
|
-
|
|
30801
|
-
|
|
30802
|
-
|
|
30803
|
-
|
|
31994
|
+
join7(runtimeEntryDir, "editorial-reading-room", "assets", name),
|
|
31995
|
+
join7(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
|
|
31996
|
+
join7(process.cwd(), "src", "editorial-reading-room", "assets", name),
|
|
31997
|
+
join7(process.cwd(), "dist", "editorial-reading-room", "assets", name)
|
|
30804
31998
|
];
|
|
30805
31999
|
for (const candidate of candidates) {
|
|
30806
32000
|
try {
|
|
@@ -30874,8 +32068,8 @@ function addHeadingIds(html) {
|
|
|
30874
32068
|
});
|
|
30875
32069
|
return { rendered, toc };
|
|
30876
32070
|
}
|
|
30877
|
-
function safeFilename(requested,
|
|
30878
|
-
const stem = (requested ||
|
|
32071
|
+
function safeFilename(requested, slug4) {
|
|
32072
|
+
const stem = (requested || slug4).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
|
|
30879
32073
|
return `${stem}.html`;
|
|
30880
32074
|
}
|
|
30881
32075
|
function escapedJsonForHtml(value) {
|
|
@@ -31031,20 +32225,20 @@ ${provenance}`;
|
|
|
31031
32225
|
var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
|
|
31032
32226
|
var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
31033
32227
|
var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
31034
|
-
function
|
|
32228
|
+
function token3() {
|
|
31035
32229
|
return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
31036
32230
|
}
|
|
31037
|
-
function
|
|
32231
|
+
function policy3() {
|
|
31038
32232
|
return {
|
|
31039
32233
|
prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
|
|
31040
32234
|
artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
|
|
31041
32235
|
downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
|
|
31042
|
-
token:
|
|
32236
|
+
token: token3()
|
|
31043
32237
|
};
|
|
31044
32238
|
}
|
|
31045
32239
|
function createEditorialReadingRoomArtifact(args) {
|
|
31046
32240
|
return createPrivateArtifact({
|
|
31047
|
-
policy:
|
|
32241
|
+
policy: policy3(),
|
|
31048
32242
|
ownerId: args.ownerId,
|
|
31049
32243
|
artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
|
|
31050
32244
|
createdAt: args.rendered.generatedAt,
|
|
@@ -31055,7 +32249,7 @@ function createEditorialReadingRoomArtifact(args) {
|
|
|
31055
32249
|
}
|
|
31056
32250
|
function renewEditorialReadingRoomArtifactDownload(args) {
|
|
31057
32251
|
return renewPrivateArtifactDownload({
|
|
31058
|
-
policy:
|
|
32252
|
+
policy: policy3(),
|
|
31059
32253
|
ownerId: args.ownerId,
|
|
31060
32254
|
artifactId: args.artifactId
|
|
31061
32255
|
});
|
|
@@ -31835,7 +33029,7 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
31835
33029
|
async function prepareCommonsEntity(input, user) {
|
|
31836
33030
|
await ensureCommonsSchema();
|
|
31837
33031
|
const title = optionalText2(input.title, 180) ?? "";
|
|
31838
|
-
const
|
|
33032
|
+
const slug4 = normalizeSlug(input.slug || title);
|
|
31839
33033
|
const schemaOrgType = optionalText2(input.schemaOrgType, 80);
|
|
31840
33034
|
const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
|
|
31841
33035
|
const entityType = contract.entityType;
|
|
@@ -31843,7 +33037,7 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31843
33037
|
const headings = extractMarkdownHeadings(bodyMarkdown);
|
|
31844
33038
|
const duplicateCandidates = await potentialDuplicateCandidates({
|
|
31845
33039
|
title,
|
|
31846
|
-
slug:
|
|
33040
|
+
slug: slug4,
|
|
31847
33041
|
source: normalizeSource(input.source),
|
|
31848
33042
|
maxCandidates: clampNumber(input.maxCandidates, 1, 20, 8)
|
|
31849
33043
|
});
|
|
@@ -31860,8 +33054,8 @@ async function prepareCommonsEntity(input, user) {
|
|
|
31860
33054
|
entityIdPrefix: "TPW-Q",
|
|
31861
33055
|
userId: user?.id ?? null,
|
|
31862
33056
|
normalizedEntityType: entityType,
|
|
31863
|
-
proposedSlug:
|
|
31864
|
-
publicUrl:
|
|
33057
|
+
proposedSlug: slug4 || null,
|
|
33058
|
+
publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
|
|
31865
33059
|
writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
|
|
31866
33060
|
},
|
|
31867
33061
|
contract,
|
|
@@ -31896,7 +33090,7 @@ async function validateCommonsEntity(input, user) {
|
|
|
31896
33090
|
const warnings = [];
|
|
31897
33091
|
const title = optionalText2(input.title, 180);
|
|
31898
33092
|
const description = optionalText2(input.description, 600);
|
|
31899
|
-
const
|
|
33093
|
+
const slug4 = normalizeSlug(input.slug || title || "");
|
|
31900
33094
|
const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
|
|
31901
33095
|
const source = normalizeSource(input.source);
|
|
31902
33096
|
const featuredImage = normalizeFeaturedImage(input.featuredImage);
|
|
@@ -31905,11 +33099,11 @@ async function validateCommonsEntity(input, user) {
|
|
|
31905
33099
|
const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
|
|
31906
33100
|
const duplicateCandidates = await potentialDuplicateCandidates({
|
|
31907
33101
|
title: title ?? "",
|
|
31908
|
-
slug:
|
|
33102
|
+
slug: slug4,
|
|
31909
33103
|
source,
|
|
31910
33104
|
maxCandidates: 8
|
|
31911
33105
|
});
|
|
31912
|
-
const existing =
|
|
33106
|
+
const existing = slug4 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug4, input.entityId) : null;
|
|
31913
33107
|
try {
|
|
31914
33108
|
normalizeCommonsClaims(input.claims);
|
|
31915
33109
|
} catch (error) {
|
|
@@ -31917,7 +33111,7 @@ async function validateCommonsEntity(input, user) {
|
|
|
31917
33111
|
}
|
|
31918
33112
|
if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
|
|
31919
33113
|
if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
|
|
31920
|
-
if (!
|
|
33114
|
+
if (!slug4) errors.push("A Commons entity needs a URL slug.");
|
|
31921
33115
|
if (!featuredImage?.url) errors.push("A featured image is required before a Commons entity can publish.");
|
|
31922
33116
|
if (!bodyMarkdown.trim() && !source.originalUrl && !source.resolvedUrl) {
|
|
31923
33117
|
errors.push("A public entity needs article body text or source evidence before publishing.");
|
|
@@ -32378,13 +33572,13 @@ async function findProposalByIdempotency(userId, idempotencyKey3) {
|
|
|
32378
33572
|
});
|
|
32379
33573
|
return result.rows[0] ? rowToProposal(result.rows[0]) : null;
|
|
32380
33574
|
}
|
|
32381
|
-
async function findEntityBySlugOrIdAnyStatus(
|
|
33575
|
+
async function findEntityBySlugOrIdAnyStatus(slug4, entityId) {
|
|
32382
33576
|
const result = entityId ? await getDb().execute({
|
|
32383
33577
|
sql: "SELECT * FROM commons_entities WHERE slug = ? OR id = ? LIMIT 1",
|
|
32384
|
-
args: [
|
|
33578
|
+
args: [slug4, entityId]
|
|
32385
33579
|
}) : await getDb().execute({
|
|
32386
33580
|
sql: "SELECT * FROM commons_entities WHERE slug = ? LIMIT 1",
|
|
32387
|
-
args: [
|
|
33581
|
+
args: [slug4]
|
|
32388
33582
|
});
|
|
32389
33583
|
return result.rows[0] ? rowToEntity(result.rows[0]) : null;
|
|
32390
33584
|
}
|
|
@@ -32576,13 +33770,13 @@ function canonicalizeSourceUrl(value) {
|
|
|
32576
33770
|
function normalizeSubmitInput(input) {
|
|
32577
33771
|
const title = requiredText(input.title, "title", 3, 180);
|
|
32578
33772
|
const description = requiredText(input.description, "description", 8, 600);
|
|
32579
|
-
const
|
|
32580
|
-
if (!
|
|
33773
|
+
const slug4 = normalizeSlug(input.slug || title);
|
|
33774
|
+
if (!slug4) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
|
|
32581
33775
|
return {
|
|
32582
33776
|
idempotencyKey: requiredText(input.idempotencyKey, "idempotencyKey", 8, 200),
|
|
32583
33777
|
title,
|
|
32584
33778
|
description,
|
|
32585
|
-
slug:
|
|
33779
|
+
slug: slug4,
|
|
32586
33780
|
entityId: normalizeEntityId(input.entityId),
|
|
32587
33781
|
entityType: normalizeEntityType(input.entityType),
|
|
32588
33782
|
disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
|
|
@@ -33131,13 +34325,13 @@ function normalizeRelatedLinks(relatedLinks) {
|
|
|
33131
34325
|
return relatedLinks.map((link) => {
|
|
33132
34326
|
const title = requiredText(link.title, "relatedLinks.title", 1, 180);
|
|
33133
34327
|
const url = optionalText2(link.url, 2e3);
|
|
33134
|
-
const
|
|
34328
|
+
const slug4 = link.slug ? normalizeSlug(link.slug) : void 0;
|
|
33135
34329
|
const entityId = normalizeEntityId(link.entityId);
|
|
33136
|
-
const needsLink = link.needsLink === true || !url && !
|
|
34330
|
+
const needsLink = link.needsLink === true || !url && !slug4 && !entityId;
|
|
33137
34331
|
return {
|
|
33138
34332
|
title,
|
|
33139
34333
|
url,
|
|
33140
|
-
slug:
|
|
34334
|
+
slug: slug4,
|
|
33141
34335
|
entityId,
|
|
33142
34336
|
relationship: optionalText2(link.relationship, 120) ?? void 0,
|
|
33143
34337
|
summary: optionalText2(link.summary, 300) ?? void 0,
|
|
@@ -33368,8 +34562,8 @@ function actorLabel(input, user) {
|
|
|
33368
34562
|
function publicBaseUrl() {
|
|
33369
34563
|
return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
|
|
33370
34564
|
}
|
|
33371
|
-
function publicUrlForSlug(
|
|
33372
|
-
return `${publicBaseUrl()}/wiki/${encodeURIComponent(
|
|
34565
|
+
function publicUrlForSlug(slug4) {
|
|
34566
|
+
return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug4)}`;
|
|
33373
34567
|
}
|
|
33374
34568
|
function allocateEntityId() {
|
|
33375
34569
|
return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
|
|
@@ -33571,11 +34765,11 @@ async function validateCommonsPublication(input, user) {
|
|
|
33571
34765
|
} catch (error) {
|
|
33572
34766
|
errors.push(error instanceof Error ? error.message : String(error));
|
|
33573
34767
|
}
|
|
33574
|
-
const
|
|
33575
|
-
const latest = publication ? await getLatestEdition(publication.id,
|
|
33576
|
-
if (latest && input.baseRevision === void 0) errors.push(`Edition ${
|
|
34768
|
+
const slug4 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
|
|
34769
|
+
const latest = publication ? await getLatestEdition(publication.id, slug4) : null;
|
|
34770
|
+
if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug4} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
|
|
33577
34771
|
if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
|
|
33578
|
-
errors.push(`Edition ${
|
|
34772
|
+
errors.push(`Edition ${slug4} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
|
|
33579
34773
|
}
|
|
33580
34774
|
}
|
|
33581
34775
|
}
|
|
@@ -34324,7 +35518,7 @@ import Papa2 from "papaparse";
|
|
|
34324
35518
|
import {
|
|
34325
35519
|
createHash as createHash15,
|
|
34326
35520
|
createHmac as createHmac3,
|
|
34327
|
-
randomBytes as
|
|
35521
|
+
randomBytes as randomBytes4,
|
|
34328
35522
|
timingSafeEqual as timingSafeEqual2
|
|
34329
35523
|
} from "crypto";
|
|
34330
35524
|
import { z as z33 } from "zod";
|
|
@@ -34536,7 +35730,7 @@ function dashboardCallbackUrl() {
|
|
|
34536
35730
|
}
|
|
34537
35731
|
async function createThorbitConnectUrl(userId) {
|
|
34538
35732
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
34539
|
-
const state =
|
|
35733
|
+
const state = randomBytes4(32).toString("base64url");
|
|
34540
35734
|
const stateHash = createHash15("sha256").update(state).digest("hex");
|
|
34541
35735
|
await getAnalyticsPool().query(
|
|
34542
35736
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
@@ -34549,8 +35743,8 @@ async function createThorbitConnectUrl(userId) {
|
|
|
34549
35743
|
url.searchParams.set("state", state);
|
|
34550
35744
|
return url.toString();
|
|
34551
35745
|
}
|
|
34552
|
-
function verifyThorbitAssertion(
|
|
34553
|
-
const [encoded, signature] =
|
|
35746
|
+
function verifyThorbitAssertion(token6) {
|
|
35747
|
+
const [encoded, signature] = token6.split(".");
|
|
34554
35748
|
if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
|
|
34555
35749
|
const expected = createHmac3("sha256", bridgeSecret()).update(encoded).digest();
|
|
34556
35750
|
const actual = Buffer.from(signature, "base64url");
|
|
@@ -36028,7 +37222,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
|
36028
37222
|
}
|
|
36029
37223
|
|
|
36030
37224
|
// src/api/scheduled-run-view-links.ts
|
|
36031
|
-
import { createHash as createHash18, randomBytes as
|
|
37225
|
+
import { createHash as createHash18, randomBytes as randomBytes5, randomUUID as randomUUID17 } from "crypto";
|
|
36032
37226
|
var schemaReady2 = false;
|
|
36033
37227
|
async function ensureScheduledRunViewLinksSchema() {
|
|
36034
37228
|
if (schemaReady2) return;
|
|
@@ -36081,8 +37275,8 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
36081
37275
|
`);
|
|
36082
37276
|
schemaReady2 = true;
|
|
36083
37277
|
}
|
|
36084
|
-
function tokenHash2(
|
|
36085
|
-
return createHash18("sha256").update(
|
|
37278
|
+
function tokenHash2(token6) {
|
|
37279
|
+
return createHash18("sha256").update(token6).digest("hex");
|
|
36086
37280
|
}
|
|
36087
37281
|
function mapRow(row) {
|
|
36088
37282
|
return {
|
|
@@ -36102,7 +37296,7 @@ function mapRow(row) {
|
|
|
36102
37296
|
async function createScheduledRunViewLink(input) {
|
|
36103
37297
|
await ensureScheduledRunViewLinksSchema();
|
|
36104
37298
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
36105
|
-
const
|
|
37299
|
+
const token6 = randomBytes5(32).toString("base64url");
|
|
36106
37300
|
const record = {
|
|
36107
37301
|
shareId: randomUUID17(),
|
|
36108
37302
|
ownerId: input.ownerId,
|
|
@@ -36111,7 +37305,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
36111
37305
|
title: input.title.slice(0, 300),
|
|
36112
37306
|
summary: input.summary.slice(0, 2e3),
|
|
36113
37307
|
rendererKey: input.rendererKey,
|
|
36114
|
-
tokenHash: tokenHash2(
|
|
37308
|
+
tokenHash: tokenHash2(token6),
|
|
36115
37309
|
expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
|
|
36116
37310
|
revokedAt: null,
|
|
36117
37311
|
createdAt: now.toISOString()
|
|
@@ -36136,9 +37330,9 @@ async function createScheduledRunViewLink(input) {
|
|
|
36136
37330
|
record.createdAt
|
|
36137
37331
|
]
|
|
36138
37332
|
});
|
|
36139
|
-
return { record, token:
|
|
37333
|
+
return { record, token: token6 };
|
|
36140
37334
|
}
|
|
36141
|
-
async function resolveScheduledRunViewLink(
|
|
37335
|
+
async function resolveScheduledRunViewLink(token6, now = /* @__PURE__ */ new Date()) {
|
|
36142
37336
|
await ensureScheduledRunViewLinksSchema();
|
|
36143
37337
|
const result = await getDb().execute({
|
|
36144
37338
|
sql: `
|
|
@@ -36150,7 +37344,7 @@ async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Dat
|
|
|
36150
37344
|
AND expires_at > ?
|
|
36151
37345
|
LIMIT 1
|
|
36152
37346
|
`,
|
|
36153
|
-
args: [tokenHash2(
|
|
37347
|
+
args: [tokenHash2(token6), now.toISOString()]
|
|
36154
37348
|
});
|
|
36155
37349
|
const row = result.rows[0];
|
|
36156
37350
|
return row ? mapRow(row) : null;
|
|
@@ -36565,7 +37759,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
36565
37759
|
const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
|
|
36566
37760
|
if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
|
|
36567
37761
|
const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
|
|
36568
|
-
const { record, token:
|
|
37762
|
+
const { record, token: token6 } = await createScheduledRunViewLink({
|
|
36569
37763
|
ownerId,
|
|
36570
37764
|
runId: result.data.run.runId,
|
|
36571
37765
|
artifactId: artifact.artifactId,
|
|
@@ -36580,7 +37774,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
|
|
|
36580
37774
|
shareId: record.shareId,
|
|
36581
37775
|
runId: record.runId,
|
|
36582
37776
|
artifactId: record.artifactId,
|
|
36583
|
-
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${
|
|
37777
|
+
url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token6}`,
|
|
36584
37778
|
expiresAt: record.expiresAt,
|
|
36585
37779
|
createdAt: record.createdAt
|
|
36586
37780
|
}, 201);
|
|
@@ -36606,15 +37800,15 @@ import { createHash as createHash21 } from "crypto";
|
|
|
36606
37800
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
36607
37801
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
36608
37802
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
36609
|
-
function
|
|
37803
|
+
function token4() {
|
|
36610
37804
|
return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
36611
37805
|
}
|
|
36612
|
-
function
|
|
37806
|
+
function policy4() {
|
|
36613
37807
|
return {
|
|
36614
37808
|
prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
|
|
36615
37809
|
artifactTtlMs: null,
|
|
36616
37810
|
downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
|
|
36617
|
-
token:
|
|
37811
|
+
token: token4()
|
|
36618
37812
|
};
|
|
36619
37813
|
}
|
|
36620
37814
|
function runStorageSegment(runId) {
|
|
@@ -36625,7 +37819,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
36625
37819
|
throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
|
|
36626
37820
|
}
|
|
36627
37821
|
const stored = await createPrivateArtifact({
|
|
36628
|
-
policy:
|
|
37822
|
+
policy: policy4(),
|
|
36629
37823
|
ownerId: args.request.ownerId,
|
|
36630
37824
|
scopeSegments: [runStorageSegment(args.request.runId)],
|
|
36631
37825
|
artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
|
|
@@ -36651,7 +37845,7 @@ async function createScheduledRunArtifact(args) {
|
|
|
36651
37845
|
async function readScheduledRunArtifact(args) {
|
|
36652
37846
|
if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
|
|
36653
37847
|
const window2 = await readPrivateArtifactWindow({
|
|
36654
|
-
policy:
|
|
37848
|
+
policy: policy4(),
|
|
36655
37849
|
artifactId: args.artifactId,
|
|
36656
37850
|
offset: 0,
|
|
36657
37851
|
maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
|
|
@@ -37214,6 +38408,282 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
37214
38408
|
return input.background === true || input.downloadImages === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
|
|
37215
38409
|
}
|
|
37216
38410
|
|
|
38411
|
+
// src/api/page-media-artifacts.ts
|
|
38412
|
+
import { createHash as createHash23, randomBytes as randomBytes6 } from "crypto";
|
|
38413
|
+
import { homedir as homedir3 } from "os";
|
|
38414
|
+
import { basename, extname, join as join8 } from "path";
|
|
38415
|
+
import pLimit6 from "p-limit";
|
|
38416
|
+
import { ZipFile as ZipFile2 } from "yazl";
|
|
38417
|
+
var PAGE_MEDIA_ARTIFACT_PREFIX = "page-media/";
|
|
38418
|
+
var PAGE_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
38419
|
+
var PAGE_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
38420
|
+
var MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
38421
|
+
var MAX_ARCHIVE_MEDIA_BYTES = 45 * 1024 * 1024;
|
|
38422
|
+
var MAX_INLINE_IMAGE_BYTES2 = 15e5;
|
|
38423
|
+
var MAX_INLINE_TOTAL_BYTES2 = 25e5;
|
|
38424
|
+
var DOWNLOAD_CONCURRENCY2 = 6;
|
|
38425
|
+
var MAX_REDIRECTS2 = 5;
|
|
38426
|
+
function token5() {
|
|
38427
|
+
return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
|
|
38428
|
+
}
|
|
38429
|
+
function policy5() {
|
|
38430
|
+
return {
|
|
38431
|
+
prefix: PAGE_MEDIA_ARTIFACT_PREFIX,
|
|
38432
|
+
artifactTtlMs: PAGE_MEDIA_ARTIFACT_TTL_MS,
|
|
38433
|
+
downloadTtlMs: PAGE_MEDIA_DOWNLOAD_TTL_MS,
|
|
38434
|
+
token: token5()
|
|
38435
|
+
};
|
|
38436
|
+
}
|
|
38437
|
+
function imageSignature(bytes, contentType) {
|
|
38438
|
+
if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
|
|
38439
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
|
|
38440
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
|
|
38441
|
+
if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
|
|
38442
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 12)).match(/^ftyp(?:avif|avis)$/)) return { mimeType: "image/avif", extension: "avif" };
|
|
38443
|
+
const prefix = Buffer.from(bytes.slice(0, Math.min(bytes.length, 1024))).toString("utf8").replace(/^\uFEFF/, "").trimStart();
|
|
38444
|
+
if ((contentType === "image/svg+xml" || prefix.startsWith("<svg") || /^<\?xml[\s\S]*?<svg/i.test(prefix)) && /<svg[\s>]/i.test(prefix)) {
|
|
38445
|
+
return { mimeType: "image/svg+xml", extension: "svg" };
|
|
38446
|
+
}
|
|
38447
|
+
return null;
|
|
38448
|
+
}
|
|
38449
|
+
function mediaSignature(bytes, contentType, expectedType) {
|
|
38450
|
+
if (expectedType === "image") return imageSignature(bytes, contentType);
|
|
38451
|
+
if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 8)) === "ftyp") return { mimeType: "video/mp4", extension: "mp4" };
|
|
38452
|
+
if (bytes.length >= 4 && bytes[0] === 26 && bytes[1] === 69 && bytes[2] === 223 && bytes[3] === 163) return { mimeType: "video/webm", extension: "webm" };
|
|
38453
|
+
if (bytes.length >= 4 && String.fromCharCode(...bytes.slice(0, 4)) === "OggS") {
|
|
38454
|
+
return expectedType === "audio" ? { mimeType: "audio/ogg", extension: "ogg" } : { mimeType: "video/ogg", extension: "ogv" };
|
|
38455
|
+
}
|
|
38456
|
+
if (expectedType === "audio" && bytes.length >= 3 && String.fromCharCode(...bytes.slice(0, 3)) === "ID3") return { mimeType: "audio/mpeg", extension: "mp3" };
|
|
38457
|
+
if (contentType && typeFromMime(contentType) === expectedType) {
|
|
38458
|
+
const extension = contentType.split("/")[1]?.replace("jpeg", "jpg").replace("mpeg", "mp3").replace("svg+xml", "svg");
|
|
38459
|
+
if (extension && /^[a-z0-9]+$/i.test(extension)) return { mimeType: contentType, extension };
|
|
38460
|
+
}
|
|
38461
|
+
return null;
|
|
38462
|
+
}
|
|
38463
|
+
function imageDimensions(bytes, mimeType) {
|
|
38464
|
+
const buffer = Buffer.from(bytes);
|
|
38465
|
+
if (mimeType === "image/png" && buffer.length >= 24) return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
|
|
38466
|
+
if (mimeType === "image/gif" && buffer.length >= 10) return { width: buffer.readUInt16LE(6), height: buffer.readUInt16LE(8) };
|
|
38467
|
+
if (mimeType === "image/webp" && buffer.length >= 30) {
|
|
38468
|
+
const chunk = buffer.toString("ascii", 12, 16);
|
|
38469
|
+
if (chunk === "VP8X") return { width: 1 + buffer.readUIntLE(24, 3), height: 1 + buffer.readUIntLE(27, 3) };
|
|
38470
|
+
if (chunk === "VP8L" && buffer[20] === 47) {
|
|
38471
|
+
const bits = buffer.readUInt32LE(21);
|
|
38472
|
+
return { width: (bits & 16383) + 1, height: (bits >> 14 & 16383) + 1 };
|
|
38473
|
+
}
|
|
38474
|
+
}
|
|
38475
|
+
if (mimeType === "image/jpeg") {
|
|
38476
|
+
let offset = 2;
|
|
38477
|
+
while (offset + 9 < buffer.length) {
|
|
38478
|
+
if (buffer[offset] !== 255) {
|
|
38479
|
+
offset += 1;
|
|
38480
|
+
continue;
|
|
38481
|
+
}
|
|
38482
|
+
const marker = buffer[offset + 1];
|
|
38483
|
+
if (marker === 216 || marker === 217) {
|
|
38484
|
+
offset += 2;
|
|
38485
|
+
continue;
|
|
38486
|
+
}
|
|
38487
|
+
const length = buffer.readUInt16BE(offset + 2);
|
|
38488
|
+
if (length < 2 || offset + length + 2 > buffer.length) break;
|
|
38489
|
+
if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker)) {
|
|
38490
|
+
return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
|
|
38491
|
+
}
|
|
38492
|
+
offset += length + 2;
|
|
38493
|
+
}
|
|
38494
|
+
}
|
|
38495
|
+
if (mimeType === "image/svg+xml") {
|
|
38496
|
+
const source = buffer.toString("utf8", 0, Math.min(buffer.length, 64 * 1024));
|
|
38497
|
+
const tag = source.match(/<svg\b[^>]*>/i)?.[0] ?? "";
|
|
38498
|
+
const width = Number(tag.match(/\bwidth=["']?([\d.]+)/i)?.[1] ?? 0);
|
|
38499
|
+
const height = Number(tag.match(/\bheight=["']?([\d.]+)/i)?.[1] ?? 0);
|
|
38500
|
+
if (width > 0 && height > 0) return { width: Math.round(width), height: Math.round(height) };
|
|
38501
|
+
const viewBox = tag.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);
|
|
38502
|
+
if (viewBox?.length === 4 && viewBox[2] > 0 && viewBox[3] > 0) return { width: Math.round(viewBox[2]), height: Math.round(viewBox[3]) };
|
|
38503
|
+
}
|
|
38504
|
+
return null;
|
|
38505
|
+
}
|
|
38506
|
+
async function readBounded2(response) {
|
|
38507
|
+
const declared = Number(response.headers.get("content-length") ?? "");
|
|
38508
|
+
if (Number.isFinite(declared) && declared > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38509
|
+
if (!response.body) {
|
|
38510
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
38511
|
+
if (bytes.length > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38512
|
+
return bytes;
|
|
38513
|
+
}
|
|
38514
|
+
const reader = response.body.getReader();
|
|
38515
|
+
const chunks = [];
|
|
38516
|
+
let total = 0;
|
|
38517
|
+
try {
|
|
38518
|
+
for (; ; ) {
|
|
38519
|
+
const { done, value } = await reader.read();
|
|
38520
|
+
if (done) break;
|
|
38521
|
+
if (!value) continue;
|
|
38522
|
+
total += value.byteLength;
|
|
38523
|
+
if (total > MAX_FILE_BYTES) throw new Error("media_too_large");
|
|
38524
|
+
chunks.push(Buffer.from(value));
|
|
38525
|
+
}
|
|
38526
|
+
} finally {
|
|
38527
|
+
reader.releaseLock();
|
|
38528
|
+
}
|
|
38529
|
+
return Buffer.concat(chunks);
|
|
38530
|
+
}
|
|
38531
|
+
async function fetchMedia(rawUrl, expectedType) {
|
|
38532
|
+
let target = rawUrl;
|
|
38533
|
+
for (let redirects = 0; redirects <= MAX_REDIRECTS2; redirects += 1) {
|
|
38534
|
+
const checked = await validatePublicHttpUrl(target, { field: "website media URL" });
|
|
38535
|
+
if (checked.error || !checked.parsed) throw new Error("media_url_rejected");
|
|
38536
|
+
const response = await fetch(checked.parsed.href, {
|
|
38537
|
+
redirect: "manual",
|
|
38538
|
+
headers: { Accept: expectedType === "image" ? "image/*" : `${expectedType}/*` },
|
|
38539
|
+
signal: AbortSignal.timeout(15e3)
|
|
38540
|
+
});
|
|
38541
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
38542
|
+
const location2 = response.headers.get("location");
|
|
38543
|
+
if (!location2 || redirects === MAX_REDIRECTS2) throw new Error("media_redirect_rejected");
|
|
38544
|
+
target = new URL(location2, checked.parsed).href;
|
|
38545
|
+
continue;
|
|
38546
|
+
}
|
|
38547
|
+
if (!response.ok) throw new Error(`media_fetch_${response.status}`);
|
|
38548
|
+
const bytes = await readBounded2(response);
|
|
38549
|
+
const declared = response.headers.get("content-type")?.split(";")[0].trim().toLowerCase() ?? null;
|
|
38550
|
+
const detected = mediaSignature(bytes, declared, expectedType);
|
|
38551
|
+
if (!detected || typeFromMime(detected.mimeType) !== expectedType) throw new Error("media_content_invalid");
|
|
38552
|
+
const dimensions = expectedType === "image" ? imageDimensions(bytes, detected.mimeType) : null;
|
|
38553
|
+
return {
|
|
38554
|
+
bytes,
|
|
38555
|
+
...detected,
|
|
38556
|
+
finalUrl: checked.parsed.href,
|
|
38557
|
+
width: dimensions?.width ?? null,
|
|
38558
|
+
height: dimensions?.height ?? null,
|
|
38559
|
+
sha256: createHash23("sha256").update(bytes).digest("hex")
|
|
38560
|
+
};
|
|
38561
|
+
}
|
|
38562
|
+
throw new Error("media_redirect_rejected");
|
|
38563
|
+
}
|
|
38564
|
+
function zipBuffer2(files) {
|
|
38565
|
+
return new Promise((resolve2, reject) => {
|
|
38566
|
+
const zip = new ZipFile2();
|
|
38567
|
+
const chunks = [];
|
|
38568
|
+
zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
|
38569
|
+
zip.outputStream.once("error", reject);
|
|
38570
|
+
zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
|
|
38571
|
+
for (const file of files) zip.addBuffer(file.content, file.path);
|
|
38572
|
+
zip.end();
|
|
38573
|
+
});
|
|
38574
|
+
}
|
|
38575
|
+
function slug3(value) {
|
|
38576
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "website";
|
|
38577
|
+
}
|
|
38578
|
+
function safeOriginalName(rawUrl) {
|
|
38579
|
+
try {
|
|
38580
|
+
const name = basename(new URL(rawUrl).pathname).replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
38581
|
+
return name.slice(0, 80);
|
|
38582
|
+
} catch {
|
|
38583
|
+
return "";
|
|
38584
|
+
}
|
|
38585
|
+
}
|
|
38586
|
+
function pageMediaArtifactOwnerId(artifactId) {
|
|
38587
|
+
return privateArtifactOwnerId(artifactId, PAGE_MEDIA_ARTIFACT_PREFIX);
|
|
38588
|
+
}
|
|
38589
|
+
async function readOwnedPageMediaArtifactBuffer(args) {
|
|
38590
|
+
if (pageMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
|
|
38591
|
+
return readPrivateArtifactBuffer({ policy: policy5(), artifactId: args.artifactId, maxBytes: 60 * 1024 * 1024 });
|
|
38592
|
+
}
|
|
38593
|
+
async function packagePageMedia(args) {
|
|
38594
|
+
const files = [];
|
|
38595
|
+
const limit = pLimit6(DOWNLOAD_CONCURRENCY2);
|
|
38596
|
+
const digestToFilename = /* @__PURE__ */ new Map();
|
|
38597
|
+
let storedBytes = 0;
|
|
38598
|
+
let inlineBytes = 0;
|
|
38599
|
+
let inlineCount = 0;
|
|
38600
|
+
await Promise.all(args.media.assets.map((asset, position) => limit(async () => {
|
|
38601
|
+
try {
|
|
38602
|
+
const fetched = await fetchMedia(asset.url, asset.type);
|
|
38603
|
+
asset.finalUrl = fetched.finalUrl;
|
|
38604
|
+
asset.mimeType = fetched.mimeType;
|
|
38605
|
+
asset.sizeBytes = fetched.bytes.length;
|
|
38606
|
+
asset.sha256 = fetched.sha256;
|
|
38607
|
+
asset.width = fetched.width ?? asset.width;
|
|
38608
|
+
asset.height = fetched.height ?? asset.height;
|
|
38609
|
+
const duplicate = digestToFilename.get(fetched.sha256);
|
|
38610
|
+
if (duplicate) {
|
|
38611
|
+
asset.filename = duplicate;
|
|
38612
|
+
asset.duplicateOf = duplicate;
|
|
38613
|
+
asset.downloadStatus = "downloaded";
|
|
38614
|
+
asset.downloadError = null;
|
|
38615
|
+
return;
|
|
38616
|
+
}
|
|
38617
|
+
if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_MEDIA_BYTES) throw new Error("archive_byte_limit_reached");
|
|
38618
|
+
storedBytes += fetched.bytes.length;
|
|
38619
|
+
const original = safeOriginalName(fetched.finalUrl);
|
|
38620
|
+
const extension = extname(original);
|
|
38621
|
+
const stem = original ? extension ? original.slice(0, -extension.length) : original : asset.type;
|
|
38622
|
+
const filename = `media/${String(position + 1).padStart(4, "0")}-${slug3(stem)}.${fetched.extension}`;
|
|
38623
|
+
digestToFilename.set(fetched.sha256, filename);
|
|
38624
|
+
files.push({ path: filename, content: fetched.bytes });
|
|
38625
|
+
asset.filename = filename;
|
|
38626
|
+
asset.duplicateOf = null;
|
|
38627
|
+
asset.downloadStatus = "downloaded";
|
|
38628
|
+
asset.downloadError = null;
|
|
38629
|
+
if (asset.type === "image" && inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES2 && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES2) {
|
|
38630
|
+
asset.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
|
|
38631
|
+
inlineBytes += fetched.bytes.length;
|
|
38632
|
+
inlineCount += 1;
|
|
38633
|
+
}
|
|
38634
|
+
} catch (error) {
|
|
38635
|
+
asset.finalUrl = null;
|
|
38636
|
+
asset.mimeType = null;
|
|
38637
|
+
asset.sizeBytes = null;
|
|
38638
|
+
asset.sha256 = null;
|
|
38639
|
+
asset.savedPath = null;
|
|
38640
|
+
asset.downloadStatus = "failed";
|
|
38641
|
+
asset.downloadError = error instanceof Error ? error.message : "media_download_failed";
|
|
38642
|
+
asset.inlinePreview = null;
|
|
38643
|
+
}
|
|
38644
|
+
})));
|
|
38645
|
+
const downloaded = args.media.assets.filter((asset) => asset.downloadStatus === "downloaded").length;
|
|
38646
|
+
const failed = args.media.assets.length - downloaded;
|
|
38647
|
+
if (failed > 0) args.media.warnings.push(`${failed} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`);
|
|
38648
|
+
if (downloaded === 0) return { media: args.media, artifact: null };
|
|
38649
|
+
const cleanAssets = args.media.assets.map(({ inlinePreview: _preview, ...asset }) => asset);
|
|
38650
|
+
const summary = {
|
|
38651
|
+
pageUrl: args.pageUrl,
|
|
38652
|
+
title: args.title,
|
|
38653
|
+
extractedAt: args.extractedAt,
|
|
38654
|
+
completeness: args.media.completeness,
|
|
38655
|
+
exhausted: args.media.exhausted,
|
|
38656
|
+
stopReason: args.media.stopReason,
|
|
38657
|
+
scrollRounds: args.media.scrollRounds,
|
|
38658
|
+
staticFound: args.media.staticFound,
|
|
38659
|
+
renderedFound: args.media.renderedFound,
|
|
38660
|
+
totalFound: args.media.totalFound,
|
|
38661
|
+
retainedAfterFilteringAndVariantCollapse: args.media.assets.length,
|
|
38662
|
+
downloaded,
|
|
38663
|
+
failed,
|
|
38664
|
+
notes: [
|
|
38665
|
+
"The inventory unions static-source and rendered-page discovery.",
|
|
38666
|
+
"complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.",
|
|
38667
|
+
"Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."
|
|
38668
|
+
]
|
|
38669
|
+
};
|
|
38670
|
+
files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
|
|
38671
|
+
files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
|
|
38672
|
+
const archive = await zipBuffer2(files);
|
|
38673
|
+
const id = randomBytes6(6).toString("hex");
|
|
38674
|
+
const pointer = await createPrivateArtifact({
|
|
38675
|
+
policy: policy5(),
|
|
38676
|
+
ownerId: args.ownerId,
|
|
38677
|
+
artifactKey: `${id}.zip`,
|
|
38678
|
+
createdAt: new Date(args.extractedAt),
|
|
38679
|
+
filename: `${slug3(args.title || new URL(args.pageUrl).hostname)}-website-media.zip`,
|
|
38680
|
+
contentType: "application/zip",
|
|
38681
|
+
content: archive
|
|
38682
|
+
});
|
|
38683
|
+
const localPath = token5() ? null : join8(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join8(homedir3(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
|
|
38684
|
+
return { media: args.media, artifact: { ...pointer, localPath } };
|
|
38685
|
+
}
|
|
38686
|
+
|
|
37217
38687
|
// src/api/site-extract-reconciliation.ts
|
|
37218
38688
|
async function redispatchFundedSiteExtractJobs(limit = 25) {
|
|
37219
38689
|
const jobs = await listFundedPendingExtractJobs(limit);
|
|
@@ -37279,13 +38749,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
37279
38749
|
}
|
|
37280
38750
|
|
|
37281
38751
|
// src/api/page-diff.ts
|
|
37282
|
-
import { createHash as
|
|
38752
|
+
import { createHash as createHash24 } from "crypto";
|
|
37283
38753
|
import { diffLines } from "diff";
|
|
37284
38754
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
37285
38755
|
var MAX_DIFF_HUNKS = 200;
|
|
37286
38756
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
37287
38757
|
function sha256Hex(value) {
|
|
37288
|
-
return
|
|
38758
|
+
return createHash24("sha256").update(value).digest("hex");
|
|
37289
38759
|
}
|
|
37290
38760
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
37291
38761
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -37378,8 +38848,8 @@ async function uploadFallbackBlob(opts) {
|
|
|
37378
38848
|
try {
|
|
37379
38849
|
const content = (opts.content ?? "").trim();
|
|
37380
38850
|
if (!content) return null;
|
|
37381
|
-
const
|
|
37382
|
-
const key = `${SCRAPE_FALLBACK_PREFIX}${
|
|
38851
|
+
const slug4 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
|
|
38852
|
+
const key = `${SCRAPE_FALLBACK_PREFIX}${slug4}.md`;
|
|
37383
38853
|
const doc = `# ${opts.title}
|
|
37384
38854
|
Source: ${opts.source}
|
|
37385
38855
|
|
|
@@ -37399,10 +38869,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
37399
38869
|
}
|
|
37400
38870
|
|
|
37401
38871
|
// src/api/scrape-image-sink.ts
|
|
37402
|
-
import { createHash as
|
|
38872
|
+
import { createHash as createHash25 } from "crypto";
|
|
37403
38873
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
37404
38874
|
function idempotencyKey2(userId, vault, input) {
|
|
37405
|
-
return `scrape-image-${
|
|
38875
|
+
return `scrape-image-${createHash25("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
37406
38876
|
}
|
|
37407
38877
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
37408
38878
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -37446,7 +38916,7 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
|
37446
38916
|
}
|
|
37447
38917
|
|
|
37448
38918
|
// src/api/archive-reader.ts
|
|
37449
|
-
import { basename, extname } from "path";
|
|
38919
|
+
import { basename as basename2, extname as extname2 } from "path";
|
|
37450
38920
|
import yauzl from "yauzl";
|
|
37451
38921
|
var MAX_ARCHIVE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
|
|
37452
38922
|
var MAX_ARCHIVE_ENTRIES = 1e4;
|
|
@@ -37501,9 +38971,9 @@ var ArchiveReadError = class extends Error {
|
|
|
37501
38971
|
retryable;
|
|
37502
38972
|
};
|
|
37503
38973
|
function readableContentType(path5) {
|
|
37504
|
-
const extensionType = READABLE_EXTENSIONS.get(
|
|
38974
|
+
const extensionType = READABLE_EXTENSIONS.get(extname2(path5).toLowerCase());
|
|
37505
38975
|
if (extensionType) return extensionType;
|
|
37506
|
-
return READABLE_FILENAMES.has(
|
|
38976
|
+
return READABLE_FILENAMES.has(basename2(path5).toLowerCase()) ? "text/plain" : null;
|
|
37507
38977
|
}
|
|
37508
38978
|
function safeEntryPath(path5) {
|
|
37509
38979
|
const normalized = path5.replaceAll("\\", "/");
|
|
@@ -37864,7 +39334,7 @@ function archiveLibrarySource(archiveUrl, path5) {
|
|
|
37864
39334
|
return source.href;
|
|
37865
39335
|
}
|
|
37866
39336
|
function archiveEntryTitle(path5) {
|
|
37867
|
-
return
|
|
39337
|
+
return basename2(path5).slice(0, 200) || "Archive entry";
|
|
37868
39338
|
}
|
|
37869
39339
|
function inferredWaybackCapturedAt(content) {
|
|
37870
39340
|
const timestamp = content.slice(0, 4e3).match(/^Archive timestamp:\s*(\d{14})\s*$/m)?.[1];
|
|
@@ -37874,7 +39344,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
37874
39344
|
}
|
|
37875
39345
|
|
|
37876
39346
|
// src/api/connection-memory-import.ts
|
|
37877
|
-
import { createHash as
|
|
39347
|
+
import { createHash as createHash26 } from "crypto";
|
|
37878
39348
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
37879
39349
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
37880
39350
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -37904,7 +39374,7 @@ function canonicalJson(value) {
|
|
|
37904
39374
|
return JSON.stringify(value);
|
|
37905
39375
|
}
|
|
37906
39376
|
function sha2562(value) {
|
|
37907
|
-
return
|
|
39377
|
+
return createHash26("sha256").update(value).digest("hex");
|
|
37908
39378
|
}
|
|
37909
39379
|
function sensitiveKey(key) {
|
|
37910
39380
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -38122,17 +39592,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
38122
39592
|
|
|
38123
39593
|
// src/api/scrape-blob-cleanup.ts
|
|
38124
39594
|
import { readdir, stat, unlink } from "fs/promises";
|
|
38125
|
-
import { homedir as
|
|
38126
|
-
import { join as
|
|
38127
|
-
async function cleanupVercel(
|
|
39595
|
+
import { homedir as homedir4 } from "os";
|
|
39596
|
+
import { join as join9 } from "path";
|
|
39597
|
+
async function cleanupVercel(token6, cutoff) {
|
|
38128
39598
|
const { list, del } = await import("@vercel/blob");
|
|
38129
39599
|
let cursor;
|
|
38130
39600
|
let deleted = 0;
|
|
38131
39601
|
do {
|
|
38132
|
-
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token:
|
|
39602
|
+
const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token6 });
|
|
38133
39603
|
const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
|
|
38134
39604
|
if (expired.length) {
|
|
38135
|
-
await del(expired.map((b) => b.url), { token:
|
|
39605
|
+
await del(expired.map((b) => b.url), { token: token6 });
|
|
38136
39606
|
deleted += expired.length;
|
|
38137
39607
|
}
|
|
38138
39608
|
cursor = res.hasMore ? res.cursor : void 0;
|
|
@@ -38140,8 +39610,8 @@ async function cleanupVercel(token4, cutoff) {
|
|
|
38140
39610
|
return { deleted, store: "vercel-blob" };
|
|
38141
39611
|
}
|
|
38142
39612
|
async function cleanupLocal(cutoff) {
|
|
38143
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
38144
|
-
const dir =
|
|
39613
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join9(homedir4(), "Downloads", "mcp-scraper");
|
|
39614
|
+
const dir = join9(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
38145
39615
|
let deleted = 0;
|
|
38146
39616
|
let entries;
|
|
38147
39617
|
try {
|
|
@@ -38150,7 +39620,7 @@ async function cleanupLocal(cutoff) {
|
|
|
38150
39620
|
return { deleted: 0, store: "local" };
|
|
38151
39621
|
}
|
|
38152
39622
|
for (const name of entries) {
|
|
38153
|
-
const path5 =
|
|
39623
|
+
const path5 = join9(dir, name);
|
|
38154
39624
|
try {
|
|
38155
39625
|
const s = await stat(path5);
|
|
38156
39626
|
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
@@ -38165,8 +39635,8 @@ async function cleanupLocal(cutoff) {
|
|
|
38165
39635
|
async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
38166
39636
|
const cutoff = Date.now() - maxAgeMs;
|
|
38167
39637
|
try {
|
|
38168
|
-
const
|
|
38169
|
-
return
|
|
39638
|
+
const token6 = process.env.BLOB_READ_WRITE_TOKEN;
|
|
39639
|
+
return token6 ? await cleanupVercel(token6, cutoff) : await cleanupLocal(cutoff);
|
|
38170
39640
|
} catch {
|
|
38171
39641
|
return { deleted: 0, store: "none" };
|
|
38172
39642
|
}
|
|
@@ -38915,7 +40385,7 @@ async function dbUsage(identity, plan) {
|
|
|
38915
40385
|
}
|
|
38916
40386
|
|
|
38917
40387
|
// src/api/nango-control.ts
|
|
38918
|
-
import { createHash as
|
|
40388
|
+
import { createHash as createHash28, randomUUID as randomUUID22 } from "crypto";
|
|
38919
40389
|
|
|
38920
40390
|
// src/api/slack-archive-analysis.ts
|
|
38921
40391
|
function isRecord2(value) {
|
|
@@ -39289,7 +40759,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
39289
40759
|
}
|
|
39290
40760
|
|
|
39291
40761
|
// src/api/main-nango-transport.ts
|
|
39292
|
-
import { createHash as
|
|
40762
|
+
import { createHash as createHash27, randomUUID as randomUUID21 } from "crypto";
|
|
39293
40763
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
39294
40764
|
|
|
39295
40765
|
// src/api/connected-tool-policy.ts
|
|
@@ -39332,9 +40802,9 @@ function buildConnectedToolPolicyRegistry(args) {
|
|
|
39332
40802
|
return policies;
|
|
39333
40803
|
}
|
|
39334
40804
|
function connectedToolPolicyOrThrow(registry, toolName) {
|
|
39335
|
-
const
|
|
39336
|
-
if (!
|
|
39337
|
-
return
|
|
40805
|
+
const policy6 = registry.get(toolName);
|
|
40806
|
+
if (!policy6) throw new Error("connected_tool_policy_missing");
|
|
40807
|
+
return policy6;
|
|
39338
40808
|
}
|
|
39339
40809
|
|
|
39340
40810
|
// src/api/main-nango-transport.ts
|
|
@@ -39680,9 +41150,9 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
39680
41150
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39681
41151
|
actionsEnabled: connection.actionsEnabled
|
|
39682
41152
|
});
|
|
39683
|
-
const readTools = [...policies.values()].filter((
|
|
39684
|
-
const actionTools = [...policies.values()].filter((
|
|
39685
|
-
const revision =
|
|
41153
|
+
const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
|
|
41154
|
+
const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
|
|
41155
|
+
const revision = createHash27("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
39686
41156
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
39687
41157
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
39688
41158
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -39691,18 +41161,18 @@ async function callNangoToolDirect(args) {
|
|
|
39691
41161
|
const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
|
|
39692
41162
|
const selected = tools.find((tool) => tool.name === args.tool);
|
|
39693
41163
|
if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
39694
|
-
const
|
|
41164
|
+
const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
39695
41165
|
tools,
|
|
39696
41166
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39697
41167
|
actionsEnabled: connection.actionsEnabled
|
|
39698
41168
|
}), selected.name);
|
|
39699
|
-
if (
|
|
41169
|
+
if (policy6.classification !== args.classification) {
|
|
39700
41170
|
throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
|
|
39701
41171
|
}
|
|
39702
|
-
if (!
|
|
41172
|
+
if (!policy6.available && policy6.blockedReason === "actions_disabled") {
|
|
39703
41173
|
throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
|
|
39704
41174
|
}
|
|
39705
|
-
if (!
|
|
41175
|
+
if (!policy6.available) {
|
|
39706
41176
|
throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
|
|
39707
41177
|
}
|
|
39708
41178
|
await preflightConnectedUsage({
|
|
@@ -39710,7 +41180,7 @@ async function callNangoToolDirect(args) {
|
|
|
39710
41180
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
39711
41181
|
});
|
|
39712
41182
|
const requestId = args.requestId?.trim() || randomUUID21();
|
|
39713
|
-
const idempotencyKey3 = `main-nango:${
|
|
41183
|
+
const idempotencyKey3 = `main-nango:${createHash27("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
39714
41184
|
const startedAt = /* @__PURE__ */ new Date();
|
|
39715
41185
|
const started = performance.now();
|
|
39716
41186
|
let result;
|
|
@@ -39735,7 +41205,7 @@ async function callNangoToolDirect(args) {
|
|
|
39735
41205
|
toolName: args.tool,
|
|
39736
41206
|
operationKind: args.operationKind ?? args.classification,
|
|
39737
41207
|
outcome: providerError ? "error" : "partial",
|
|
39738
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
41208
|
+
requestId: requestId.length <= 200 ? requestId : createHash27("sha256").update(requestId).digest("hex"),
|
|
39739
41209
|
startedAt: startedAt.toISOString(),
|
|
39740
41210
|
completedAt: completedAt.toISOString()
|
|
39741
41211
|
}
|
|
@@ -39747,13 +41217,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
39747
41217
|
const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
|
|
39748
41218
|
const tool = tools.find((candidate) => candidate.name === toolName);
|
|
39749
41219
|
if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
|
|
39750
|
-
const
|
|
41220
|
+
const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
|
|
39751
41221
|
tools,
|
|
39752
41222
|
connectionActive: connection.lifecycleStatus === "connected",
|
|
39753
41223
|
actionsEnabled: connection.actionsEnabled
|
|
39754
41224
|
}), tool.name);
|
|
39755
41225
|
const classification = classifyConnectedTool(tool);
|
|
39756
|
-
const callable =
|
|
41226
|
+
const callable = policy6.available;
|
|
39757
41227
|
const projected = {
|
|
39758
41228
|
name: tool.name,
|
|
39759
41229
|
...tool.title ? { title: tool.title } : {},
|
|
@@ -39766,18 +41236,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
39766
41236
|
...projected,
|
|
39767
41237
|
classification,
|
|
39768
41238
|
callable,
|
|
39769
|
-
blockedReason:
|
|
39770
|
-
requiredPermissions:
|
|
39771
|
-
missingPermissions:
|
|
39772
|
-
requiredFeatures:
|
|
39773
|
-
missingFeatures:
|
|
41239
|
+
blockedReason: policy6.blockedReason,
|
|
41240
|
+
requiredPermissions: policy6.requiredPermissions,
|
|
41241
|
+
missingPermissions: policy6.missingPermissions,
|
|
41242
|
+
requiredFeatures: policy6.requiredFeatures,
|
|
41243
|
+
missingFeatures: policy6.missingFeatures,
|
|
39774
41244
|
transport: "nango",
|
|
39775
41245
|
providerConfigKey: connection.providerConfigKey,
|
|
39776
41246
|
providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
|
|
39777
41247
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
39778
41248
|
protocolVersion: null,
|
|
39779
41249
|
schemaSource: "live_tools_list",
|
|
39780
|
-
schemaHash:
|
|
41250
|
+
schemaHash: createHash27("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
39781
41251
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
39782
41252
|
};
|
|
39783
41253
|
}
|
|
@@ -40501,8 +41971,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
40501
41971
|
return data.connection.actionsEnabled === true;
|
|
40502
41972
|
}
|
|
40503
41973
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
40504
|
-
const requestId = `main-connected-action:${
|
|
40505
|
-
const requestDigest =
|
|
41974
|
+
const requestId = `main-connected-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
|
|
41975
|
+
const requestDigest = createHash28("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
40506
41976
|
if (mainOwnsIntegrations()) {
|
|
40507
41977
|
const selectedTool = tool?.trim();
|
|
40508
41978
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -40653,7 +42123,7 @@ function canonicalJson2(value) {
|
|
|
40653
42123
|
return JSON.stringify(value);
|
|
40654
42124
|
}
|
|
40655
42125
|
function projectedToolSchemaHash(tool) {
|
|
40656
|
-
return
|
|
42126
|
+
return createHash28("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
40657
42127
|
}
|
|
40658
42128
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
40659
42129
|
if (mainOwnsIntegrations()) {
|
|
@@ -40932,7 +42402,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
40932
42402
|
}
|
|
40933
42403
|
|
|
40934
42404
|
// src/api/resend-control.ts
|
|
40935
|
-
import { createHash as
|
|
42405
|
+
import { createHash as createHash29 } from "crypto";
|
|
40936
42406
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
40937
42407
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
40938
42408
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -41191,7 +42661,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
41191
42661
|
return isRecord5(data) ? data.result ?? data : data;
|
|
41192
42662
|
}
|
|
41193
42663
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
41194
|
-
const requestId = `main-resend-action:${
|
|
42664
|
+
const requestId = `main-resend-action:${createHash29("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
41195
42665
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
41196
42666
|
method: "POST",
|
|
41197
42667
|
headers: { "x-request-id": requestId },
|
|
@@ -41318,9 +42788,9 @@ var auth4 = createMiddleware5(async (c, next) => {
|
|
|
41318
42788
|
return next();
|
|
41319
42789
|
});
|
|
41320
42790
|
var sessionAuth = createMiddleware5(async (c, next) => {
|
|
41321
|
-
const
|
|
41322
|
-
if (!
|
|
41323
|
-
const userId = verifySession(
|
|
42791
|
+
const token6 = getCookie2(c, "session");
|
|
42792
|
+
if (!token6) return c.json({ error: "Not authenticated" }, 401);
|
|
42793
|
+
const userId = verifySession(token6);
|
|
41324
42794
|
if (!userId) return c.json({ error: "Session expired" }, 401);
|
|
41325
42795
|
const user = await getUserById(userId);
|
|
41326
42796
|
if (!user) return c.json({ error: "User not found" }, 401);
|
|
@@ -41465,8 +42935,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
|
41465
42935
|
console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
|
|
41466
42936
|
}
|
|
41467
42937
|
})();
|
|
41468
|
-
const
|
|
41469
|
-
setCookie2(c, "session",
|
|
42938
|
+
const token6 = signSession(user.id);
|
|
42939
|
+
setCookie2(c, "session", token6, sessionCookieOptions2);
|
|
41470
42940
|
return c.json({ email: user.email, api_key: newKey }, 201);
|
|
41471
42941
|
} catch (err) {
|
|
41472
42942
|
const msg = err instanceof Error ? err.message : "";
|
|
@@ -41484,8 +42954,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
|
|
|
41484
42954
|
if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
|
|
41485
42955
|
return c.json({ error: "Invalid email or password" }, 401);
|
|
41486
42956
|
}
|
|
41487
|
-
const
|
|
41488
|
-
setCookie2(c, "session",
|
|
42957
|
+
const token6 = signSession(user.id);
|
|
42958
|
+
setCookie2(c, "session", token6, sessionCookieOptions2);
|
|
41489
42959
|
return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
|
|
41490
42960
|
});
|
|
41491
42961
|
app.post("/auth/logout", requireAllowedOrigin, (c) => {
|
|
@@ -41540,9 +43010,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
41540
43010
|
if (emailLimited) return emailLimited;
|
|
41541
43011
|
const user = await getUserByEmail(normalizedEmail);
|
|
41542
43012
|
if (!user) return c.json({ ok: true });
|
|
41543
|
-
const
|
|
43013
|
+
const token6 = await createPasswordResetToken(user.id);
|
|
41544
43014
|
const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
|
|
41545
|
-
const resetUrl = `${appUrl}/reset-password?token=${
|
|
43015
|
+
const resetUrl = `${appUrl}/reset-password?token=${token6}`;
|
|
41546
43016
|
if (process.env.RESEND_API_KEY) {
|
|
41547
43017
|
try {
|
|
41548
43018
|
const resend = new Resend3(process.env.RESEND_API_KEY);
|
|
@@ -41562,12 +43032,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
|
|
|
41562
43032
|
return c.json({ ok: true });
|
|
41563
43033
|
});
|
|
41564
43034
|
app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
|
|
41565
|
-
const { token:
|
|
41566
|
-
if (!
|
|
43035
|
+
const { token: token6, password } = await c.req.json();
|
|
43036
|
+
if (!token6 || !password) return c.json({ error: "Token and password required" }, 400);
|
|
41567
43037
|
if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
|
|
41568
43038
|
const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
|
|
41569
43039
|
if (limited) return limited;
|
|
41570
|
-
const userId = await consumePasswordResetToken(
|
|
43040
|
+
const userId = await consumePasswordResetToken(token6);
|
|
41571
43041
|
if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
|
|
41572
43042
|
await setPassword(userId, password);
|
|
41573
43043
|
return c.json({ ok: true });
|
|
@@ -41604,9 +43074,9 @@ function sessionBootstrapPayload(user) {
|
|
|
41604
43074
|
}
|
|
41605
43075
|
app.get("/me", async (c) => {
|
|
41606
43076
|
c.header("Cache-Control", "no-store");
|
|
41607
|
-
const
|
|
41608
|
-
if (!
|
|
41609
|
-
const userId = verifySession(
|
|
43077
|
+
const token6 = getCookie2(c, "session");
|
|
43078
|
+
if (!token6) return c.json({ authenticated: false });
|
|
43079
|
+
const userId = verifySession(token6);
|
|
41610
43080
|
if (!userId) {
|
|
41611
43081
|
deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
|
|
41612
43082
|
return c.json({ authenticated: false });
|
|
@@ -43232,6 +44702,8 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43232
44702
|
}
|
|
43233
44703
|
const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
|
|
43234
44704
|
const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
|
|
44705
|
+
const maxMediaAssets = bodyResult.data.maxMediaAssets ?? 100;
|
|
44706
|
+
const maxInlineImages = bodyResult.data.maxInlineImages ?? 3;
|
|
43235
44707
|
if (!allowLocal) {
|
|
43236
44708
|
const checked = await validatePublicHttpUrl(url, { field: "URL" });
|
|
43237
44709
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
@@ -43266,7 +44738,14 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43266
44738
|
const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
|
|
43267
44739
|
const [result, pageData] = await Promise.all([
|
|
43268
44740
|
extractKpo({ url: extractionUrl, kernelApiKey }),
|
|
43269
|
-
screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, {
|
|
44741
|
+
screenshot2 || extractBranding2 || preserveMedia ? capturePageData(canonicalUrl, {
|
|
44742
|
+
kernelApiKey,
|
|
44743
|
+
device,
|
|
44744
|
+
screenshot: !!screenshot2,
|
|
44745
|
+
branding: !!extractBranding2,
|
|
44746
|
+
media: preserveMedia,
|
|
44747
|
+
maxMediaAssets: Math.min(1e3, Math.max(250, maxMediaAssets * 4))
|
|
44748
|
+
}).catch((err) => {
|
|
43270
44749
|
console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
|
|
43271
44750
|
return null;
|
|
43272
44751
|
}) : null
|
|
@@ -43277,7 +44756,28 @@ app.post("/extract-url", auth4, async (c) => {
|
|
|
43277
44756
|
if (screenshotBuf) {
|
|
43278
44757
|
screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
|
|
43279
44758
|
}
|
|
43280
|
-
|
|
44759
|
+
let mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, {
|
|
44760
|
+
types: mediaTypes ?? ["image", "video", "audio"],
|
|
44761
|
+
outputDir: null,
|
|
44762
|
+
rendered: pageData?.renderedMedia ?? null,
|
|
44763
|
+
maxAssets: maxMediaAssets
|
|
44764
|
+
}) : null;
|
|
44765
|
+
if (mediaMeta) {
|
|
44766
|
+
try {
|
|
44767
|
+
const packaged = await packagePageMedia({
|
|
44768
|
+
ownerId: String(user.id),
|
|
44769
|
+
pageUrl: canonicalUrl,
|
|
44770
|
+
title: result.title ?? null,
|
|
44771
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
44772
|
+
media: mediaMeta,
|
|
44773
|
+
maxInlineImages
|
|
44774
|
+
});
|
|
44775
|
+
mediaMeta = packaged.media;
|
|
44776
|
+
mediaMeta.artifact = packaged.artifact;
|
|
44777
|
+
} catch (error) {
|
|
44778
|
+
mediaMeta.warnings.push(`Media packaging failed: ${error instanceof Error ? error.message : "media_packaging_failed"}`);
|
|
44779
|
+
}
|
|
44780
|
+
}
|
|
43281
44781
|
const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
|
|
43282
44782
|
html: result.bodyHtml,
|
|
43283
44783
|
meta: result.meta,
|
|
@@ -43460,7 +44960,7 @@ app.post("/archive/read", auth4, async (c) => {
|
|
|
43460
44960
|
try {
|
|
43461
44961
|
let ownedArchive = null;
|
|
43462
44962
|
if (artifactId) {
|
|
43463
|
-
const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
44963
|
+
const buffer = artifactId.startsWith(MAPS_MEDIA_ARTIFACT_PREFIX) ? await readOwnedMapsMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : artifactId.startsWith(PAGE_MEDIA_ARTIFACT_PREFIX) ? await readOwnedPageMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
|
|
43464
44964
|
if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
|
|
43465
44965
|
ownedArchive = { buffer, ref: artifactId };
|
|
43466
44966
|
}
|
|
@@ -44270,7 +45770,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
44270
45770
|
return c.json({ error: "Unauthorized" }, 401);
|
|
44271
45771
|
}
|
|
44272
45772
|
const startedAt = Date.now();
|
|
44273
|
-
const { drainQueue } = await import("./worker-
|
|
45773
|
+
const { drainQueue } = await import("./worker-JL4TG6IF.js");
|
|
44274
45774
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
44275
45775
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
44276
45776
|
const unfinished = [];
|
|
@@ -44341,7 +45841,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
44341
45841
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
44342
45842
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
44343
45843
|
} = await import("./site-extract-repository-3VQAALNR.js");
|
|
44344
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
45844
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
|
|
44345
45845
|
const existing = await getExtractJob2(jobId2);
|
|
44346
45846
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
44347
45847
|
if (existing.status !== "failed") {
|
|
@@ -44553,4 +46053,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
44553
46053
|
export {
|
|
44554
46054
|
app
|
|
44555
46055
|
};
|
|
44556
|
-
//# sourceMappingURL=server-
|
|
46056
|
+
//# sourceMappingURL=server-7EXEDKAU.js.map
|