mcp-scraper 0.56.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +564 -156
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +25 -2
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-NP66MZ5A.js → chunk-5XYBKE3X.js} +2 -2
- package/dist/{chunk-LUJYVNO7.js → chunk-HU4R6BP2.js} +2 -2
- package/dist/{chunk-LUJYVNO7.js.map → chunk-HU4R6BP2.js.map} +1 -1
- package/dist/chunk-QQ6ZBYZZ.js +7 -0
- package/dist/chunk-QQ6ZBYZZ.js.map +1 -0
- package/dist/{chunk-WO2SSENO.js → chunk-R6CO77BN.js} +3 -3
- package/dist/{chunk-WO2SSENO.js.map → chunk-R6CO77BN.js.map} +1 -1
- package/dist/{chunk-JEZW6N2X.js → chunk-ULAJAZQF.js} +26 -3
- package/dist/chunk-ULAJAZQF.js.map +1 -0
- package/dist/{extract-bundle-6KYTD55T.js → extract-bundle-H7JEELY7.js} +3 -3
- package/dist/{server-SETYJPYO.js → server-NBGZJX4P.js} +447 -72
- package/dist/server-NBGZJX4P.js.map +1 -0
- package/dist/{site-extract-repository-YGNYFTNE.js → site-extract-repository-CFMSWAVL.js} +3 -3
- package/dist/{worker-DTVBN3RQ.js → worker-S2WXELUC.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-245IAJ7R.js +0 -7
- package/dist/chunk-245IAJ7R.js.map +0 -1
- package/dist/chunk-JEZW6N2X.js.map +0 -1
- package/dist/server-SETYJPYO.js.map +0 -1
- /package/dist/{chunk-NP66MZ5A.js.map → chunk-5XYBKE3X.js.map} +0 -0
- /package/dist/{extract-bundle-6KYTD55T.js.map → extract-bundle-H7JEELY7.js.map} +0 -0
- /package/dist/{site-extract-repository-YGNYFTNE.js.map → site-extract-repository-CFMSWAVL.js.map} +0 -0
- /package/dist/{worker-DTVBN3RQ.js.map → worker-S2WXELUC.js.map} +0 -0
|
@@ -169,7 +169,7 @@ import {
|
|
|
169
169
|
resolveDeploymentProfile,
|
|
170
170
|
resolveLocalSourcebookSchemaType,
|
|
171
171
|
transcribeMediaUrl
|
|
172
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-ULAJAZQF.js";
|
|
173
173
|
import {
|
|
174
174
|
auditImageUrls,
|
|
175
175
|
auditImages,
|
|
@@ -225,7 +225,7 @@ import {
|
|
|
225
225
|
} from "./chunk-FG6LJGJ7.js";
|
|
226
226
|
import {
|
|
227
227
|
PACKAGE_VERSION
|
|
228
|
-
} from "./chunk-
|
|
228
|
+
} from "./chunk-QQ6ZBYZZ.js";
|
|
229
229
|
import {
|
|
230
230
|
abandonExtractSettlement,
|
|
231
231
|
countSuccessfulPages,
|
|
@@ -248,7 +248,7 @@ import {
|
|
|
248
248
|
setExtractJobTotal,
|
|
249
249
|
settleExtractJob,
|
|
250
250
|
terminalExtractJobStatus
|
|
251
|
-
} from "./chunk-
|
|
251
|
+
} from "./chunk-5XYBKE3X.js";
|
|
252
252
|
import {
|
|
253
253
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
254
254
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -301,7 +301,7 @@ import {
|
|
|
301
301
|
tierChangeLeavesPrivateOffer,
|
|
302
302
|
tierCreditsPerInterval,
|
|
303
303
|
vendorCostUsdToMc
|
|
304
|
-
} from "./chunk-
|
|
304
|
+
} from "./chunk-R6CO77BN.js";
|
|
305
305
|
import {
|
|
306
306
|
CaptchaError,
|
|
307
307
|
MapsSearchIncompletePageError,
|
|
@@ -9393,7 +9393,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9393
9393
|
});
|
|
9394
9394
|
}) : null;
|
|
9395
9395
|
const finalized = await step.run("finalize", async () => {
|
|
9396
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9396
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
9397
9397
|
const current = await getExtractJob(jobId2);
|
|
9398
9398
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9399
9399
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -32972,7 +32972,224 @@ import { Hono as Hono31 } from "hono";
|
|
|
32972
32972
|
import { z as z32 } from "zod";
|
|
32973
32973
|
|
|
32974
32974
|
// src/api/commons-repository.ts
|
|
32975
|
-
import { createHash as
|
|
32975
|
+
import { createHash as createHash15, randomUUID as randomUUID17 } from "crypto";
|
|
32976
|
+
|
|
32977
|
+
// src/api/commons-image-store.ts
|
|
32978
|
+
import { createHash as createHash13 } from "crypto";
|
|
32979
|
+
import { lookup as dnsLookup } from "dns/promises";
|
|
32980
|
+
import { isIP } from "net";
|
|
32981
|
+
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
32982
|
+
var COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
32983
|
+
var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
32984
|
+
var CommonsImageError = class extends Error {
|
|
32985
|
+
constructor(code, message, httpStatus = 400) {
|
|
32986
|
+
super(message);
|
|
32987
|
+
this.code = code;
|
|
32988
|
+
this.httpStatus = httpStatus;
|
|
32989
|
+
this.name = "CommonsImageError";
|
|
32990
|
+
}
|
|
32991
|
+
code;
|
|
32992
|
+
httpStatus;
|
|
32993
|
+
};
|
|
32994
|
+
var imageSchemaReady = false;
|
|
32995
|
+
async function ensureCommonsImageSchema() {
|
|
32996
|
+
if (imageSchemaReady) return;
|
|
32997
|
+
await getDb().execute(`
|
|
32998
|
+
CREATE TABLE IF NOT EXISTS commons_images (
|
|
32999
|
+
id TEXT PRIMARY KEY,
|
|
33000
|
+
user_id INTEGER NOT NULL,
|
|
33001
|
+
digest TEXT NOT NULL,
|
|
33002
|
+
storage_key TEXT NOT NULL,
|
|
33003
|
+
url TEXT NOT NULL,
|
|
33004
|
+
content_type TEXT NOT NULL,
|
|
33005
|
+
bytes INTEGER NOT NULL,
|
|
33006
|
+
source_url TEXT,
|
|
33007
|
+
alt TEXT,
|
|
33008
|
+
license TEXT,
|
|
33009
|
+
attribution TEXT,
|
|
33010
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
33011
|
+
)
|
|
33012
|
+
`);
|
|
33013
|
+
await getDb().execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
|
|
33014
|
+
imageSchemaReady = true;
|
|
33015
|
+
}
|
|
33016
|
+
function sniffImageFormat(bytes) {
|
|
33017
|
+
if (bytes.length < 12) return null;
|
|
33018
|
+
if (bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return "image/jpeg";
|
|
33019
|
+
if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return "image/png";
|
|
33020
|
+
if (bytes.subarray(0, 6).toString("ascii") === "GIF87a" || bytes.subarray(0, 6).toString("ascii") === "GIF89a") return "image/gif";
|
|
33021
|
+
if (bytes.subarray(0, 4).toString("ascii") === "RIFF" && bytes.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";
|
|
33022
|
+
return null;
|
|
33023
|
+
}
|
|
33024
|
+
function isBlockedAddress(address) {
|
|
33025
|
+
if (isIP(address) === 6) {
|
|
33026
|
+
const normalized = address.toLowerCase();
|
|
33027
|
+
return normalized === "::1" || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("fe80");
|
|
33028
|
+
}
|
|
33029
|
+
const parts = address.split(".").map(Number);
|
|
33030
|
+
if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) return true;
|
|
33031
|
+
const [a, b] = parts;
|
|
33032
|
+
if (a === 10 || a === 127 || a === 0) return true;
|
|
33033
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
33034
|
+
if (a === 192 && b === 168) return true;
|
|
33035
|
+
if (a === 169 && b === 254) return true;
|
|
33036
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
33037
|
+
return false;
|
|
33038
|
+
}
|
|
33039
|
+
async function assertPublicHttpsUrl(rawUrl) {
|
|
33040
|
+
let url;
|
|
33041
|
+
try {
|
|
33042
|
+
url = new URL(rawUrl);
|
|
33043
|
+
} catch {
|
|
33044
|
+
throw new CommonsImageError("image_url_invalid", `"${rawUrl}" is not a valid URL.`);
|
|
33045
|
+
}
|
|
33046
|
+
if (url.protocol !== "https:") {
|
|
33047
|
+
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
33048
|
+
}
|
|
33049
|
+
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
33050
|
+
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
33051
|
+
if (!addresses.length) {
|
|
33052
|
+
throw new CommonsImageError("image_host_unresolvable", `Could not resolve ${url.hostname}.`);
|
|
33053
|
+
}
|
|
33054
|
+
if (addresses.some((entry) => isBlockedAddress(entry.address))) {
|
|
33055
|
+
throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
|
|
33056
|
+
}
|
|
33057
|
+
return url;
|
|
33058
|
+
}
|
|
33059
|
+
async function downloadImage(sourceUrl) {
|
|
33060
|
+
let current = sourceUrl;
|
|
33061
|
+
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
33062
|
+
const url = await assertPublicHttpsUrl(current);
|
|
33063
|
+
const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
|
|
33064
|
+
if (response.status >= 300 && response.status < 400) {
|
|
33065
|
+
const location2 = response.headers.get("location");
|
|
33066
|
+
if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
|
|
33067
|
+
current = new URL(location2, url).toString();
|
|
33068
|
+
continue;
|
|
33069
|
+
}
|
|
33070
|
+
if (!response.ok) {
|
|
33071
|
+
throw new CommonsImageError("image_source_unavailable", `Image source returned HTTP ${response.status}.`, 422);
|
|
33072
|
+
}
|
|
33073
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
33074
|
+
if (declaredLength > COMMONS_IMAGE_MAX_BYTES) {
|
|
33075
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33076
|
+
}
|
|
33077
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
33078
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33079
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33080
|
+
}
|
|
33081
|
+
return bytes;
|
|
33082
|
+
}
|
|
33083
|
+
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
33084
|
+
}
|
|
33085
|
+
function decodeBase64Image(value) {
|
|
33086
|
+
const payload = value.includes(",") && value.trim().startsWith("data:") ? value.slice(value.indexOf(",") + 1) : value;
|
|
33087
|
+
const bytes = Buffer.from(payload, "base64");
|
|
33088
|
+
if (!bytes.length) throw new CommonsImageError("image_base64_invalid", "imageBase64 did not decode to any bytes.");
|
|
33089
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33090
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33091
|
+
}
|
|
33092
|
+
return bytes;
|
|
33093
|
+
}
|
|
33094
|
+
function isCommonsHostedImageUrl(url) {
|
|
33095
|
+
return /^https:\/\/[^/]*(\.public\.blob\.vercel-storage\.com|blob\.vercel-storage\.com)\//i.test(url) || url.startsWith("file://");
|
|
33096
|
+
}
|
|
33097
|
+
var hostOverride = null;
|
|
33098
|
+
async function hostCommonsImage(input) {
|
|
33099
|
+
if (hostOverride) return hostOverride(input);
|
|
33100
|
+
await ensureCommonsImageSchema();
|
|
33101
|
+
if (!input.sourceUrl && !input.imageBase64) {
|
|
33102
|
+
throw new CommonsImageError("image_source_missing", "Provide either sourceUrl or imageBase64.");
|
|
33103
|
+
}
|
|
33104
|
+
const bytes = input.imageBase64 ? decodeBase64Image(input.imageBase64) : await downloadImage(String(input.sourceUrl));
|
|
33105
|
+
const contentType = sniffImageFormat(bytes);
|
|
33106
|
+
if (!contentType) {
|
|
33107
|
+
throw new CommonsImageError(
|
|
33108
|
+
"image_format_unsupported",
|
|
33109
|
+
"Commons hosts JPEG, PNG, GIF, and WebP. SVG and other formats are rejected because the wiki serves them to the public web.",
|
|
33110
|
+
415
|
|
33111
|
+
);
|
|
33112
|
+
}
|
|
33113
|
+
const digest2 = createHash13("sha256").update(bytes).digest("hex");
|
|
33114
|
+
const existing = await getDb().execute({
|
|
33115
|
+
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
33116
|
+
args: [digest2]
|
|
33117
|
+
});
|
|
33118
|
+
const reusable = existing.rows[0];
|
|
33119
|
+
if (reusable) {
|
|
33120
|
+
return {
|
|
33121
|
+
id: String(reusable.id),
|
|
33122
|
+
url: String(reusable.url),
|
|
33123
|
+
contentType: String(reusable.content_type),
|
|
33124
|
+
bytes: Number(reusable.bytes),
|
|
33125
|
+
digest: digest2,
|
|
33126
|
+
sourceUrl: reusable.source_url ? String(reusable.source_url) : null,
|
|
33127
|
+
reused: true
|
|
33128
|
+
};
|
|
33129
|
+
}
|
|
33130
|
+
const extension = contentType.split("/")[1] === "jpeg" ? "jpg" : contentType.split("/")[1];
|
|
33131
|
+
const storageKey = `commons/images/${digest2.slice(0, 2)}/${digest2}.${extension}`;
|
|
33132
|
+
const stored = await getBlobStore().put(storageKey, bytes, contentType);
|
|
33133
|
+
const id = `TPW-IMG-${digest2.slice(0, 16).toUpperCase()}`;
|
|
33134
|
+
await getDb().execute({
|
|
33135
|
+
sql: `
|
|
33136
|
+
INSERT INTO commons_images (id, user_id, digest, storage_key, url, content_type, bytes, source_url, alt, license, attribution, created_at)
|
|
33137
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
33138
|
+
`,
|
|
33139
|
+
args: [
|
|
33140
|
+
id,
|
|
33141
|
+
input.userId,
|
|
33142
|
+
digest2,
|
|
33143
|
+
storageKey,
|
|
33144
|
+
stored.url,
|
|
33145
|
+
contentType,
|
|
33146
|
+
bytes.length,
|
|
33147
|
+
input.sourceUrl ?? null,
|
|
33148
|
+
input.alt ?? null,
|
|
33149
|
+
input.license ?? null,
|
|
33150
|
+
input.attribution ?? null,
|
|
33151
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
33152
|
+
]
|
|
33153
|
+
});
|
|
33154
|
+
return { id, url: stored.url, contentType, bytes: bytes.length, digest: digest2, sourceUrl: input.sourceUrl ?? null, reused: false };
|
|
33155
|
+
}
|
|
33156
|
+
async function hostEntityImages(input) {
|
|
33157
|
+
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
33158
|
+
const featuredUrl = input.featuredImage?.url;
|
|
33159
|
+
if (featuredUrl && !isCommonsHostedImageUrl(featuredUrl)) {
|
|
33160
|
+
const hosted = await hostCommonsImage({
|
|
33161
|
+
userId: input.userId,
|
|
33162
|
+
sourceUrl: featuredUrl,
|
|
33163
|
+
alt: input.featuredImage?.alt,
|
|
33164
|
+
license: input.featuredImage?.license,
|
|
33165
|
+
attribution: input.featuredImage?.sourceUrl
|
|
33166
|
+
}).catch((error) => {
|
|
33167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33168
|
+
throw new CommonsImageError(
|
|
33169
|
+
error instanceof CommonsImageError ? error.code : "featured_image_unhostable",
|
|
33170
|
+
`The featured image could not be hosted, so this entity would publish with a link that can break: ${message}`,
|
|
33171
|
+
422
|
|
33172
|
+
);
|
|
33173
|
+
});
|
|
33174
|
+
rewrite.featuredImageUrl = hosted.url;
|
|
33175
|
+
}
|
|
33176
|
+
for (const item of input.media ?? []) {
|
|
33177
|
+
if (!item.url || isCommonsHostedImageUrl(item.url)) continue;
|
|
33178
|
+
try {
|
|
33179
|
+
const hosted = await hostCommonsImage({
|
|
33180
|
+
userId: input.userId,
|
|
33181
|
+
sourceUrl: item.url,
|
|
33182
|
+
alt: item.alt,
|
|
33183
|
+
license: item.license,
|
|
33184
|
+
attribution: item.sourceUrl
|
|
33185
|
+
});
|
|
33186
|
+
rewrite.mediaUrls.set(item.url, hosted.url);
|
|
33187
|
+
} catch (error) {
|
|
33188
|
+
rewrite.dropped.push({ url: item.url, reason: error instanceof Error ? error.message : String(error) });
|
|
33189
|
+
}
|
|
33190
|
+
}
|
|
33191
|
+
return rewrite;
|
|
33192
|
+
}
|
|
32976
33193
|
|
|
32977
33194
|
// src/api/schema-presence.ts
|
|
32978
33195
|
var cache = null;
|
|
@@ -33000,7 +33217,7 @@ async function schemaObjectsPresent(names) {
|
|
|
33000
33217
|
}
|
|
33001
33218
|
|
|
33002
33219
|
// src/api/commons-linksets.ts
|
|
33003
|
-
import { createHash as
|
|
33220
|
+
import { createHash as createHash14 } from "crypto";
|
|
33004
33221
|
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
33005
33222
|
var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
33006
33223
|
var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -33107,7 +33324,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
33107
33324
|
context[claim.predicate] = targets;
|
|
33108
33325
|
}
|
|
33109
33326
|
const document2 = { linkset: [context] };
|
|
33110
|
-
const etag = `"${
|
|
33327
|
+
const etag = `"${createHash14("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
33111
33328
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
33112
33329
|
}
|
|
33113
33330
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -33640,23 +33857,100 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
33640
33857
|
const pattern = `%${escapeLike(term)}%`;
|
|
33641
33858
|
args.push(pattern, pattern, pattern, pattern, pattern);
|
|
33642
33859
|
}
|
|
33643
|
-
|
|
33644
|
-
|
|
33860
|
+
if (!terms.length) {
|
|
33861
|
+
args.push(limit, offset);
|
|
33862
|
+
const result = await getDb().execute({
|
|
33863
|
+
sql: `
|
|
33864
|
+
SELECT *
|
|
33865
|
+
FROM commons_entities
|
|
33866
|
+
WHERE ${clauses.join(" AND ")}
|
|
33867
|
+
ORDER BY updated_at DESC, title ASC
|
|
33868
|
+
LIMIT ? OFFSET ?
|
|
33869
|
+
`,
|
|
33870
|
+
args
|
|
33871
|
+
});
|
|
33872
|
+
const results2 = result.rows.map((row) => rowToEntity(row));
|
|
33873
|
+
await resolveRelatedLinksAgainstPublishedEntities(results2);
|
|
33874
|
+
return { results: results2, filters: mergedFilters, limit, offset };
|
|
33875
|
+
}
|
|
33876
|
+
const candidates = await getDb().execute({
|
|
33645
33877
|
sql: `
|
|
33646
|
-
SELECT
|
|
33878
|
+
SELECT id, title, tags_json, keywords_json, description, search_text, updated_at,
|
|
33879
|
+
(
|
|
33880
|
+
SELECT group_concat(
|
|
33881
|
+
coalesce(commons_claims.title, '') || ' ' ||
|
|
33882
|
+
coalesce(commons_claims.predicate, '') || ' ' ||
|
|
33883
|
+
coalesce(commons_claims.predicate_label, '') || ' ' ||
|
|
33884
|
+
coalesce(commons_claims.object_url, ''), ' ')
|
|
33885
|
+
FROM commons_claims
|
|
33886
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
33887
|
+
) AS claim_text
|
|
33647
33888
|
FROM commons_entities
|
|
33648
33889
|
WHERE ${clauses.join(" AND ")}
|
|
33649
33890
|
ORDER BY updated_at DESC, title ASC
|
|
33650
|
-
LIMIT
|
|
33891
|
+
LIMIT ${SEARCH_CANDIDATE_LIMIT}
|
|
33651
33892
|
`,
|
|
33652
33893
|
args
|
|
33653
33894
|
});
|
|
33654
|
-
|
|
33655
|
-
|
|
33656
|
-
|
|
33657
|
-
|
|
33658
|
-
|
|
33659
|
-
|
|
33895
|
+
const ranked = candidates.rows.map((row) => {
|
|
33896
|
+
const record = row;
|
|
33897
|
+
const score = relevanceScore(terms, {
|
|
33898
|
+
title: String(record.title ?? ""),
|
|
33899
|
+
description: String(record.description ?? ""),
|
|
33900
|
+
tags: parseJsonArray(record.tags_json),
|
|
33901
|
+
keywords: parseJsonArray(record.keywords_json),
|
|
33902
|
+
searchText: String(record.search_text ?? ""),
|
|
33903
|
+
claimText: String(record.claim_text ?? "")
|
|
33904
|
+
});
|
|
33905
|
+
return { id: String(record.id), updatedAt: String(record.updated_at ?? ""), title: String(record.title ?? ""), score };
|
|
33906
|
+
}).filter((candidate) => candidate.score > 0).sort((left, right) => right.score - left.score || right.updatedAt.localeCompare(left.updatedAt) || left.title.localeCompare(right.title));
|
|
33907
|
+
const pageIds = ranked.slice(offset, offset + limit).map((candidate) => candidate.id);
|
|
33908
|
+
if (!pageIds.length) return { results: [], filters: mergedFilters, limit, offset };
|
|
33909
|
+
const page = await getDb().execute({
|
|
33910
|
+
sql: `SELECT * FROM commons_entities WHERE id IN (${pageIds.map(() => "?").join(", ")})`,
|
|
33911
|
+
args: pageIds
|
|
33912
|
+
});
|
|
33913
|
+
const byId = new Map(page.rows.map((row) => {
|
|
33914
|
+
const entity = rowToEntity(row);
|
|
33915
|
+
return [entity.id, entity];
|
|
33916
|
+
}));
|
|
33917
|
+
const results = pageIds.map((id) => byId.get(id)).filter((entity) => Boolean(entity));
|
|
33918
|
+
await resolveRelatedLinksAgainstPublishedEntities(results);
|
|
33919
|
+
return { results, filters: mergedFilters, limit, offset };
|
|
33920
|
+
}
|
|
33921
|
+
var SEARCH_CANDIDATE_LIMIT = 500;
|
|
33922
|
+
function parseJsonArray(value) {
|
|
33923
|
+
try {
|
|
33924
|
+
const parsed = JSON.parse(String(value ?? "[]"));
|
|
33925
|
+
return Array.isArray(parsed) ? parsed.map((item) => String(item)) : [];
|
|
33926
|
+
} catch {
|
|
33927
|
+
return [];
|
|
33928
|
+
}
|
|
33929
|
+
}
|
|
33930
|
+
function containsWord(haystack, term) {
|
|
33931
|
+
if (!haystack || !term) return false;
|
|
33932
|
+
const pattern = new RegExp(`(^|[^a-z0-9])${term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}([^a-z0-9]|$)`, "i");
|
|
33933
|
+
return pattern.test(haystack);
|
|
33934
|
+
}
|
|
33935
|
+
function relevanceScore(terms, fields) {
|
|
33936
|
+
let score = 0;
|
|
33937
|
+
const tagText = fields.tags.join(" ");
|
|
33938
|
+
const keywordText = fields.keywords.join(" ");
|
|
33939
|
+
for (const term of terms) {
|
|
33940
|
+
const inTitle = containsWord(fields.title, term);
|
|
33941
|
+
const inTags = containsWord(tagText, term) || containsWord(keywordText, term);
|
|
33942
|
+
const inDescription = containsWord(fields.description, term);
|
|
33943
|
+
const inBody = containsWord(fields.searchText, term);
|
|
33944
|
+
const inClaims = containsWord(fields.claimText, term);
|
|
33945
|
+
if (!inTitle && !inTags && !inDescription && !inBody && !inClaims) return 0;
|
|
33946
|
+
if (inTitle) score += 100;
|
|
33947
|
+
if (inTags) score += 40;
|
|
33948
|
+
if (inDescription) score += 20;
|
|
33949
|
+
if (inClaims) score += 10;
|
|
33950
|
+
if (inBody) score += 5;
|
|
33951
|
+
}
|
|
33952
|
+
if (terms.every((term) => containsWord(fields.title, term))) score += 50;
|
|
33953
|
+
return score;
|
|
33660
33954
|
}
|
|
33661
33955
|
async function prepareCommonsEntity(input, user) {
|
|
33662
33956
|
await ensureCommonsSchema();
|
|
@@ -33803,7 +34097,10 @@ async function getCommonsEntity(idOrSlug) {
|
|
|
33803
34097
|
`,
|
|
33804
34098
|
args: [lookup, normalizeSlug(lookup)]
|
|
33805
34099
|
});
|
|
33806
|
-
|
|
34100
|
+
if (!result.rows[0]) return null;
|
|
34101
|
+
const entity = rowToEntity(result.rows[0]);
|
|
34102
|
+
await resolveRelatedLinksAgainstPublishedEntities([entity]);
|
|
34103
|
+
return entity;
|
|
33807
34104
|
}
|
|
33808
34105
|
async function classifyCommonsWrite(input, user) {
|
|
33809
34106
|
await ensureCommonsSchema();
|
|
@@ -33826,9 +34123,25 @@ async function submitCommonsEntity(input, user) {
|
|
|
33826
34123
|
applied: existingProposal.status === "accepted",
|
|
33827
34124
|
reviewReason: existingProposal.reviewReason,
|
|
33828
34125
|
conflictNote: null,
|
|
34126
|
+
droppedImages: [],
|
|
33829
34127
|
publicUrl: entity2?.status === "published" ? publicUrlForSlug(entity2.slug) : null
|
|
33830
34128
|
};
|
|
33831
34129
|
}
|
|
34130
|
+
const hostedImages = await hostEntityImages({
|
|
34131
|
+
userId: Number(user.id),
|
|
34132
|
+
featuredImage: normalized.featuredImage,
|
|
34133
|
+
media: normalized.media
|
|
34134
|
+
});
|
|
34135
|
+
if (hostedImages.featuredImageUrl && normalized.featuredImage) {
|
|
34136
|
+
normalized.featuredImage = { ...normalized.featuredImage, url: hostedImages.featuredImageUrl };
|
|
34137
|
+
}
|
|
34138
|
+
if (hostedImages.mediaUrls.size || hostedImages.dropped.length) {
|
|
34139
|
+
const droppedUrls = new Set(hostedImages.dropped.map((item) => item.url));
|
|
34140
|
+
normalized.media = normalized.media.filter((item) => !droppedUrls.has(item.url)).map((item) => {
|
|
34141
|
+
const hostedUrl = hostedImages.mediaUrls.get(item.url);
|
|
34142
|
+
return hostedUrl ? { ...item, url: hostedUrl } : item;
|
|
34143
|
+
});
|
|
34144
|
+
}
|
|
33832
34145
|
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
33833
34146
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
33834
34147
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
@@ -33923,6 +34236,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
33923
34236
|
applied: shouldApply,
|
|
33924
34237
|
reviewReason: safety.reason,
|
|
33925
34238
|
conflictNote: safety.conflictNote,
|
|
34239
|
+
droppedImages: hostedImages.dropped,
|
|
33926
34240
|
publicUrl: entity?.status === "published" && shouldApply ? publicUrlForSlug(entity.slug) : null
|
|
33927
34241
|
};
|
|
33928
34242
|
}
|
|
@@ -34061,8 +34375,9 @@ async function listCommonsNeedsLinks(filters = {}) {
|
|
|
34061
34375
|
});
|
|
34062
34376
|
const concepts = /* @__PURE__ */ new Map();
|
|
34063
34377
|
const queryTerms = tokenize(cleanFilters.query);
|
|
34064
|
-
|
|
34065
|
-
|
|
34378
|
+
const scanned = result.rows.map((row) => rowToEntity(row));
|
|
34379
|
+
await resolveRelatedLinksAgainstPublishedEntities(scanned);
|
|
34380
|
+
for (const entity of scanned) {
|
|
34066
34381
|
for (const link of entity.articleSections.relatedLinks) {
|
|
34067
34382
|
if (!relatedLinkNeedsPage(link)) continue;
|
|
34068
34383
|
const haystack = [link.title, link.summary, link.relationship, entity.title, entity.description, entity.tags.join(" ")].filter(Boolean).join(" ").toLowerCase();
|
|
@@ -34770,7 +35085,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
34770
35085
|
return documents;
|
|
34771
35086
|
}
|
|
34772
35087
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
34773
|
-
return `commons-index-${
|
|
35088
|
+
return `commons-index-${createHash15("sha256").update(`${entityId}
|
|
34774
35089
|
${documentType}
|
|
34775
35090
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
34776
35091
|
}
|
|
@@ -35105,6 +35420,44 @@ function sanitizeNeedsLinkFilters(filters) {
|
|
|
35105
35420
|
offset: clampNumber(filters.offset, 0, 1e4, 0)
|
|
35106
35421
|
};
|
|
35107
35422
|
}
|
|
35423
|
+
async function publishedEntityIndexFor(titles) {
|
|
35424
|
+
const index = /* @__PURE__ */ new Map();
|
|
35425
|
+
const slugKeys = uniqueStrings(titles.map((title) => normalizeNeedsLinkTitle(title))).filter(Boolean);
|
|
35426
|
+
const titleKeys = uniqueStrings(titles.map((title) => String(title || "").trim().toLowerCase())).filter(Boolean);
|
|
35427
|
+
if (!slugKeys.length && !titleKeys.length) return index;
|
|
35428
|
+
const result = await getDb().execute({
|
|
35429
|
+
sql: `
|
|
35430
|
+
SELECT id, slug, title
|
|
35431
|
+
FROM commons_entities
|
|
35432
|
+
WHERE status = 'published'
|
|
35433
|
+
AND (slug IN (${slugKeys.map(() => "?").join(", ") || "NULL"})
|
|
35434
|
+
OR lower(title) IN (${titleKeys.map(() => "?").join(", ") || "NULL"}))
|
|
35435
|
+
`,
|
|
35436
|
+
args: [...slugKeys, ...titleKeys]
|
|
35437
|
+
});
|
|
35438
|
+
for (const row of result.rows) {
|
|
35439
|
+
const record = row;
|
|
35440
|
+
const entry = { id: String(record.id), slug: String(record.slug), title: String(record.title) };
|
|
35441
|
+
index.set(entry.slug, entry);
|
|
35442
|
+
index.set(normalizeNeedsLinkTitle(entry.title), entry);
|
|
35443
|
+
}
|
|
35444
|
+
return index;
|
|
35445
|
+
}
|
|
35446
|
+
function unresolvedRelatedLinkTitles(entities) {
|
|
35447
|
+
return entities.flatMap((entity) => entity.articleSections.relatedLinks.filter((link) => relatedLinkNeedsPage(link)).map((link) => link.title));
|
|
35448
|
+
}
|
|
35449
|
+
async function resolveRelatedLinksAgainstPublishedEntities(entities) {
|
|
35450
|
+
const index = await publishedEntityIndexFor(unresolvedRelatedLinkTitles(entities));
|
|
35451
|
+
if (!index.size) return;
|
|
35452
|
+
for (const entity of entities) {
|
|
35453
|
+
entity.articleSections.relatedLinks = entity.articleSections.relatedLinks.map((link) => {
|
|
35454
|
+
if (!relatedLinkNeedsPage(link)) return link;
|
|
35455
|
+
const match = index.get(normalizeNeedsLinkTitle(link.title));
|
|
35456
|
+
if (!match || match.id === entity.id) return link;
|
|
35457
|
+
return { ...link, slug: match.slug, entityId: match.id, needsLink: false };
|
|
35458
|
+
});
|
|
35459
|
+
}
|
|
35460
|
+
}
|
|
35108
35461
|
function relatedLinkNeedsPage(link) {
|
|
35109
35462
|
if (link.needsLink === true) return true;
|
|
35110
35463
|
if (link.entityId || link.slug) return false;
|
|
@@ -35309,7 +35662,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
35309
35662
|
};
|
|
35310
35663
|
|
|
35311
35664
|
// src/api/commons-publication-repository.ts
|
|
35312
|
-
import { createHash as
|
|
35665
|
+
import { createHash as createHash16, randomUUID as randomUUID18 } from "crypto";
|
|
35313
35666
|
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
35314
35667
|
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
35315
35668
|
"admin",
|
|
@@ -35507,7 +35860,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
35507
35860
|
JSON.stringify(editionInput.articles),
|
|
35508
35861
|
html,
|
|
35509
35862
|
rendered.filename,
|
|
35510
|
-
|
|
35863
|
+
createHash16("sha256").update(html).digest("hex"),
|
|
35511
35864
|
rendered.articleCount,
|
|
35512
35865
|
rendered.wordCount,
|
|
35513
35866
|
Buffer.byteLength(html),
|
|
@@ -35944,6 +36297,28 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
35944
36297
|
return publicationError(c, error);
|
|
35945
36298
|
}
|
|
35946
36299
|
});
|
|
36300
|
+
var HostImageSchema = z32.object({
|
|
36301
|
+
sourceUrl: z32.string().url().optional(),
|
|
36302
|
+
imageBase64: z32.string().min(16).optional(),
|
|
36303
|
+
alt: z32.string().max(500).optional(),
|
|
36304
|
+
license: z32.string().max(240).optional(),
|
|
36305
|
+
attribution: z32.string().max(500).optional()
|
|
36306
|
+
}).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
|
|
36307
|
+
message: "Provide either sourceUrl or imageBase64."
|
|
36308
|
+
});
|
|
36309
|
+
commonsApp.post("/images", auth2, async (c) => {
|
|
36310
|
+
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
36311
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
36312
|
+
try {
|
|
36313
|
+
const hosted = await hostCommonsImage({ ...parsed.data, userId: Number(c.get("user").id) });
|
|
36314
|
+
return c.json({ ok: true, data: hosted }, hosted.reused ? 200 : 201);
|
|
36315
|
+
} catch (error) {
|
|
36316
|
+
if (error instanceof CommonsImageError) {
|
|
36317
|
+
return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
|
|
36318
|
+
}
|
|
36319
|
+
throw error;
|
|
36320
|
+
}
|
|
36321
|
+
});
|
|
35947
36322
|
commonsApp.get("/entities", async (c) => {
|
|
35948
36323
|
const result = await searchCommonsEntities(filtersFromQuery(c.req.query()));
|
|
35949
36324
|
return c.json({ ok: true, data: result });
|
|
@@ -36156,12 +36531,12 @@ function xmlEscape(value) {
|
|
|
36156
36531
|
import { Hono as Hono32 } from "hono";
|
|
36157
36532
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
36158
36533
|
import { z as z34 } from "zod";
|
|
36159
|
-
import { createHash as
|
|
36534
|
+
import { createHash as createHash18 } from "crypto";
|
|
36160
36535
|
import Papa2 from "papaparse";
|
|
36161
36536
|
|
|
36162
36537
|
// src/api/analytics-entitlement.ts
|
|
36163
36538
|
import {
|
|
36164
|
-
createHash as
|
|
36539
|
+
createHash as createHash17,
|
|
36165
36540
|
createHmac as createHmac3,
|
|
36166
36541
|
randomBytes as randomBytes4,
|
|
36167
36542
|
timingSafeEqual as timingSafeEqual2
|
|
@@ -36376,7 +36751,7 @@ function dashboardCallbackUrl() {
|
|
|
36376
36751
|
async function createThorbitConnectUrl(userId) {
|
|
36377
36752
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
36378
36753
|
const state = randomBytes4(32).toString("base64url");
|
|
36379
|
-
const stateHash =
|
|
36754
|
+
const stateHash = createHash17("sha256").update(state).digest("hex");
|
|
36380
36755
|
await getAnalyticsPool().query(
|
|
36381
36756
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
36382
36757
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -36405,7 +36780,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
36405
36780
|
return parsed.data.entitlement;
|
|
36406
36781
|
}
|
|
36407
36782
|
async function consumeThorbitConnectCallback(input) {
|
|
36408
|
-
const stateHash =
|
|
36783
|
+
const stateHash = createHash17("sha256").update(input.state).digest("hex");
|
|
36409
36784
|
const client2 = await getAnalyticsPool().connect();
|
|
36410
36785
|
try {
|
|
36411
36786
|
await client2.query("BEGIN");
|
|
@@ -36881,7 +37256,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
|
|
|
36881
37256
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
36882
37257
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
36883
37258
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
36884
|
-
const suffix2 =
|
|
37259
|
+
const suffix2 = createHash18("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
36885
37260
|
const path5 = `Leads/person-${suffix2}`;
|
|
36886
37261
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
36887
37262
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -36980,9 +37355,9 @@ Submitted the published form.
|
|
|
36980
37355
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
36981
37356
|
});
|
|
36982
37357
|
const match = {
|
|
36983
|
-
...email ? { emailSha256:
|
|
37358
|
+
...email ? { emailSha256: createHash18("sha256").update(email).digest("hex") } : {},
|
|
36984
37359
|
...phone ? {
|
|
36985
|
-
phoneSha256:
|
|
37360
|
+
phoneSha256: createHash18("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
36986
37361
|
} : {}
|
|
36987
37362
|
};
|
|
36988
37363
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -37410,7 +37785,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
|
|
|
37410
37785
|
rejectedCount += 1;
|
|
37411
37786
|
continue;
|
|
37412
37787
|
}
|
|
37413
|
-
const digest2 =
|
|
37788
|
+
const digest2 = createHash18("sha256").update(
|
|
37414
37789
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
37415
37790
|
).digest("hex").slice(0, 16);
|
|
37416
37791
|
const path5 = `Leads/person-${digest2}`;
|
|
@@ -37861,13 +38236,13 @@ import { Hono as Hono33 } from "hono";
|
|
|
37861
38236
|
import { z as z35 } from "zod";
|
|
37862
38237
|
|
|
37863
38238
|
// src/api/scheduled-artifact-owner.ts
|
|
37864
|
-
import { createHash as
|
|
38239
|
+
import { createHash as createHash19 } from "crypto";
|
|
37865
38240
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
37866
|
-
return
|
|
38241
|
+
return createHash19("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
37867
38242
|
}
|
|
37868
38243
|
|
|
37869
38244
|
// src/api/scheduled-run-view-links.ts
|
|
37870
|
-
import { createHash as
|
|
38245
|
+
import { createHash as createHash20, randomBytes as randomBytes5, randomUUID as randomUUID19 } from "crypto";
|
|
37871
38246
|
var schemaReady2 = false;
|
|
37872
38247
|
async function ensureScheduledRunViewLinksSchema() {
|
|
37873
38248
|
if (schemaReady2) return;
|
|
@@ -37921,7 +38296,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
37921
38296
|
schemaReady2 = true;
|
|
37922
38297
|
}
|
|
37923
38298
|
function tokenHash2(token6) {
|
|
37924
|
-
return
|
|
38299
|
+
return createHash20("sha256").update(token6).digest("hex");
|
|
37925
38300
|
}
|
|
37926
38301
|
function mapRow(row) {
|
|
37927
38302
|
return {
|
|
@@ -38008,7 +38383,7 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
|
|
|
38008
38383
|
}
|
|
38009
38384
|
|
|
38010
38385
|
// src/personal-authority/render.ts
|
|
38011
|
-
import { createHash as
|
|
38386
|
+
import { createHash as createHash21 } from "crypto";
|
|
38012
38387
|
function escapeHtml7(value) {
|
|
38013
38388
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38014
38389
|
"&": "&",
|
|
@@ -38068,13 +38443,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
38068
38443
|
html,
|
|
38069
38444
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38070
38445
|
bytes,
|
|
38071
|
-
sha256:
|
|
38446
|
+
sha256: createHash21("sha256").update(html).digest("hex"),
|
|
38072
38447
|
generatedAt: generatedAtIso
|
|
38073
38448
|
};
|
|
38074
38449
|
}
|
|
38075
38450
|
|
|
38076
38451
|
// src/personal-authority/render-v2.ts
|
|
38077
|
-
import { createHash as
|
|
38452
|
+
import { createHash as createHash22 } from "crypto";
|
|
38078
38453
|
var FONT_STACKS = Object.freeze({
|
|
38079
38454
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
38080
38455
|
"modern-sans": "Avenir Next, Avenir, Helvetica Neue, Arial, sans-serif",
|
|
@@ -38178,13 +38553,13 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
38178
38553
|
html,
|
|
38179
38554
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38180
38555
|
bytes,
|
|
38181
|
-
sha256:
|
|
38556
|
+
sha256: createHash22("sha256").update(html).digest("hex"),
|
|
38182
38557
|
generatedAt: generatedAtIso
|
|
38183
38558
|
};
|
|
38184
38559
|
}
|
|
38185
38560
|
|
|
38186
38561
|
// src/newsroom-publisher/render.ts
|
|
38187
|
-
import { createHash as
|
|
38562
|
+
import { createHash as createHash23 } from "crypto";
|
|
38188
38563
|
function escapeHtml9(value) {
|
|
38189
38564
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38190
38565
|
"&": "&",
|
|
@@ -38290,13 +38665,13 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
38290
38665
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
38291
38666
|
</style></head><body data-theme="${escapeHtml9(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
38292
38667
|
const bytes = Buffer.byteLength(html);
|
|
38293
|
-
const sha2563 =
|
|
38668
|
+
const sha2563 = createHash23("sha256").update(html).digest("hex");
|
|
38294
38669
|
const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
38295
38670
|
return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
|
|
38296
38671
|
}
|
|
38297
38672
|
|
|
38298
38673
|
// src/blog-article/render.ts
|
|
38299
|
-
import { createHash as
|
|
38674
|
+
import { createHash as createHash24 } from "crypto";
|
|
38300
38675
|
function escapeHtml10(value) {
|
|
38301
38676
|
return value.replace(
|
|
38302
38677
|
/[&<>"']/g,
|
|
@@ -38384,7 +38759,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
38384
38759
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
38385
38760
|
</style></head><body data-theme="${escapeHtml10(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml10(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml10(input.eyebrow)}</span><h1>${escapeHtml10(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml10(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml10(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml10(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml10(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
38386
38761
|
const bytes = Buffer.byteLength(html);
|
|
38387
|
-
const sha2563 =
|
|
38762
|
+
const sha2563 = createHash24("sha256").update(html).digest("hex");
|
|
38388
38763
|
return {
|
|
38389
38764
|
html,
|
|
38390
38765
|
filename: "blog-article.html",
|
|
@@ -38687,7 +39062,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
38687
39062
|
import { Hono as Hono34 } from "hono";
|
|
38688
39063
|
|
|
38689
39064
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
38690
|
-
import { createHash as
|
|
39065
|
+
import { createHash as createHash25 } from "crypto";
|
|
38691
39066
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
38692
39067
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
38693
39068
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -38703,7 +39078,7 @@ function policy4() {
|
|
|
38703
39078
|
};
|
|
38704
39079
|
}
|
|
38705
39080
|
function runStorageSegment(runId) {
|
|
38706
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
39081
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash25("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
38707
39082
|
}
|
|
38708
39083
|
async function createScheduledRunArtifact(args) {
|
|
38709
39084
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -38810,7 +39185,7 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
38810
39185
|
import { Hono as Hono35 } from "hono";
|
|
38811
39186
|
|
|
38812
39187
|
// src/api/scheduler-integration-auth.ts
|
|
38813
|
-
import { createHash as
|
|
39188
|
+
import { createHash as createHash26, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
38814
39189
|
|
|
38815
39190
|
// src/api/service-connections.ts
|
|
38816
39191
|
import { randomUUID as randomUUID20 } from "crypto";
|
|
@@ -39108,7 +39483,7 @@ function signingSecret() {
|
|
|
39108
39483
|
return secret;
|
|
39109
39484
|
}
|
|
39110
39485
|
function schedulerIntegrationSignature(args) {
|
|
39111
|
-
const bodyHash =
|
|
39486
|
+
const bodyHash = createHash26("sha256").update(args.body).digest("hex");
|
|
39112
39487
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
39113
39488
|
${args.path}
|
|
39114
39489
|
${args.timestamp}
|
|
@@ -39300,7 +39675,7 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
39300
39675
|
}
|
|
39301
39676
|
|
|
39302
39677
|
// src/api/page-media-artifacts.ts
|
|
39303
|
-
import { createHash as
|
|
39678
|
+
import { createHash as createHash27, randomBytes as randomBytes6 } from "crypto";
|
|
39304
39679
|
import { homedir as homedir3 } from "os";
|
|
39305
39680
|
import { basename, extname, join as join8 } from "path";
|
|
39306
39681
|
import pLimit6 from "p-limit";
|
|
@@ -39447,7 +39822,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
39447
39822
|
finalUrl: checked.parsed.href,
|
|
39448
39823
|
width: dimensions?.width ?? null,
|
|
39449
39824
|
height: dimensions?.height ?? null,
|
|
39450
|
-
sha256:
|
|
39825
|
+
sha256: createHash27("sha256").update(bytes).digest("hex")
|
|
39451
39826
|
};
|
|
39452
39827
|
}
|
|
39453
39828
|
throw new Error("media_redirect_rejected");
|
|
@@ -39640,13 +40015,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
39640
40015
|
}
|
|
39641
40016
|
|
|
39642
40017
|
// src/api/page-diff.ts
|
|
39643
|
-
import { createHash as
|
|
40018
|
+
import { createHash as createHash28 } from "crypto";
|
|
39644
40019
|
import { diffLines } from "diff";
|
|
39645
40020
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
39646
40021
|
var MAX_DIFF_HUNKS = 200;
|
|
39647
40022
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
39648
40023
|
function sha256Hex(value) {
|
|
39649
|
-
return
|
|
40024
|
+
return createHash28("sha256").update(value).digest("hex");
|
|
39650
40025
|
}
|
|
39651
40026
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
39652
40027
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -39760,10 +40135,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
39760
40135
|
}
|
|
39761
40136
|
|
|
39762
40137
|
// src/api/scrape-image-sink.ts
|
|
39763
|
-
import { createHash as
|
|
40138
|
+
import { createHash as createHash29 } from "crypto";
|
|
39764
40139
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
39765
40140
|
function idempotencyKey2(userId, vault, input) {
|
|
39766
|
-
return `scrape-image-${
|
|
40141
|
+
return `scrape-image-${createHash29("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
39767
40142
|
}
|
|
39768
40143
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
39769
40144
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -40235,7 +40610,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
40235
40610
|
}
|
|
40236
40611
|
|
|
40237
40612
|
// src/api/connection-memory-import.ts
|
|
40238
|
-
import { createHash as
|
|
40613
|
+
import { createHash as createHash30 } from "crypto";
|
|
40239
40614
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
40240
40615
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
40241
40616
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -40265,7 +40640,7 @@ function canonicalJson(value) {
|
|
|
40265
40640
|
return JSON.stringify(value);
|
|
40266
40641
|
}
|
|
40267
40642
|
function sha2562(value) {
|
|
40268
|
-
return
|
|
40643
|
+
return createHash30("sha256").update(value).digest("hex");
|
|
40269
40644
|
}
|
|
40270
40645
|
function sensitiveKey(key) {
|
|
40271
40646
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -41276,7 +41651,7 @@ async function dbUsage(identity, plan) {
|
|
|
41276
41651
|
}
|
|
41277
41652
|
|
|
41278
41653
|
// src/api/nango-control.ts
|
|
41279
|
-
import { createHash as
|
|
41654
|
+
import { createHash as createHash32, randomUUID as randomUUID24 } from "crypto";
|
|
41280
41655
|
|
|
41281
41656
|
// src/api/slack-archive-analysis.ts
|
|
41282
41657
|
function isRecord2(value) {
|
|
@@ -41650,7 +42025,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
41650
42025
|
}
|
|
41651
42026
|
|
|
41652
42027
|
// src/api/main-nango-transport.ts
|
|
41653
|
-
import { createHash as
|
|
42028
|
+
import { createHash as createHash31, randomUUID as randomUUID23 } from "crypto";
|
|
41654
42029
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
41655
42030
|
|
|
41656
42031
|
// src/api/connected-tool-policy.ts
|
|
@@ -42043,7 +42418,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
42043
42418
|
});
|
|
42044
42419
|
const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
|
|
42045
42420
|
const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
|
|
42046
|
-
const revision =
|
|
42421
|
+
const revision = createHash31("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
42047
42422
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
42048
42423
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
42049
42424
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -42071,7 +42446,7 @@ async function callNangoToolDirect(args) {
|
|
|
42071
42446
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
42072
42447
|
});
|
|
42073
42448
|
const requestId = args.requestId?.trim() || randomUUID23();
|
|
42074
|
-
const idempotencyKey3 = `main-nango:${
|
|
42449
|
+
const idempotencyKey3 = `main-nango:${createHash31("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
42075
42450
|
const startedAt = /* @__PURE__ */ new Date();
|
|
42076
42451
|
const started = performance.now();
|
|
42077
42452
|
let result;
|
|
@@ -42096,7 +42471,7 @@ async function callNangoToolDirect(args) {
|
|
|
42096
42471
|
toolName: args.tool,
|
|
42097
42472
|
operationKind: args.operationKind ?? args.classification,
|
|
42098
42473
|
outcome: providerError ? "error" : "partial",
|
|
42099
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
42474
|
+
requestId: requestId.length <= 200 ? requestId : createHash31("sha256").update(requestId).digest("hex"),
|
|
42100
42475
|
startedAt: startedAt.toISOString(),
|
|
42101
42476
|
completedAt: completedAt.toISOString()
|
|
42102
42477
|
}
|
|
@@ -42138,7 +42513,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
42138
42513
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
42139
42514
|
protocolVersion: null,
|
|
42140
42515
|
schemaSource: "live_tools_list",
|
|
42141
|
-
schemaHash:
|
|
42516
|
+
schemaHash: createHash31("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
42142
42517
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
42143
42518
|
};
|
|
42144
42519
|
}
|
|
@@ -42862,8 +43237,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
42862
43237
|
return data.connection.actionsEnabled === true;
|
|
42863
43238
|
}
|
|
42864
43239
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
42865
|
-
const requestId = `main-connected-action:${
|
|
42866
|
-
const requestDigest =
|
|
43240
|
+
const requestId = `main-connected-action:${createHash32("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID24()).digest("hex")}`;
|
|
43241
|
+
const requestDigest = createHash32("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
42867
43242
|
if (mainOwnsIntegrations()) {
|
|
42868
43243
|
const selectedTool = tool?.trim();
|
|
42869
43244
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -43014,7 +43389,7 @@ function canonicalJson2(value) {
|
|
|
43014
43389
|
return JSON.stringify(value);
|
|
43015
43390
|
}
|
|
43016
43391
|
function projectedToolSchemaHash(tool) {
|
|
43017
|
-
return
|
|
43392
|
+
return createHash32("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
43018
43393
|
}
|
|
43019
43394
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
43020
43395
|
if (mainOwnsIntegrations()) {
|
|
@@ -43293,7 +43668,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
43293
43668
|
}
|
|
43294
43669
|
|
|
43295
43670
|
// src/api/resend-control.ts
|
|
43296
|
-
import { createHash as
|
|
43671
|
+
import { createHash as createHash33 } from "crypto";
|
|
43297
43672
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
43298
43673
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
43299
43674
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -43552,7 +43927,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
43552
43927
|
return isRecord5(data) ? data.result ?? data : data;
|
|
43553
43928
|
}
|
|
43554
43929
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
43555
|
-
const requestId = `main-resend-action:${
|
|
43930
|
+
const requestId = `main-resend-action:${createHash33("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
43556
43931
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
43557
43932
|
method: "POST",
|
|
43558
43933
|
headers: { "x-request-id": requestId },
|
|
@@ -46778,7 +47153,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
46778
47153
|
return c.json({ error: "Unauthorized" }, 401);
|
|
46779
47154
|
}
|
|
46780
47155
|
const startedAt = Date.now();
|
|
46781
|
-
const { drainQueue } = await import("./worker-
|
|
47156
|
+
const { drainQueue } = await import("./worker-S2WXELUC.js");
|
|
46782
47157
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
46783
47158
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
46784
47159
|
const unfinished = [];
|
|
@@ -46848,8 +47223,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46848
47223
|
finishExtractJob: finishExtractJob2,
|
|
46849
47224
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
46850
47225
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
46851
|
-
} = await import("./site-extract-repository-
|
|
46852
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
47226
|
+
} = await import("./site-extract-repository-CFMSWAVL.js");
|
|
47227
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
46853
47228
|
const existing = await getExtractJob2(jobId2);
|
|
46854
47229
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
46855
47230
|
if (existing.status !== "failed") {
|
|
@@ -46876,7 +47251,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46876
47251
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
46877
47252
|
let settlement = "already_settled_or_refunded";
|
|
46878
47253
|
if (job.billedMc == null && job.userId != null) {
|
|
46879
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
47254
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CFMSWAVL.js");
|
|
46880
47255
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
46881
47256
|
const successful = await countSuccessfulPages2(jobId2);
|
|
46882
47257
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -47061,4 +47436,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
47061
47436
|
export {
|
|
47062
47437
|
app
|
|
47063
47438
|
};
|
|
47064
|
-
//# sourceMappingURL=server-
|
|
47439
|
+
//# sourceMappingURL=server-NBGZJX4P.js.map
|