mcp-scraper 0.56.0 → 0.57.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +576 -156
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +25 -2
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-JEZW6N2X.js → chunk-2KL4MSZP.js} +26 -3
- package/dist/chunk-2KL4MSZP.js.map +1 -0
- package/dist/{chunk-NP66MZ5A.js → chunk-5XYBKE3X.js} +2 -2
- package/dist/{chunk-LUJYVNO7.js → chunk-HU4R6BP2.js} +2 -2
- package/dist/{chunk-LUJYVNO7.js.map → chunk-HU4R6BP2.js.map} +1 -1
- package/dist/{chunk-WO2SSENO.js → chunk-R6CO77BN.js} +3 -3
- package/dist/{chunk-WO2SSENO.js.map → chunk-R6CO77BN.js.map} +1 -1
- package/dist/chunk-VTBJVKBE.js +7 -0
- package/dist/chunk-VTBJVKBE.js.map +1 -0
- package/dist/{extract-bundle-6KYTD55T.js → extract-bundle-H7JEELY7.js} +3 -3
- package/dist/{server-SETYJPYO.js → server-DSCKFM7K.js} +459 -72
- package/dist/server-DSCKFM7K.js.map +1 -0
- package/dist/{site-extract-repository-YGNYFTNE.js → site-extract-repository-CFMSWAVL.js} +3 -3
- package/dist/{worker-DTVBN3RQ.js → worker-S2WXELUC.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-245IAJ7R.js +0 -7
- package/dist/chunk-245IAJ7R.js.map +0 -1
- package/dist/chunk-JEZW6N2X.js.map +0 -1
- package/dist/server-SETYJPYO.js.map +0 -1
- /package/dist/{chunk-NP66MZ5A.js.map → chunk-5XYBKE3X.js.map} +0 -0
- /package/dist/{extract-bundle-6KYTD55T.js.map → extract-bundle-H7JEELY7.js.map} +0 -0
- /package/dist/{site-extract-repository-YGNYFTNE.js.map → site-extract-repository-CFMSWAVL.js.map} +0 -0
- /package/dist/{worker-DTVBN3RQ.js.map → worker-S2WXELUC.js.map} +0 -0
|
@@ -169,7 +169,7 @@ import {
|
|
|
169
169
|
resolveDeploymentProfile,
|
|
170
170
|
resolveLocalSourcebookSchemaType,
|
|
171
171
|
transcribeMediaUrl
|
|
172
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-2KL4MSZP.js";
|
|
173
173
|
import {
|
|
174
174
|
auditImageUrls,
|
|
175
175
|
auditImages,
|
|
@@ -225,7 +225,7 @@ import {
|
|
|
225
225
|
} from "./chunk-FG6LJGJ7.js";
|
|
226
226
|
import {
|
|
227
227
|
PACKAGE_VERSION
|
|
228
|
-
} from "./chunk-
|
|
228
|
+
} from "./chunk-VTBJVKBE.js";
|
|
229
229
|
import {
|
|
230
230
|
abandonExtractSettlement,
|
|
231
231
|
countSuccessfulPages,
|
|
@@ -248,7 +248,7 @@ import {
|
|
|
248
248
|
setExtractJobTotal,
|
|
249
249
|
settleExtractJob,
|
|
250
250
|
terminalExtractJobStatus
|
|
251
|
-
} from "./chunk-
|
|
251
|
+
} from "./chunk-5XYBKE3X.js";
|
|
252
252
|
import {
|
|
253
253
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
254
254
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -301,7 +301,7 @@ import {
|
|
|
301
301
|
tierChangeLeavesPrivateOffer,
|
|
302
302
|
tierCreditsPerInterval,
|
|
303
303
|
vendorCostUsdToMc
|
|
304
|
-
} from "./chunk-
|
|
304
|
+
} from "./chunk-R6CO77BN.js";
|
|
305
305
|
import {
|
|
306
306
|
CaptchaError,
|
|
307
307
|
MapsSearchIncompletePageError,
|
|
@@ -9393,7 +9393,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9393
9393
|
});
|
|
9394
9394
|
}) : null;
|
|
9395
9395
|
const finalized = await step.run("finalize", async () => {
|
|
9396
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9396
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
9397
9397
|
const current = await getExtractJob(jobId2);
|
|
9398
9398
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9399
9399
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -32972,7 +32972,236 @@ import { Hono as Hono31 } from "hono";
|
|
|
32972
32972
|
import { z as z32 } from "zod";
|
|
32973
32973
|
|
|
32974
32974
|
// src/api/commons-repository.ts
|
|
32975
|
-
import { createHash as
|
|
32975
|
+
import { createHash as createHash15, randomUUID as randomUUID17 } from "crypto";
|
|
32976
|
+
|
|
32977
|
+
// src/api/commons-image-store.ts
|
|
32978
|
+
import { createHash as createHash13 } from "crypto";
|
|
32979
|
+
import { lookup as dnsLookup } from "dns/promises";
|
|
32980
|
+
import { isIP } from "net";
|
|
32981
|
+
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
32982
|
+
var COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
32983
|
+
var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
32984
|
+
var CommonsImageError = class extends Error {
|
|
32985
|
+
constructor(code, message, httpStatus = 400) {
|
|
32986
|
+
super(message);
|
|
32987
|
+
this.code = code;
|
|
32988
|
+
this.httpStatus = httpStatus;
|
|
32989
|
+
this.name = "CommonsImageError";
|
|
32990
|
+
}
|
|
32991
|
+
code;
|
|
32992
|
+
httpStatus;
|
|
32993
|
+
};
|
|
32994
|
+
var imageSchemaReady = false;
|
|
32995
|
+
async function ensureCommonsImageSchema() {
|
|
32996
|
+
if (imageSchemaReady) return;
|
|
32997
|
+
await getDb().execute(`
|
|
32998
|
+
CREATE TABLE IF NOT EXISTS commons_images (
|
|
32999
|
+
id TEXT PRIMARY KEY,
|
|
33000
|
+
user_id INTEGER NOT NULL,
|
|
33001
|
+
digest TEXT NOT NULL,
|
|
33002
|
+
storage_key TEXT NOT NULL,
|
|
33003
|
+
url TEXT NOT NULL,
|
|
33004
|
+
content_type TEXT NOT NULL,
|
|
33005
|
+
bytes INTEGER NOT NULL,
|
|
33006
|
+
source_url TEXT,
|
|
33007
|
+
alt TEXT,
|
|
33008
|
+
license TEXT,
|
|
33009
|
+
attribution TEXT,
|
|
33010
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
33011
|
+
)
|
|
33012
|
+
`);
|
|
33013
|
+
await getDb().execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
|
|
33014
|
+
imageSchemaReady = true;
|
|
33015
|
+
}
|
|
33016
|
+
function sniffImageFormat(bytes) {
|
|
33017
|
+
if (bytes.length < 12) return null;
|
|
33018
|
+
if (bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return "image/jpeg";
|
|
33019
|
+
if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return "image/png";
|
|
33020
|
+
if (bytes.subarray(0, 6).toString("ascii") === "GIF87a" || bytes.subarray(0, 6).toString("ascii") === "GIF89a") return "image/gif";
|
|
33021
|
+
if (bytes.subarray(0, 4).toString("ascii") === "RIFF" && bytes.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";
|
|
33022
|
+
return null;
|
|
33023
|
+
}
|
|
33024
|
+
function isBlockedAddress(address) {
|
|
33025
|
+
if (isIP(address) === 6) {
|
|
33026
|
+
const normalized = address.toLowerCase();
|
|
33027
|
+
return normalized === "::1" || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("fe80");
|
|
33028
|
+
}
|
|
33029
|
+
const parts = address.split(".").map(Number);
|
|
33030
|
+
if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) return true;
|
|
33031
|
+
const [a, b] = parts;
|
|
33032
|
+
if (a === 10 || a === 127 || a === 0) return true;
|
|
33033
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
33034
|
+
if (a === 192 && b === 168) return true;
|
|
33035
|
+
if (a === 169 && b === 254) return true;
|
|
33036
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
33037
|
+
return false;
|
|
33038
|
+
}
|
|
33039
|
+
async function assertPublicHttpsUrl(rawUrl) {
|
|
33040
|
+
let url;
|
|
33041
|
+
try {
|
|
33042
|
+
url = new URL(rawUrl);
|
|
33043
|
+
} catch {
|
|
33044
|
+
throw new CommonsImageError("image_url_invalid", `"${rawUrl}" is not a valid URL.`);
|
|
33045
|
+
}
|
|
33046
|
+
if (url.protocol !== "https:") {
|
|
33047
|
+
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
33048
|
+
}
|
|
33049
|
+
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
33050
|
+
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
33051
|
+
if (!addresses.length) {
|
|
33052
|
+
throw new CommonsImageError("image_host_unresolvable", `Could not resolve ${url.hostname}.`);
|
|
33053
|
+
}
|
|
33054
|
+
if (addresses.some((entry) => isBlockedAddress(entry.address))) {
|
|
33055
|
+
throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
|
|
33056
|
+
}
|
|
33057
|
+
return url;
|
|
33058
|
+
}
|
|
33059
|
+
async function downloadImage(sourceUrl) {
|
|
33060
|
+
let current = sourceUrl;
|
|
33061
|
+
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
33062
|
+
const url = await assertPublicHttpsUrl(current);
|
|
33063
|
+
const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
|
|
33064
|
+
if (response.status >= 300 && response.status < 400) {
|
|
33065
|
+
const location2 = response.headers.get("location");
|
|
33066
|
+
if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
|
|
33067
|
+
current = new URL(location2, url).toString();
|
|
33068
|
+
continue;
|
|
33069
|
+
}
|
|
33070
|
+
if (!response.ok) {
|
|
33071
|
+
throw new CommonsImageError("image_source_unavailable", `Image source returned HTTP ${response.status}.`, 422);
|
|
33072
|
+
}
|
|
33073
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
33074
|
+
if (declaredLength > COMMONS_IMAGE_MAX_BYTES) {
|
|
33075
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33076
|
+
}
|
|
33077
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
33078
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33079
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33080
|
+
}
|
|
33081
|
+
return bytes;
|
|
33082
|
+
}
|
|
33083
|
+
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
33084
|
+
}
|
|
33085
|
+
function decodeBase64Image(value) {
|
|
33086
|
+
const payload = value.includes(",") && value.trim().startsWith("data:") ? value.slice(value.indexOf(",") + 1) : value;
|
|
33087
|
+
const bytes = Buffer.from(payload, "base64");
|
|
33088
|
+
if (!bytes.length) throw new CommonsImageError("image_base64_invalid", "imageBase64 did not decode to any bytes.");
|
|
33089
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33090
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33091
|
+
}
|
|
33092
|
+
return bytes;
|
|
33093
|
+
}
|
|
33094
|
+
function commonsImageBlobStore() {
|
|
33095
|
+
const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
|
|
33096
|
+
return token6 ? new VercelBlobStore(token6) : getBlobStore();
|
|
33097
|
+
}
|
|
33098
|
+
function isCommonsHostedImageUrl(url) {
|
|
33099
|
+
return /^https:\/\/[^/]*(\.public\.blob\.vercel-storage\.com|blob\.vercel-storage\.com)\//i.test(url);
|
|
33100
|
+
}
|
|
33101
|
+
var hostOverride = null;
|
|
33102
|
+
async function hostCommonsImage(input) {
|
|
33103
|
+
if (hostOverride) return hostOverride(input);
|
|
33104
|
+
await ensureCommonsImageSchema();
|
|
33105
|
+
if (!input.sourceUrl && !input.imageBase64) {
|
|
33106
|
+
throw new CommonsImageError("image_source_missing", "Provide either sourceUrl or imageBase64.");
|
|
33107
|
+
}
|
|
33108
|
+
const bytes = input.imageBase64 ? decodeBase64Image(input.imageBase64) : await downloadImage(String(input.sourceUrl));
|
|
33109
|
+
const contentType = sniffImageFormat(bytes);
|
|
33110
|
+
if (!contentType) {
|
|
33111
|
+
throw new CommonsImageError(
|
|
33112
|
+
"image_format_unsupported",
|
|
33113
|
+
"Commons hosts JPEG, PNG, GIF, and WebP. SVG and other formats are rejected because the wiki serves them to the public web.",
|
|
33114
|
+
415
|
|
33115
|
+
);
|
|
33116
|
+
}
|
|
33117
|
+
const digest2 = createHash13("sha256").update(bytes).digest("hex");
|
|
33118
|
+
const existing = await getDb().execute({
|
|
33119
|
+
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
33120
|
+
args: [digest2]
|
|
33121
|
+
});
|
|
33122
|
+
const reusable = existing.rows[0];
|
|
33123
|
+
if (reusable) {
|
|
33124
|
+
return {
|
|
33125
|
+
id: String(reusable.id),
|
|
33126
|
+
url: String(reusable.url),
|
|
33127
|
+
contentType: String(reusable.content_type),
|
|
33128
|
+
bytes: Number(reusable.bytes),
|
|
33129
|
+
digest: digest2,
|
|
33130
|
+
sourceUrl: reusable.source_url ? String(reusable.source_url) : null,
|
|
33131
|
+
reused: true
|
|
33132
|
+
};
|
|
33133
|
+
}
|
|
33134
|
+
const extension = contentType.split("/")[1] === "jpeg" ? "jpg" : contentType.split("/")[1];
|
|
33135
|
+
const storageKey = `commons/images/${digest2.slice(0, 2)}/${digest2}.${extension}`;
|
|
33136
|
+
const store = commonsImageBlobStore();
|
|
33137
|
+
if (store.kind !== "vercel-blob") {
|
|
33138
|
+
throw new CommonsImageError(
|
|
33139
|
+
"image_storage_unconfigured",
|
|
33140
|
+
"Commons image hosting needs a Vercel Blob token. Refusing to write a local file path into a public wiki entity.",
|
|
33141
|
+
422
|
|
33142
|
+
);
|
|
33143
|
+
}
|
|
33144
|
+
const stored = await store.put(storageKey, bytes, contentType);
|
|
33145
|
+
const id = `TPW-IMG-${digest2.slice(0, 16).toUpperCase()}`;
|
|
33146
|
+
await getDb().execute({
|
|
33147
|
+
sql: `
|
|
33148
|
+
INSERT INTO commons_images (id, user_id, digest, storage_key, url, content_type, bytes, source_url, alt, license, attribution, created_at)
|
|
33149
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
33150
|
+
`,
|
|
33151
|
+
args: [
|
|
33152
|
+
id,
|
|
33153
|
+
input.userId,
|
|
33154
|
+
digest2,
|
|
33155
|
+
storageKey,
|
|
33156
|
+
stored.url,
|
|
33157
|
+
contentType,
|
|
33158
|
+
bytes.length,
|
|
33159
|
+
input.sourceUrl ?? null,
|
|
33160
|
+
input.alt ?? null,
|
|
33161
|
+
input.license ?? null,
|
|
33162
|
+
input.attribution ?? null,
|
|
33163
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
33164
|
+
]
|
|
33165
|
+
});
|
|
33166
|
+
return { id, url: stored.url, contentType, bytes: bytes.length, digest: digest2, sourceUrl: input.sourceUrl ?? null, reused: false };
|
|
33167
|
+
}
|
|
33168
|
+
async function hostEntityImages(input) {
|
|
33169
|
+
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
33170
|
+
const featuredUrl = input.featuredImage?.url;
|
|
33171
|
+
if (featuredUrl && !isCommonsHostedImageUrl(featuredUrl)) {
|
|
33172
|
+
const hosted = await hostCommonsImage({
|
|
33173
|
+
userId: input.userId,
|
|
33174
|
+
sourceUrl: featuredUrl,
|
|
33175
|
+
alt: input.featuredImage?.alt,
|
|
33176
|
+
license: input.featuredImage?.license,
|
|
33177
|
+
attribution: input.featuredImage?.sourceUrl
|
|
33178
|
+
}).catch((error) => {
|
|
33179
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33180
|
+
throw new CommonsImageError(
|
|
33181
|
+
error instanceof CommonsImageError ? error.code : "featured_image_unhostable",
|
|
33182
|
+
`The featured image could not be hosted, so this entity would publish with a link that can break: ${message}`,
|
|
33183
|
+
422
|
|
33184
|
+
);
|
|
33185
|
+
});
|
|
33186
|
+
rewrite.featuredImageUrl = hosted.url;
|
|
33187
|
+
}
|
|
33188
|
+
for (const item of input.media ?? []) {
|
|
33189
|
+
if (!item.url || isCommonsHostedImageUrl(item.url)) continue;
|
|
33190
|
+
try {
|
|
33191
|
+
const hosted = await hostCommonsImage({
|
|
33192
|
+
userId: input.userId,
|
|
33193
|
+
sourceUrl: item.url,
|
|
33194
|
+
alt: item.alt,
|
|
33195
|
+
license: item.license,
|
|
33196
|
+
attribution: item.sourceUrl
|
|
33197
|
+
});
|
|
33198
|
+
rewrite.mediaUrls.set(item.url, hosted.url);
|
|
33199
|
+
} catch (error) {
|
|
33200
|
+
rewrite.dropped.push({ url: item.url, reason: error instanceof Error ? error.message : String(error) });
|
|
33201
|
+
}
|
|
33202
|
+
}
|
|
33203
|
+
return rewrite;
|
|
33204
|
+
}
|
|
32976
33205
|
|
|
32977
33206
|
// src/api/schema-presence.ts
|
|
32978
33207
|
var cache = null;
|
|
@@ -33000,7 +33229,7 @@ async function schemaObjectsPresent(names) {
|
|
|
33000
33229
|
}
|
|
33001
33230
|
|
|
33002
33231
|
// src/api/commons-linksets.ts
|
|
33003
|
-
import { createHash as
|
|
33232
|
+
import { createHash as createHash14 } from "crypto";
|
|
33004
33233
|
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
33005
33234
|
var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
33006
33235
|
var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -33107,7 +33336,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
33107
33336
|
context[claim.predicate] = targets;
|
|
33108
33337
|
}
|
|
33109
33338
|
const document2 = { linkset: [context] };
|
|
33110
|
-
const etag = `"${
|
|
33339
|
+
const etag = `"${createHash14("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
33111
33340
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
33112
33341
|
}
|
|
33113
33342
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -33640,23 +33869,100 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
33640
33869
|
const pattern = `%${escapeLike(term)}%`;
|
|
33641
33870
|
args.push(pattern, pattern, pattern, pattern, pattern);
|
|
33642
33871
|
}
|
|
33643
|
-
|
|
33644
|
-
|
|
33872
|
+
if (!terms.length) {
|
|
33873
|
+
args.push(limit, offset);
|
|
33874
|
+
const result = await getDb().execute({
|
|
33875
|
+
sql: `
|
|
33876
|
+
SELECT *
|
|
33877
|
+
FROM commons_entities
|
|
33878
|
+
WHERE ${clauses.join(" AND ")}
|
|
33879
|
+
ORDER BY updated_at DESC, title ASC
|
|
33880
|
+
LIMIT ? OFFSET ?
|
|
33881
|
+
`,
|
|
33882
|
+
args
|
|
33883
|
+
});
|
|
33884
|
+
const results2 = result.rows.map((row) => rowToEntity(row));
|
|
33885
|
+
await resolveRelatedLinksAgainstPublishedEntities(results2);
|
|
33886
|
+
return { results: results2, filters: mergedFilters, limit, offset };
|
|
33887
|
+
}
|
|
33888
|
+
const candidates = await getDb().execute({
|
|
33645
33889
|
sql: `
|
|
33646
|
-
SELECT
|
|
33890
|
+
SELECT id, title, tags_json, keywords_json, description, search_text, updated_at,
|
|
33891
|
+
(
|
|
33892
|
+
SELECT group_concat(
|
|
33893
|
+
coalesce(commons_claims.title, '') || ' ' ||
|
|
33894
|
+
coalesce(commons_claims.predicate, '') || ' ' ||
|
|
33895
|
+
coalesce(commons_claims.predicate_label, '') || ' ' ||
|
|
33896
|
+
coalesce(commons_claims.object_url, ''), ' ')
|
|
33897
|
+
FROM commons_claims
|
|
33898
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
33899
|
+
) AS claim_text
|
|
33647
33900
|
FROM commons_entities
|
|
33648
33901
|
WHERE ${clauses.join(" AND ")}
|
|
33649
33902
|
ORDER BY updated_at DESC, title ASC
|
|
33650
|
-
LIMIT
|
|
33903
|
+
LIMIT ${SEARCH_CANDIDATE_LIMIT}
|
|
33651
33904
|
`,
|
|
33652
33905
|
args
|
|
33653
33906
|
});
|
|
33654
|
-
|
|
33655
|
-
|
|
33656
|
-
|
|
33657
|
-
|
|
33658
|
-
|
|
33659
|
-
|
|
33907
|
+
const ranked = candidates.rows.map((row) => {
|
|
33908
|
+
const record = row;
|
|
33909
|
+
const score = relevanceScore(terms, {
|
|
33910
|
+
title: String(record.title ?? ""),
|
|
33911
|
+
description: String(record.description ?? ""),
|
|
33912
|
+
tags: parseJsonArray(record.tags_json),
|
|
33913
|
+
keywords: parseJsonArray(record.keywords_json),
|
|
33914
|
+
searchText: String(record.search_text ?? ""),
|
|
33915
|
+
claimText: String(record.claim_text ?? "")
|
|
33916
|
+
});
|
|
33917
|
+
return { id: String(record.id), updatedAt: String(record.updated_at ?? ""), title: String(record.title ?? ""), score };
|
|
33918
|
+
}).filter((candidate) => candidate.score > 0).sort((left, right) => right.score - left.score || right.updatedAt.localeCompare(left.updatedAt) || left.title.localeCompare(right.title));
|
|
33919
|
+
const pageIds = ranked.slice(offset, offset + limit).map((candidate) => candidate.id);
|
|
33920
|
+
if (!pageIds.length) return { results: [], filters: mergedFilters, limit, offset };
|
|
33921
|
+
const page = await getDb().execute({
|
|
33922
|
+
sql: `SELECT * FROM commons_entities WHERE id IN (${pageIds.map(() => "?").join(", ")})`,
|
|
33923
|
+
args: pageIds
|
|
33924
|
+
});
|
|
33925
|
+
const byId = new Map(page.rows.map((row) => {
|
|
33926
|
+
const entity = rowToEntity(row);
|
|
33927
|
+
return [entity.id, entity];
|
|
33928
|
+
}));
|
|
33929
|
+
const results = pageIds.map((id) => byId.get(id)).filter((entity) => Boolean(entity));
|
|
33930
|
+
await resolveRelatedLinksAgainstPublishedEntities(results);
|
|
33931
|
+
return { results, filters: mergedFilters, limit, offset };
|
|
33932
|
+
}
|
|
33933
|
+
var SEARCH_CANDIDATE_LIMIT = 500;
|
|
33934
|
+
function parseJsonArray(value) {
|
|
33935
|
+
try {
|
|
33936
|
+
const parsed = JSON.parse(String(value ?? "[]"));
|
|
33937
|
+
return Array.isArray(parsed) ? parsed.map((item) => String(item)) : [];
|
|
33938
|
+
} catch {
|
|
33939
|
+
return [];
|
|
33940
|
+
}
|
|
33941
|
+
}
|
|
33942
|
+
function containsWord(haystack, term) {
|
|
33943
|
+
if (!haystack || !term) return false;
|
|
33944
|
+
const pattern = new RegExp(`(^|[^a-z0-9])${term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}([^a-z0-9]|$)`, "i");
|
|
33945
|
+
return pattern.test(haystack);
|
|
33946
|
+
}
|
|
33947
|
+
function relevanceScore(terms, fields) {
|
|
33948
|
+
let score = 0;
|
|
33949
|
+
const tagText = fields.tags.join(" ");
|
|
33950
|
+
const keywordText = fields.keywords.join(" ");
|
|
33951
|
+
for (const term of terms) {
|
|
33952
|
+
const inTitle = containsWord(fields.title, term);
|
|
33953
|
+
const inTags = containsWord(tagText, term) || containsWord(keywordText, term);
|
|
33954
|
+
const inDescription = containsWord(fields.description, term);
|
|
33955
|
+
const inBody = containsWord(fields.searchText, term);
|
|
33956
|
+
const inClaims = containsWord(fields.claimText, term);
|
|
33957
|
+
if (!inTitle && !inTags && !inDescription && !inBody && !inClaims) return 0;
|
|
33958
|
+
if (inTitle) score += 100;
|
|
33959
|
+
if (inTags) score += 40;
|
|
33960
|
+
if (inDescription) score += 20;
|
|
33961
|
+
if (inClaims) score += 10;
|
|
33962
|
+
if (inBody) score += 5;
|
|
33963
|
+
}
|
|
33964
|
+
if (terms.every((term) => containsWord(fields.title, term))) score += 50;
|
|
33965
|
+
return score;
|
|
33660
33966
|
}
|
|
33661
33967
|
async function prepareCommonsEntity(input, user) {
|
|
33662
33968
|
await ensureCommonsSchema();
|
|
@@ -33803,7 +34109,10 @@ async function getCommonsEntity(idOrSlug) {
|
|
|
33803
34109
|
`,
|
|
33804
34110
|
args: [lookup, normalizeSlug(lookup)]
|
|
33805
34111
|
});
|
|
33806
|
-
|
|
34112
|
+
if (!result.rows[0]) return null;
|
|
34113
|
+
const entity = rowToEntity(result.rows[0]);
|
|
34114
|
+
await resolveRelatedLinksAgainstPublishedEntities([entity]);
|
|
34115
|
+
return entity;
|
|
33807
34116
|
}
|
|
33808
34117
|
async function classifyCommonsWrite(input, user) {
|
|
33809
34118
|
await ensureCommonsSchema();
|
|
@@ -33826,9 +34135,25 @@ async function submitCommonsEntity(input, user) {
|
|
|
33826
34135
|
applied: existingProposal.status === "accepted",
|
|
33827
34136
|
reviewReason: existingProposal.reviewReason,
|
|
33828
34137
|
conflictNote: null,
|
|
34138
|
+
droppedImages: [],
|
|
33829
34139
|
publicUrl: entity2?.status === "published" ? publicUrlForSlug(entity2.slug) : null
|
|
33830
34140
|
};
|
|
33831
34141
|
}
|
|
34142
|
+
const hostedImages = await hostEntityImages({
|
|
34143
|
+
userId: Number(user.id),
|
|
34144
|
+
featuredImage: normalized.featuredImage,
|
|
34145
|
+
media: normalized.media
|
|
34146
|
+
});
|
|
34147
|
+
if (hostedImages.featuredImageUrl && normalized.featuredImage) {
|
|
34148
|
+
normalized.featuredImage = { ...normalized.featuredImage, url: hostedImages.featuredImageUrl };
|
|
34149
|
+
}
|
|
34150
|
+
if (hostedImages.mediaUrls.size || hostedImages.dropped.length) {
|
|
34151
|
+
const droppedUrls = new Set(hostedImages.dropped.map((item) => item.url));
|
|
34152
|
+
normalized.media = normalized.media.filter((item) => !droppedUrls.has(item.url)).map((item) => {
|
|
34153
|
+
const hostedUrl = hostedImages.mediaUrls.get(item.url);
|
|
34154
|
+
return hostedUrl ? { ...item, url: hostedUrl } : item;
|
|
34155
|
+
});
|
|
34156
|
+
}
|
|
33832
34157
|
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
33833
34158
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
33834
34159
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
@@ -33923,6 +34248,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
33923
34248
|
applied: shouldApply,
|
|
33924
34249
|
reviewReason: safety.reason,
|
|
33925
34250
|
conflictNote: safety.conflictNote,
|
|
34251
|
+
droppedImages: hostedImages.dropped,
|
|
33926
34252
|
publicUrl: entity?.status === "published" && shouldApply ? publicUrlForSlug(entity.slug) : null
|
|
33927
34253
|
};
|
|
33928
34254
|
}
|
|
@@ -34061,8 +34387,9 @@ async function listCommonsNeedsLinks(filters = {}) {
|
|
|
34061
34387
|
});
|
|
34062
34388
|
const concepts = /* @__PURE__ */ new Map();
|
|
34063
34389
|
const queryTerms = tokenize(cleanFilters.query);
|
|
34064
|
-
|
|
34065
|
-
|
|
34390
|
+
const scanned = result.rows.map((row) => rowToEntity(row));
|
|
34391
|
+
await resolveRelatedLinksAgainstPublishedEntities(scanned);
|
|
34392
|
+
for (const entity of scanned) {
|
|
34066
34393
|
for (const link of entity.articleSections.relatedLinks) {
|
|
34067
34394
|
if (!relatedLinkNeedsPage(link)) continue;
|
|
34068
34395
|
const haystack = [link.title, link.summary, link.relationship, entity.title, entity.description, entity.tags.join(" ")].filter(Boolean).join(" ").toLowerCase();
|
|
@@ -34770,7 +35097,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
34770
35097
|
return documents;
|
|
34771
35098
|
}
|
|
34772
35099
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
34773
|
-
return `commons-index-${
|
|
35100
|
+
return `commons-index-${createHash15("sha256").update(`${entityId}
|
|
34774
35101
|
${documentType}
|
|
34775
35102
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
34776
35103
|
}
|
|
@@ -35105,6 +35432,44 @@ function sanitizeNeedsLinkFilters(filters) {
|
|
|
35105
35432
|
offset: clampNumber(filters.offset, 0, 1e4, 0)
|
|
35106
35433
|
};
|
|
35107
35434
|
}
|
|
35435
|
+
async function publishedEntityIndexFor(titles) {
|
|
35436
|
+
const index = /* @__PURE__ */ new Map();
|
|
35437
|
+
const slugKeys = uniqueStrings(titles.map((title) => normalizeNeedsLinkTitle(title))).filter(Boolean);
|
|
35438
|
+
const titleKeys = uniqueStrings(titles.map((title) => String(title || "").trim().toLowerCase())).filter(Boolean);
|
|
35439
|
+
if (!slugKeys.length && !titleKeys.length) return index;
|
|
35440
|
+
const result = await getDb().execute({
|
|
35441
|
+
sql: `
|
|
35442
|
+
SELECT id, slug, title
|
|
35443
|
+
FROM commons_entities
|
|
35444
|
+
WHERE status = 'published'
|
|
35445
|
+
AND (slug IN (${slugKeys.map(() => "?").join(", ") || "NULL"})
|
|
35446
|
+
OR lower(title) IN (${titleKeys.map(() => "?").join(", ") || "NULL"}))
|
|
35447
|
+
`,
|
|
35448
|
+
args: [...slugKeys, ...titleKeys]
|
|
35449
|
+
});
|
|
35450
|
+
for (const row of result.rows) {
|
|
35451
|
+
const record = row;
|
|
35452
|
+
const entry = { id: String(record.id), slug: String(record.slug), title: String(record.title) };
|
|
35453
|
+
index.set(entry.slug, entry);
|
|
35454
|
+
index.set(normalizeNeedsLinkTitle(entry.title), entry);
|
|
35455
|
+
}
|
|
35456
|
+
return index;
|
|
35457
|
+
}
|
|
35458
|
+
function unresolvedRelatedLinkTitles(entities) {
|
|
35459
|
+
return entities.flatMap((entity) => entity.articleSections.relatedLinks.filter((link) => relatedLinkNeedsPage(link)).map((link) => link.title));
|
|
35460
|
+
}
|
|
35461
|
+
async function resolveRelatedLinksAgainstPublishedEntities(entities) {
|
|
35462
|
+
const index = await publishedEntityIndexFor(unresolvedRelatedLinkTitles(entities));
|
|
35463
|
+
if (!index.size) return;
|
|
35464
|
+
for (const entity of entities) {
|
|
35465
|
+
entity.articleSections.relatedLinks = entity.articleSections.relatedLinks.map((link) => {
|
|
35466
|
+
if (!relatedLinkNeedsPage(link)) return link;
|
|
35467
|
+
const match = index.get(normalizeNeedsLinkTitle(link.title));
|
|
35468
|
+
if (!match || match.id === entity.id) return link;
|
|
35469
|
+
return { ...link, slug: match.slug, entityId: match.id, needsLink: false };
|
|
35470
|
+
});
|
|
35471
|
+
}
|
|
35472
|
+
}
|
|
35108
35473
|
function relatedLinkNeedsPage(link) {
|
|
35109
35474
|
if (link.needsLink === true) return true;
|
|
35110
35475
|
if (link.entityId || link.slug) return false;
|
|
@@ -35309,7 +35674,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
35309
35674
|
};
|
|
35310
35675
|
|
|
35311
35676
|
// src/api/commons-publication-repository.ts
|
|
35312
|
-
import { createHash as
|
|
35677
|
+
import { createHash as createHash16, randomUUID as randomUUID18 } from "crypto";
|
|
35313
35678
|
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
35314
35679
|
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
35315
35680
|
"admin",
|
|
@@ -35507,7 +35872,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
35507
35872
|
JSON.stringify(editionInput.articles),
|
|
35508
35873
|
html,
|
|
35509
35874
|
rendered.filename,
|
|
35510
|
-
|
|
35875
|
+
createHash16("sha256").update(html).digest("hex"),
|
|
35511
35876
|
rendered.articleCount,
|
|
35512
35877
|
rendered.wordCount,
|
|
35513
35878
|
Buffer.byteLength(html),
|
|
@@ -35944,6 +36309,28 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
35944
36309
|
return publicationError(c, error);
|
|
35945
36310
|
}
|
|
35946
36311
|
});
|
|
36312
|
+
var HostImageSchema = z32.object({
|
|
36313
|
+
sourceUrl: z32.string().url().optional(),
|
|
36314
|
+
imageBase64: z32.string().min(16).optional(),
|
|
36315
|
+
alt: z32.string().max(500).optional(),
|
|
36316
|
+
license: z32.string().max(240).optional(),
|
|
36317
|
+
attribution: z32.string().max(500).optional()
|
|
36318
|
+
}).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
|
|
36319
|
+
message: "Provide either sourceUrl or imageBase64."
|
|
36320
|
+
});
|
|
36321
|
+
commonsApp.post("/images", auth2, async (c) => {
|
|
36322
|
+
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
36323
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
36324
|
+
try {
|
|
36325
|
+
const hosted = await hostCommonsImage({ ...parsed.data, userId: Number(c.get("user").id) });
|
|
36326
|
+
return c.json({ ok: true, data: hosted }, hosted.reused ? 200 : 201);
|
|
36327
|
+
} catch (error) {
|
|
36328
|
+
if (error instanceof CommonsImageError) {
|
|
36329
|
+
return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
|
|
36330
|
+
}
|
|
36331
|
+
throw error;
|
|
36332
|
+
}
|
|
36333
|
+
});
|
|
35947
36334
|
commonsApp.get("/entities", async (c) => {
|
|
35948
36335
|
const result = await searchCommonsEntities(filtersFromQuery(c.req.query()));
|
|
35949
36336
|
return c.json({ ok: true, data: result });
|
|
@@ -36156,12 +36543,12 @@ function xmlEscape(value) {
|
|
|
36156
36543
|
import { Hono as Hono32 } from "hono";
|
|
36157
36544
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
36158
36545
|
import { z as z34 } from "zod";
|
|
36159
|
-
import { createHash as
|
|
36546
|
+
import { createHash as createHash18 } from "crypto";
|
|
36160
36547
|
import Papa2 from "papaparse";
|
|
36161
36548
|
|
|
36162
36549
|
// src/api/analytics-entitlement.ts
|
|
36163
36550
|
import {
|
|
36164
|
-
createHash as
|
|
36551
|
+
createHash as createHash17,
|
|
36165
36552
|
createHmac as createHmac3,
|
|
36166
36553
|
randomBytes as randomBytes4,
|
|
36167
36554
|
timingSafeEqual as timingSafeEqual2
|
|
@@ -36376,7 +36763,7 @@ function dashboardCallbackUrl() {
|
|
|
36376
36763
|
async function createThorbitConnectUrl(userId) {
|
|
36377
36764
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
36378
36765
|
const state = randomBytes4(32).toString("base64url");
|
|
36379
|
-
const stateHash =
|
|
36766
|
+
const stateHash = createHash17("sha256").update(state).digest("hex");
|
|
36380
36767
|
await getAnalyticsPool().query(
|
|
36381
36768
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
36382
36769
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -36405,7 +36792,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
36405
36792
|
return parsed.data.entitlement;
|
|
36406
36793
|
}
|
|
36407
36794
|
async function consumeThorbitConnectCallback(input) {
|
|
36408
|
-
const stateHash =
|
|
36795
|
+
const stateHash = createHash17("sha256").update(input.state).digest("hex");
|
|
36409
36796
|
const client2 = await getAnalyticsPool().connect();
|
|
36410
36797
|
try {
|
|
36411
36798
|
await client2.query("BEGIN");
|
|
@@ -36881,7 +37268,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
|
|
|
36881
37268
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
36882
37269
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
36883
37270
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
36884
|
-
const suffix2 =
|
|
37271
|
+
const suffix2 = createHash18("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
36885
37272
|
const path5 = `Leads/person-${suffix2}`;
|
|
36886
37273
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
36887
37274
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -36980,9 +37367,9 @@ Submitted the published form.
|
|
|
36980
37367
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
36981
37368
|
});
|
|
36982
37369
|
const match = {
|
|
36983
|
-
...email ? { emailSha256:
|
|
37370
|
+
...email ? { emailSha256: createHash18("sha256").update(email).digest("hex") } : {},
|
|
36984
37371
|
...phone ? {
|
|
36985
|
-
phoneSha256:
|
|
37372
|
+
phoneSha256: createHash18("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
36986
37373
|
} : {}
|
|
36987
37374
|
};
|
|
36988
37375
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -37410,7 +37797,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
|
|
|
37410
37797
|
rejectedCount += 1;
|
|
37411
37798
|
continue;
|
|
37412
37799
|
}
|
|
37413
|
-
const digest2 =
|
|
37800
|
+
const digest2 = createHash18("sha256").update(
|
|
37414
37801
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
37415
37802
|
).digest("hex").slice(0, 16);
|
|
37416
37803
|
const path5 = `Leads/person-${digest2}`;
|
|
@@ -37861,13 +38248,13 @@ import { Hono as Hono33 } from "hono";
|
|
|
37861
38248
|
import { z as z35 } from "zod";
|
|
37862
38249
|
|
|
37863
38250
|
// src/api/scheduled-artifact-owner.ts
|
|
37864
|
-
import { createHash as
|
|
38251
|
+
import { createHash as createHash19 } from "crypto";
|
|
37865
38252
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
37866
|
-
return
|
|
38253
|
+
return createHash19("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
37867
38254
|
}
|
|
37868
38255
|
|
|
37869
38256
|
// src/api/scheduled-run-view-links.ts
|
|
37870
|
-
import { createHash as
|
|
38257
|
+
import { createHash as createHash20, randomBytes as randomBytes5, randomUUID as randomUUID19 } from "crypto";
|
|
37871
38258
|
var schemaReady2 = false;
|
|
37872
38259
|
async function ensureScheduledRunViewLinksSchema() {
|
|
37873
38260
|
if (schemaReady2) return;
|
|
@@ -37921,7 +38308,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
37921
38308
|
schemaReady2 = true;
|
|
37922
38309
|
}
|
|
37923
38310
|
function tokenHash2(token6) {
|
|
37924
|
-
return
|
|
38311
|
+
return createHash20("sha256").update(token6).digest("hex");
|
|
37925
38312
|
}
|
|
37926
38313
|
function mapRow(row) {
|
|
37927
38314
|
return {
|
|
@@ -38008,7 +38395,7 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
|
|
|
38008
38395
|
}
|
|
38009
38396
|
|
|
38010
38397
|
// src/personal-authority/render.ts
|
|
38011
|
-
import { createHash as
|
|
38398
|
+
import { createHash as createHash21 } from "crypto";
|
|
38012
38399
|
function escapeHtml7(value) {
|
|
38013
38400
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38014
38401
|
"&": "&",
|
|
@@ -38068,13 +38455,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
38068
38455
|
html,
|
|
38069
38456
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38070
38457
|
bytes,
|
|
38071
|
-
sha256:
|
|
38458
|
+
sha256: createHash21("sha256").update(html).digest("hex"),
|
|
38072
38459
|
generatedAt: generatedAtIso
|
|
38073
38460
|
};
|
|
38074
38461
|
}
|
|
38075
38462
|
|
|
38076
38463
|
// src/personal-authority/render-v2.ts
|
|
38077
|
-
import { createHash as
|
|
38464
|
+
import { createHash as createHash22 } from "crypto";
|
|
38078
38465
|
var FONT_STACKS = Object.freeze({
|
|
38079
38466
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
38080
38467
|
"modern-sans": "Avenir Next, Avenir, Helvetica Neue, Arial, sans-serif",
|
|
@@ -38178,13 +38565,13 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
38178
38565
|
html,
|
|
38179
38566
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38180
38567
|
bytes,
|
|
38181
|
-
sha256:
|
|
38568
|
+
sha256: createHash22("sha256").update(html).digest("hex"),
|
|
38182
38569
|
generatedAt: generatedAtIso
|
|
38183
38570
|
};
|
|
38184
38571
|
}
|
|
38185
38572
|
|
|
38186
38573
|
// src/newsroom-publisher/render.ts
|
|
38187
|
-
import { createHash as
|
|
38574
|
+
import { createHash as createHash23 } from "crypto";
|
|
38188
38575
|
function escapeHtml9(value) {
|
|
38189
38576
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38190
38577
|
"&": "&",
|
|
@@ -38290,13 +38677,13 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
38290
38677
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
38291
38678
|
</style></head><body data-theme="${escapeHtml9(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
38292
38679
|
const bytes = Buffer.byteLength(html);
|
|
38293
|
-
const sha2563 =
|
|
38680
|
+
const sha2563 = createHash23("sha256").update(html).digest("hex");
|
|
38294
38681
|
const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
38295
38682
|
return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
|
|
38296
38683
|
}
|
|
38297
38684
|
|
|
38298
38685
|
// src/blog-article/render.ts
|
|
38299
|
-
import { createHash as
|
|
38686
|
+
import { createHash as createHash24 } from "crypto";
|
|
38300
38687
|
function escapeHtml10(value) {
|
|
38301
38688
|
return value.replace(
|
|
38302
38689
|
/[&<>"']/g,
|
|
@@ -38384,7 +38771,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
38384
38771
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
38385
38772
|
</style></head><body data-theme="${escapeHtml10(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml10(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml10(input.eyebrow)}</span><h1>${escapeHtml10(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml10(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml10(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml10(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml10(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
38386
38773
|
const bytes = Buffer.byteLength(html);
|
|
38387
|
-
const sha2563 =
|
|
38774
|
+
const sha2563 = createHash24("sha256").update(html).digest("hex");
|
|
38388
38775
|
return {
|
|
38389
38776
|
html,
|
|
38390
38777
|
filename: "blog-article.html",
|
|
@@ -38687,7 +39074,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
38687
39074
|
import { Hono as Hono34 } from "hono";
|
|
38688
39075
|
|
|
38689
39076
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
38690
|
-
import { createHash as
|
|
39077
|
+
import { createHash as createHash25 } from "crypto";
|
|
38691
39078
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
38692
39079
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
38693
39080
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -38703,7 +39090,7 @@ function policy4() {
|
|
|
38703
39090
|
};
|
|
38704
39091
|
}
|
|
38705
39092
|
function runStorageSegment(runId) {
|
|
38706
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
39093
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash25("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
38707
39094
|
}
|
|
38708
39095
|
async function createScheduledRunArtifact(args) {
|
|
38709
39096
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -38810,7 +39197,7 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
38810
39197
|
import { Hono as Hono35 } from "hono";
|
|
38811
39198
|
|
|
38812
39199
|
// src/api/scheduler-integration-auth.ts
|
|
38813
|
-
import { createHash as
|
|
39200
|
+
import { createHash as createHash26, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
38814
39201
|
|
|
38815
39202
|
// src/api/service-connections.ts
|
|
38816
39203
|
import { randomUUID as randomUUID20 } from "crypto";
|
|
@@ -39108,7 +39495,7 @@ function signingSecret() {
|
|
|
39108
39495
|
return secret;
|
|
39109
39496
|
}
|
|
39110
39497
|
function schedulerIntegrationSignature(args) {
|
|
39111
|
-
const bodyHash =
|
|
39498
|
+
const bodyHash = createHash26("sha256").update(args.body).digest("hex");
|
|
39112
39499
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
39113
39500
|
${args.path}
|
|
39114
39501
|
${args.timestamp}
|
|
@@ -39300,7 +39687,7 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
39300
39687
|
}
|
|
39301
39688
|
|
|
39302
39689
|
// src/api/page-media-artifacts.ts
|
|
39303
|
-
import { createHash as
|
|
39690
|
+
import { createHash as createHash27, randomBytes as randomBytes6 } from "crypto";
|
|
39304
39691
|
import { homedir as homedir3 } from "os";
|
|
39305
39692
|
import { basename, extname, join as join8 } from "path";
|
|
39306
39693
|
import pLimit6 from "p-limit";
|
|
@@ -39447,7 +39834,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
39447
39834
|
finalUrl: checked.parsed.href,
|
|
39448
39835
|
width: dimensions?.width ?? null,
|
|
39449
39836
|
height: dimensions?.height ?? null,
|
|
39450
|
-
sha256:
|
|
39837
|
+
sha256: createHash27("sha256").update(bytes).digest("hex")
|
|
39451
39838
|
};
|
|
39452
39839
|
}
|
|
39453
39840
|
throw new Error("media_redirect_rejected");
|
|
@@ -39640,13 +40027,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
39640
40027
|
}
|
|
39641
40028
|
|
|
39642
40029
|
// src/api/page-diff.ts
|
|
39643
|
-
import { createHash as
|
|
40030
|
+
import { createHash as createHash28 } from "crypto";
|
|
39644
40031
|
import { diffLines } from "diff";
|
|
39645
40032
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
39646
40033
|
var MAX_DIFF_HUNKS = 200;
|
|
39647
40034
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
39648
40035
|
function sha256Hex(value) {
|
|
39649
|
-
return
|
|
40036
|
+
return createHash28("sha256").update(value).digest("hex");
|
|
39650
40037
|
}
|
|
39651
40038
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
39652
40039
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -39760,10 +40147,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
39760
40147
|
}
|
|
39761
40148
|
|
|
39762
40149
|
// src/api/scrape-image-sink.ts
|
|
39763
|
-
import { createHash as
|
|
40150
|
+
import { createHash as createHash29 } from "crypto";
|
|
39764
40151
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
39765
40152
|
function idempotencyKey2(userId, vault, input) {
|
|
39766
|
-
return `scrape-image-${
|
|
40153
|
+
return `scrape-image-${createHash29("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
39767
40154
|
}
|
|
39768
40155
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
39769
40156
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -40235,7 +40622,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
40235
40622
|
}
|
|
40236
40623
|
|
|
40237
40624
|
// src/api/connection-memory-import.ts
|
|
40238
|
-
import { createHash as
|
|
40625
|
+
import { createHash as createHash30 } from "crypto";
|
|
40239
40626
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
40240
40627
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
40241
40628
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -40265,7 +40652,7 @@ function canonicalJson(value) {
|
|
|
40265
40652
|
return JSON.stringify(value);
|
|
40266
40653
|
}
|
|
40267
40654
|
function sha2562(value) {
|
|
40268
|
-
return
|
|
40655
|
+
return createHash30("sha256").update(value).digest("hex");
|
|
40269
40656
|
}
|
|
40270
40657
|
function sensitiveKey(key) {
|
|
40271
40658
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -41276,7 +41663,7 @@ async function dbUsage(identity, plan) {
|
|
|
41276
41663
|
}
|
|
41277
41664
|
|
|
41278
41665
|
// src/api/nango-control.ts
|
|
41279
|
-
import { createHash as
|
|
41666
|
+
import { createHash as createHash32, randomUUID as randomUUID24 } from "crypto";
|
|
41280
41667
|
|
|
41281
41668
|
// src/api/slack-archive-analysis.ts
|
|
41282
41669
|
function isRecord2(value) {
|
|
@@ -41650,7 +42037,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
41650
42037
|
}
|
|
41651
42038
|
|
|
41652
42039
|
// src/api/main-nango-transport.ts
|
|
41653
|
-
import { createHash as
|
|
42040
|
+
import { createHash as createHash31, randomUUID as randomUUID23 } from "crypto";
|
|
41654
42041
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
41655
42042
|
|
|
41656
42043
|
// src/api/connected-tool-policy.ts
|
|
@@ -42043,7 +42430,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
42043
42430
|
});
|
|
42044
42431
|
const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
|
|
42045
42432
|
const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
|
|
42046
|
-
const revision =
|
|
42433
|
+
const revision = createHash31("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
42047
42434
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
42048
42435
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
42049
42436
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -42071,7 +42458,7 @@ async function callNangoToolDirect(args) {
|
|
|
42071
42458
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
42072
42459
|
});
|
|
42073
42460
|
const requestId = args.requestId?.trim() || randomUUID23();
|
|
42074
|
-
const idempotencyKey3 = `main-nango:${
|
|
42461
|
+
const idempotencyKey3 = `main-nango:${createHash31("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
42075
42462
|
const startedAt = /* @__PURE__ */ new Date();
|
|
42076
42463
|
const started = performance.now();
|
|
42077
42464
|
let result;
|
|
@@ -42096,7 +42483,7 @@ async function callNangoToolDirect(args) {
|
|
|
42096
42483
|
toolName: args.tool,
|
|
42097
42484
|
operationKind: args.operationKind ?? args.classification,
|
|
42098
42485
|
outcome: providerError ? "error" : "partial",
|
|
42099
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
42486
|
+
requestId: requestId.length <= 200 ? requestId : createHash31("sha256").update(requestId).digest("hex"),
|
|
42100
42487
|
startedAt: startedAt.toISOString(),
|
|
42101
42488
|
completedAt: completedAt.toISOString()
|
|
42102
42489
|
}
|
|
@@ -42138,7 +42525,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
42138
42525
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
42139
42526
|
protocolVersion: null,
|
|
42140
42527
|
schemaSource: "live_tools_list",
|
|
42141
|
-
schemaHash:
|
|
42528
|
+
schemaHash: createHash31("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
42142
42529
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
42143
42530
|
};
|
|
42144
42531
|
}
|
|
@@ -42862,8 +43249,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
42862
43249
|
return data.connection.actionsEnabled === true;
|
|
42863
43250
|
}
|
|
42864
43251
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
42865
|
-
const requestId = `main-connected-action:${
|
|
42866
|
-
const requestDigest =
|
|
43252
|
+
const requestId = `main-connected-action:${createHash32("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID24()).digest("hex")}`;
|
|
43253
|
+
const requestDigest = createHash32("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
42867
43254
|
if (mainOwnsIntegrations()) {
|
|
42868
43255
|
const selectedTool = tool?.trim();
|
|
42869
43256
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -43014,7 +43401,7 @@ function canonicalJson2(value) {
|
|
|
43014
43401
|
return JSON.stringify(value);
|
|
43015
43402
|
}
|
|
43016
43403
|
function projectedToolSchemaHash(tool) {
|
|
43017
|
-
return
|
|
43404
|
+
return createHash32("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
43018
43405
|
}
|
|
43019
43406
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
43020
43407
|
if (mainOwnsIntegrations()) {
|
|
@@ -43293,7 +43680,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
43293
43680
|
}
|
|
43294
43681
|
|
|
43295
43682
|
// src/api/resend-control.ts
|
|
43296
|
-
import { createHash as
|
|
43683
|
+
import { createHash as createHash33 } from "crypto";
|
|
43297
43684
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
43298
43685
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
43299
43686
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -43552,7 +43939,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
43552
43939
|
return isRecord5(data) ? data.result ?? data : data;
|
|
43553
43940
|
}
|
|
43554
43941
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
43555
|
-
const requestId = `main-resend-action:${
|
|
43942
|
+
const requestId = `main-resend-action:${createHash33("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
43556
43943
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
43557
43944
|
method: "POST",
|
|
43558
43945
|
headers: { "x-request-id": requestId },
|
|
@@ -46778,7 +47165,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
46778
47165
|
return c.json({ error: "Unauthorized" }, 401);
|
|
46779
47166
|
}
|
|
46780
47167
|
const startedAt = Date.now();
|
|
46781
|
-
const { drainQueue } = await import("./worker-
|
|
47168
|
+
const { drainQueue } = await import("./worker-S2WXELUC.js");
|
|
46782
47169
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
46783
47170
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
46784
47171
|
const unfinished = [];
|
|
@@ -46848,8 +47235,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46848
47235
|
finishExtractJob: finishExtractJob2,
|
|
46849
47236
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
46850
47237
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
46851
|
-
} = await import("./site-extract-repository-
|
|
46852
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
47238
|
+
} = await import("./site-extract-repository-CFMSWAVL.js");
|
|
47239
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
46853
47240
|
const existing = await getExtractJob2(jobId2);
|
|
46854
47241
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
46855
47242
|
if (existing.status !== "failed") {
|
|
@@ -46876,7 +47263,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46876
47263
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
46877
47264
|
let settlement = "already_settled_or_refunded";
|
|
46878
47265
|
if (job.billedMc == null && job.userId != null) {
|
|
46879
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
47266
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CFMSWAVL.js");
|
|
46880
47267
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
46881
47268
|
const successful = await countSuccessfulPages2(jobId2);
|
|
46882
47269
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -47061,4 +47448,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
47061
47448
|
export {
|
|
47062
47449
|
app
|
|
47063
47450
|
};
|
|
47064
|
-
//# sourceMappingURL=server-
|
|
47451
|
+
//# sourceMappingURL=server-DSCKFM7K.js.map
|