mcp-scraper 0.55.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +615 -172
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +49 -4
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-2FL5FXMV.js → chunk-5XYBKE3X.js} +2 -2
- package/dist/{chunk-PGI2QPFE.js → chunk-HU4R6BP2.js} +2 -2
- package/dist/{chunk-PGI2QPFE.js.map → chunk-HU4R6BP2.js.map} +1 -1
- package/dist/chunk-QQ6ZBYZZ.js +7 -0
- package/dist/chunk-QQ6ZBYZZ.js.map +1 -0
- package/dist/{chunk-N4LT53JX.js → chunk-R6CO77BN.js} +3 -3
- package/dist/{chunk-N4LT53JX.js.map → chunk-R6CO77BN.js.map} +1 -1
- package/dist/{chunk-WYLIDW4S.js → chunk-ULAJAZQF.js} +50 -5
- package/dist/chunk-ULAJAZQF.js.map +1 -0
- package/dist/{extract-bundle-3CRG7FAN.js → extract-bundle-H7JEELY7.js} +3 -3
- package/dist/{server-WKBCZ2TH.js → server-NBGZJX4P.js} +474 -86
- package/dist/server-NBGZJX4P.js.map +1 -0
- package/dist/{site-extract-repository-QSRPXNMI.js → site-extract-repository-CFMSWAVL.js} +3 -3
- package/dist/{worker-W5C6SKZT.js → worker-S2WXELUC.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-V2QWNGEK.js +0 -7
- package/dist/chunk-V2QWNGEK.js.map +0 -1
- package/dist/chunk-WYLIDW4S.js.map +0 -1
- package/dist/server-WKBCZ2TH.js.map +0 -1
- /package/dist/{chunk-2FL5FXMV.js.map → chunk-5XYBKE3X.js.map} +0 -0
- /package/dist/{extract-bundle-3CRG7FAN.js.map → extract-bundle-H7JEELY7.js.map} +0 -0
- /package/dist/{site-extract-repository-QSRPXNMI.js.map → site-extract-repository-CFMSWAVL.js.map} +0 -0
- /package/dist/{worker-W5C6SKZT.js.map → worker-S2WXELUC.js.map} +0 -0
|
@@ -169,7 +169,7 @@ import {
|
|
|
169
169
|
resolveDeploymentProfile,
|
|
170
170
|
resolveLocalSourcebookSchemaType,
|
|
171
171
|
transcribeMediaUrl
|
|
172
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-ULAJAZQF.js";
|
|
173
173
|
import {
|
|
174
174
|
auditImageUrls,
|
|
175
175
|
auditImages,
|
|
@@ -225,7 +225,7 @@ import {
|
|
|
225
225
|
} from "./chunk-FG6LJGJ7.js";
|
|
226
226
|
import {
|
|
227
227
|
PACKAGE_VERSION
|
|
228
|
-
} from "./chunk-
|
|
228
|
+
} from "./chunk-QQ6ZBYZZ.js";
|
|
229
229
|
import {
|
|
230
230
|
abandonExtractSettlement,
|
|
231
231
|
countSuccessfulPages,
|
|
@@ -248,7 +248,7 @@ import {
|
|
|
248
248
|
setExtractJobTotal,
|
|
249
249
|
settleExtractJob,
|
|
250
250
|
terminalExtractJobStatus
|
|
251
|
-
} from "./chunk-
|
|
251
|
+
} from "./chunk-5XYBKE3X.js";
|
|
252
252
|
import {
|
|
253
253
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
254
254
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -301,7 +301,7 @@ import {
|
|
|
301
301
|
tierChangeLeavesPrivateOffer,
|
|
302
302
|
tierCreditsPerInterval,
|
|
303
303
|
vendorCostUsdToMc
|
|
304
|
-
} from "./chunk-
|
|
304
|
+
} from "./chunk-R6CO77BN.js";
|
|
305
305
|
import {
|
|
306
306
|
CaptchaError,
|
|
307
307
|
MapsSearchIncompletePageError,
|
|
@@ -9393,7 +9393,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9393
9393
|
});
|
|
9394
9394
|
}) : null;
|
|
9395
9395
|
const finalized = await step.run("finalize", async () => {
|
|
9396
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9396
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
9397
9397
|
const current = await getExtractJob(jobId2);
|
|
9398
9398
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9399
9399
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -32972,7 +32972,224 @@ import { Hono as Hono31 } from "hono";
|
|
|
32972
32972
|
import { z as z32 } from "zod";
|
|
32973
32973
|
|
|
32974
32974
|
// src/api/commons-repository.ts
|
|
32975
|
-
import { createHash as
|
|
32975
|
+
import { createHash as createHash15, randomUUID as randomUUID17 } from "crypto";
|
|
32976
|
+
|
|
32977
|
+
// src/api/commons-image-store.ts
|
|
32978
|
+
import { createHash as createHash13 } from "crypto";
|
|
32979
|
+
import { lookup as dnsLookup } from "dns/promises";
|
|
32980
|
+
import { isIP } from "net";
|
|
32981
|
+
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
32982
|
+
var COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
32983
|
+
var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
32984
|
+
var CommonsImageError = class extends Error {
|
|
32985
|
+
constructor(code, message, httpStatus = 400) {
|
|
32986
|
+
super(message);
|
|
32987
|
+
this.code = code;
|
|
32988
|
+
this.httpStatus = httpStatus;
|
|
32989
|
+
this.name = "CommonsImageError";
|
|
32990
|
+
}
|
|
32991
|
+
code;
|
|
32992
|
+
httpStatus;
|
|
32993
|
+
};
|
|
32994
|
+
var imageSchemaReady = false;
|
|
32995
|
+
async function ensureCommonsImageSchema() {
|
|
32996
|
+
if (imageSchemaReady) return;
|
|
32997
|
+
await getDb().execute(`
|
|
32998
|
+
CREATE TABLE IF NOT EXISTS commons_images (
|
|
32999
|
+
id TEXT PRIMARY KEY,
|
|
33000
|
+
user_id INTEGER NOT NULL,
|
|
33001
|
+
digest TEXT NOT NULL,
|
|
33002
|
+
storage_key TEXT NOT NULL,
|
|
33003
|
+
url TEXT NOT NULL,
|
|
33004
|
+
content_type TEXT NOT NULL,
|
|
33005
|
+
bytes INTEGER NOT NULL,
|
|
33006
|
+
source_url TEXT,
|
|
33007
|
+
alt TEXT,
|
|
33008
|
+
license TEXT,
|
|
33009
|
+
attribution TEXT,
|
|
33010
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
33011
|
+
)
|
|
33012
|
+
`);
|
|
33013
|
+
await getDb().execute(`CREATE UNIQUE INDEX IF NOT EXISTS commons_images_digest ON commons_images(digest)`);
|
|
33014
|
+
imageSchemaReady = true;
|
|
33015
|
+
}
|
|
33016
|
+
function sniffImageFormat(bytes) {
|
|
33017
|
+
if (bytes.length < 12) return null;
|
|
33018
|
+
if (bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return "image/jpeg";
|
|
33019
|
+
if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return "image/png";
|
|
33020
|
+
if (bytes.subarray(0, 6).toString("ascii") === "GIF87a" || bytes.subarray(0, 6).toString("ascii") === "GIF89a") return "image/gif";
|
|
33021
|
+
if (bytes.subarray(0, 4).toString("ascii") === "RIFF" && bytes.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";
|
|
33022
|
+
return null;
|
|
33023
|
+
}
|
|
33024
|
+
function isBlockedAddress(address) {
|
|
33025
|
+
if (isIP(address) === 6) {
|
|
33026
|
+
const normalized = address.toLowerCase();
|
|
33027
|
+
return normalized === "::1" || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("fe80");
|
|
33028
|
+
}
|
|
33029
|
+
const parts = address.split(".").map(Number);
|
|
33030
|
+
if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) return true;
|
|
33031
|
+
const [a, b] = parts;
|
|
33032
|
+
if (a === 10 || a === 127 || a === 0) return true;
|
|
33033
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
33034
|
+
if (a === 192 && b === 168) return true;
|
|
33035
|
+
if (a === 169 && b === 254) return true;
|
|
33036
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
33037
|
+
return false;
|
|
33038
|
+
}
|
|
33039
|
+
async function assertPublicHttpsUrl(rawUrl) {
|
|
33040
|
+
let url;
|
|
33041
|
+
try {
|
|
33042
|
+
url = new URL(rawUrl);
|
|
33043
|
+
} catch {
|
|
33044
|
+
throw new CommonsImageError("image_url_invalid", `"${rawUrl}" is not a valid URL.`);
|
|
33045
|
+
}
|
|
33046
|
+
if (url.protocol !== "https:") {
|
|
33047
|
+
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
33048
|
+
}
|
|
33049
|
+
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
33050
|
+
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
33051
|
+
if (!addresses.length) {
|
|
33052
|
+
throw new CommonsImageError("image_host_unresolvable", `Could not resolve ${url.hostname}.`);
|
|
33053
|
+
}
|
|
33054
|
+
if (addresses.some((entry) => isBlockedAddress(entry.address))) {
|
|
33055
|
+
throw new CommonsImageError("image_host_blocked", "Commons image sources cannot resolve to private or loopback addresses.");
|
|
33056
|
+
}
|
|
33057
|
+
return url;
|
|
33058
|
+
}
|
|
33059
|
+
async function downloadImage(sourceUrl) {
|
|
33060
|
+
let current = sourceUrl;
|
|
33061
|
+
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
33062
|
+
const url = await assertPublicHttpsUrl(current);
|
|
33063
|
+
const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(COMMONS_IMAGE_FETCH_TIMEOUT_MS) });
|
|
33064
|
+
if (response.status >= 300 && response.status < 400) {
|
|
33065
|
+
const location2 = response.headers.get("location");
|
|
33066
|
+
if (!location2) throw new CommonsImageError("image_redirect_invalid", "Image source redirected without a destination.");
|
|
33067
|
+
current = new URL(location2, url).toString();
|
|
33068
|
+
continue;
|
|
33069
|
+
}
|
|
33070
|
+
if (!response.ok) {
|
|
33071
|
+
throw new CommonsImageError("image_source_unavailable", `Image source returned HTTP ${response.status}.`, 422);
|
|
33072
|
+
}
|
|
33073
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
33074
|
+
if (declaredLength > COMMONS_IMAGE_MAX_BYTES) {
|
|
33075
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33076
|
+
}
|
|
33077
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
33078
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33079
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33080
|
+
}
|
|
33081
|
+
return bytes;
|
|
33082
|
+
}
|
|
33083
|
+
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
33084
|
+
}
|
|
33085
|
+
function decodeBase64Image(value) {
|
|
33086
|
+
const payload = value.includes(",") && value.trim().startsWith("data:") ? value.slice(value.indexOf(",") + 1) : value;
|
|
33087
|
+
const bytes = Buffer.from(payload, "base64");
|
|
33088
|
+
if (!bytes.length) throw new CommonsImageError("image_base64_invalid", "imageBase64 did not decode to any bytes.");
|
|
33089
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
33090
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
33091
|
+
}
|
|
33092
|
+
return bytes;
|
|
33093
|
+
}
|
|
33094
|
+
function isCommonsHostedImageUrl(url) {
|
|
33095
|
+
return /^https:\/\/[^/]*(\.public\.blob\.vercel-storage\.com|blob\.vercel-storage\.com)\//i.test(url) || url.startsWith("file://");
|
|
33096
|
+
}
|
|
33097
|
+
var hostOverride = null;
|
|
33098
|
+
async function hostCommonsImage(input) {
|
|
33099
|
+
if (hostOverride) return hostOverride(input);
|
|
33100
|
+
await ensureCommonsImageSchema();
|
|
33101
|
+
if (!input.sourceUrl && !input.imageBase64) {
|
|
33102
|
+
throw new CommonsImageError("image_source_missing", "Provide either sourceUrl or imageBase64.");
|
|
33103
|
+
}
|
|
33104
|
+
const bytes = input.imageBase64 ? decodeBase64Image(input.imageBase64) : await downloadImage(String(input.sourceUrl));
|
|
33105
|
+
const contentType = sniffImageFormat(bytes);
|
|
33106
|
+
if (!contentType) {
|
|
33107
|
+
throw new CommonsImageError(
|
|
33108
|
+
"image_format_unsupported",
|
|
33109
|
+
"Commons hosts JPEG, PNG, GIF, and WebP. SVG and other formats are rejected because the wiki serves them to the public web.",
|
|
33110
|
+
415
|
|
33111
|
+
);
|
|
33112
|
+
}
|
|
33113
|
+
const digest2 = createHash13("sha256").update(bytes).digest("hex");
|
|
33114
|
+
const existing = await getDb().execute({
|
|
33115
|
+
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
33116
|
+
args: [digest2]
|
|
33117
|
+
});
|
|
33118
|
+
const reusable = existing.rows[0];
|
|
33119
|
+
if (reusable) {
|
|
33120
|
+
return {
|
|
33121
|
+
id: String(reusable.id),
|
|
33122
|
+
url: String(reusable.url),
|
|
33123
|
+
contentType: String(reusable.content_type),
|
|
33124
|
+
bytes: Number(reusable.bytes),
|
|
33125
|
+
digest: digest2,
|
|
33126
|
+
sourceUrl: reusable.source_url ? String(reusable.source_url) : null,
|
|
33127
|
+
reused: true
|
|
33128
|
+
};
|
|
33129
|
+
}
|
|
33130
|
+
const extension = contentType.split("/")[1] === "jpeg" ? "jpg" : contentType.split("/")[1];
|
|
33131
|
+
const storageKey = `commons/images/${digest2.slice(0, 2)}/${digest2}.${extension}`;
|
|
33132
|
+
const stored = await getBlobStore().put(storageKey, bytes, contentType);
|
|
33133
|
+
const id = `TPW-IMG-${digest2.slice(0, 16).toUpperCase()}`;
|
|
33134
|
+
await getDb().execute({
|
|
33135
|
+
sql: `
|
|
33136
|
+
INSERT INTO commons_images (id, user_id, digest, storage_key, url, content_type, bytes, source_url, alt, license, attribution, created_at)
|
|
33137
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
33138
|
+
`,
|
|
33139
|
+
args: [
|
|
33140
|
+
id,
|
|
33141
|
+
input.userId,
|
|
33142
|
+
digest2,
|
|
33143
|
+
storageKey,
|
|
33144
|
+
stored.url,
|
|
33145
|
+
contentType,
|
|
33146
|
+
bytes.length,
|
|
33147
|
+
input.sourceUrl ?? null,
|
|
33148
|
+
input.alt ?? null,
|
|
33149
|
+
input.license ?? null,
|
|
33150
|
+
input.attribution ?? null,
|
|
33151
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
33152
|
+
]
|
|
33153
|
+
});
|
|
33154
|
+
return { id, url: stored.url, contentType, bytes: bytes.length, digest: digest2, sourceUrl: input.sourceUrl ?? null, reused: false };
|
|
33155
|
+
}
|
|
33156
|
+
async function hostEntityImages(input) {
|
|
33157
|
+
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
33158
|
+
const featuredUrl = input.featuredImage?.url;
|
|
33159
|
+
if (featuredUrl && !isCommonsHostedImageUrl(featuredUrl)) {
|
|
33160
|
+
const hosted = await hostCommonsImage({
|
|
33161
|
+
userId: input.userId,
|
|
33162
|
+
sourceUrl: featuredUrl,
|
|
33163
|
+
alt: input.featuredImage?.alt,
|
|
33164
|
+
license: input.featuredImage?.license,
|
|
33165
|
+
attribution: input.featuredImage?.sourceUrl
|
|
33166
|
+
}).catch((error) => {
|
|
33167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33168
|
+
throw new CommonsImageError(
|
|
33169
|
+
error instanceof CommonsImageError ? error.code : "featured_image_unhostable",
|
|
33170
|
+
`The featured image could not be hosted, so this entity would publish with a link that can break: ${message}`,
|
|
33171
|
+
422
|
|
33172
|
+
);
|
|
33173
|
+
});
|
|
33174
|
+
rewrite.featuredImageUrl = hosted.url;
|
|
33175
|
+
}
|
|
33176
|
+
for (const item of input.media ?? []) {
|
|
33177
|
+
if (!item.url || isCommonsHostedImageUrl(item.url)) continue;
|
|
33178
|
+
try {
|
|
33179
|
+
const hosted = await hostCommonsImage({
|
|
33180
|
+
userId: input.userId,
|
|
33181
|
+
sourceUrl: item.url,
|
|
33182
|
+
alt: item.alt,
|
|
33183
|
+
license: item.license,
|
|
33184
|
+
attribution: item.sourceUrl
|
|
33185
|
+
});
|
|
33186
|
+
rewrite.mediaUrls.set(item.url, hosted.url);
|
|
33187
|
+
} catch (error) {
|
|
33188
|
+
rewrite.dropped.push({ url: item.url, reason: error instanceof Error ? error.message : String(error) });
|
|
33189
|
+
}
|
|
33190
|
+
}
|
|
33191
|
+
return rewrite;
|
|
33192
|
+
}
|
|
32976
33193
|
|
|
32977
33194
|
// src/api/schema-presence.ts
|
|
32978
33195
|
var cache = null;
|
|
@@ -33000,7 +33217,7 @@ async function schemaObjectsPresent(names) {
|
|
|
33000
33217
|
}
|
|
33001
33218
|
|
|
33002
33219
|
// src/api/commons-linksets.ts
|
|
33003
|
-
import { createHash as
|
|
33220
|
+
import { createHash as createHash14 } from "crypto";
|
|
33004
33221
|
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
33005
33222
|
var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
33006
33223
|
var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -33107,7 +33324,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
33107
33324
|
context[claim.predicate] = targets;
|
|
33108
33325
|
}
|
|
33109
33326
|
const document2 = { linkset: [context] };
|
|
33110
|
-
const etag = `"${
|
|
33327
|
+
const etag = `"${createHash14("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
33111
33328
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
33112
33329
|
}
|
|
33113
33330
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -33640,23 +33857,100 @@ async function searchCommonsEntities(filters = {}, userId) {
|
|
|
33640
33857
|
const pattern = `%${escapeLike(term)}%`;
|
|
33641
33858
|
args.push(pattern, pattern, pattern, pattern, pattern);
|
|
33642
33859
|
}
|
|
33643
|
-
|
|
33644
|
-
|
|
33860
|
+
if (!terms.length) {
|
|
33861
|
+
args.push(limit, offset);
|
|
33862
|
+
const result = await getDb().execute({
|
|
33863
|
+
sql: `
|
|
33864
|
+
SELECT *
|
|
33865
|
+
FROM commons_entities
|
|
33866
|
+
WHERE ${clauses.join(" AND ")}
|
|
33867
|
+
ORDER BY updated_at DESC, title ASC
|
|
33868
|
+
LIMIT ? OFFSET ?
|
|
33869
|
+
`,
|
|
33870
|
+
args
|
|
33871
|
+
});
|
|
33872
|
+
const results2 = result.rows.map((row) => rowToEntity(row));
|
|
33873
|
+
await resolveRelatedLinksAgainstPublishedEntities(results2);
|
|
33874
|
+
return { results: results2, filters: mergedFilters, limit, offset };
|
|
33875
|
+
}
|
|
33876
|
+
const candidates = await getDb().execute({
|
|
33645
33877
|
sql: `
|
|
33646
|
-
SELECT
|
|
33878
|
+
SELECT id, title, tags_json, keywords_json, description, search_text, updated_at,
|
|
33879
|
+
(
|
|
33880
|
+
SELECT group_concat(
|
|
33881
|
+
coalesce(commons_claims.title, '') || ' ' ||
|
|
33882
|
+
coalesce(commons_claims.predicate, '') || ' ' ||
|
|
33883
|
+
coalesce(commons_claims.predicate_label, '') || ' ' ||
|
|
33884
|
+
coalesce(commons_claims.object_url, ''), ' ')
|
|
33885
|
+
FROM commons_claims
|
|
33886
|
+
WHERE commons_claims.subject_entity_id = commons_entities.id
|
|
33887
|
+
) AS claim_text
|
|
33647
33888
|
FROM commons_entities
|
|
33648
33889
|
WHERE ${clauses.join(" AND ")}
|
|
33649
33890
|
ORDER BY updated_at DESC, title ASC
|
|
33650
|
-
LIMIT
|
|
33891
|
+
LIMIT ${SEARCH_CANDIDATE_LIMIT}
|
|
33651
33892
|
`,
|
|
33652
33893
|
args
|
|
33653
33894
|
});
|
|
33654
|
-
|
|
33655
|
-
|
|
33656
|
-
|
|
33657
|
-
|
|
33658
|
-
|
|
33659
|
-
|
|
33895
|
+
const ranked = candidates.rows.map((row) => {
|
|
33896
|
+
const record = row;
|
|
33897
|
+
const score = relevanceScore(terms, {
|
|
33898
|
+
title: String(record.title ?? ""),
|
|
33899
|
+
description: String(record.description ?? ""),
|
|
33900
|
+
tags: parseJsonArray(record.tags_json),
|
|
33901
|
+
keywords: parseJsonArray(record.keywords_json),
|
|
33902
|
+
searchText: String(record.search_text ?? ""),
|
|
33903
|
+
claimText: String(record.claim_text ?? "")
|
|
33904
|
+
});
|
|
33905
|
+
return { id: String(record.id), updatedAt: String(record.updated_at ?? ""), title: String(record.title ?? ""), score };
|
|
33906
|
+
}).filter((candidate) => candidate.score > 0).sort((left, right) => right.score - left.score || right.updatedAt.localeCompare(left.updatedAt) || left.title.localeCompare(right.title));
|
|
33907
|
+
const pageIds = ranked.slice(offset, offset + limit).map((candidate) => candidate.id);
|
|
33908
|
+
if (!pageIds.length) return { results: [], filters: mergedFilters, limit, offset };
|
|
33909
|
+
const page = await getDb().execute({
|
|
33910
|
+
sql: `SELECT * FROM commons_entities WHERE id IN (${pageIds.map(() => "?").join(", ")})`,
|
|
33911
|
+
args: pageIds
|
|
33912
|
+
});
|
|
33913
|
+
const byId = new Map(page.rows.map((row) => {
|
|
33914
|
+
const entity = rowToEntity(row);
|
|
33915
|
+
return [entity.id, entity];
|
|
33916
|
+
}));
|
|
33917
|
+
const results = pageIds.map((id) => byId.get(id)).filter((entity) => Boolean(entity));
|
|
33918
|
+
await resolveRelatedLinksAgainstPublishedEntities(results);
|
|
33919
|
+
return { results, filters: mergedFilters, limit, offset };
|
|
33920
|
+
}
|
|
33921
|
+
var SEARCH_CANDIDATE_LIMIT = 500;
|
|
33922
|
+
function parseJsonArray(value) {
|
|
33923
|
+
try {
|
|
33924
|
+
const parsed = JSON.parse(String(value ?? "[]"));
|
|
33925
|
+
return Array.isArray(parsed) ? parsed.map((item) => String(item)) : [];
|
|
33926
|
+
} catch {
|
|
33927
|
+
return [];
|
|
33928
|
+
}
|
|
33929
|
+
}
|
|
33930
|
+
function containsWord(haystack, term) {
|
|
33931
|
+
if (!haystack || !term) return false;
|
|
33932
|
+
const pattern = new RegExp(`(^|[^a-z0-9])${term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}([^a-z0-9]|$)`, "i");
|
|
33933
|
+
return pattern.test(haystack);
|
|
33934
|
+
}
|
|
33935
|
+
function relevanceScore(terms, fields) {
|
|
33936
|
+
let score = 0;
|
|
33937
|
+
const tagText = fields.tags.join(" ");
|
|
33938
|
+
const keywordText = fields.keywords.join(" ");
|
|
33939
|
+
for (const term of terms) {
|
|
33940
|
+
const inTitle = containsWord(fields.title, term);
|
|
33941
|
+
const inTags = containsWord(tagText, term) || containsWord(keywordText, term);
|
|
33942
|
+
const inDescription = containsWord(fields.description, term);
|
|
33943
|
+
const inBody = containsWord(fields.searchText, term);
|
|
33944
|
+
const inClaims = containsWord(fields.claimText, term);
|
|
33945
|
+
if (!inTitle && !inTags && !inDescription && !inBody && !inClaims) return 0;
|
|
33946
|
+
if (inTitle) score += 100;
|
|
33947
|
+
if (inTags) score += 40;
|
|
33948
|
+
if (inDescription) score += 20;
|
|
33949
|
+
if (inClaims) score += 10;
|
|
33950
|
+
if (inBody) score += 5;
|
|
33951
|
+
}
|
|
33952
|
+
if (terms.every((term) => containsWord(fields.title, term))) score += 50;
|
|
33953
|
+
return score;
|
|
33660
33954
|
}
|
|
33661
33955
|
async function prepareCommonsEntity(input, user) {
|
|
33662
33956
|
await ensureCommonsSchema();
|
|
@@ -33755,18 +34049,17 @@ async function validateCommonsEntity(input, user) {
|
|
|
33755
34049
|
errors.push(`Unsupported placeholder sections must be omitted instead of published: ${placeholderSections.join(", ")}.`);
|
|
33756
34050
|
}
|
|
33757
34051
|
if (existing && input.baseRevision === void 0) {
|
|
33758
|
-
warnings.push(
|
|
34052
|
+
warnings.push(`An existing entity matches this slug or entityId. Submit will edit it on top of revision ${existing.revision} and record the edit on the contribution ledger.`);
|
|
33759
34053
|
}
|
|
33760
34054
|
if (existing && input.baseRevision !== void 0 && input.baseRevision !== existing.revision) {
|
|
33761
|
-
warnings.push(`The matching entity is revision ${existing.revision}; this payload targets revision ${input.baseRevision}. Submit will
|
|
34055
|
+
warnings.push(`The matching entity is revision ${existing.revision}; this payload targets revision ${input.baseRevision}. Submit will apply on top of revision ${existing.revision} and note the divergence on the ledger.`);
|
|
33762
34056
|
}
|
|
33763
34057
|
if (input.reviewPolicy === "always_review") {
|
|
33764
34058
|
warnings.push("reviewPolicy is always_review, so the proposal will not auto-publish even if the payload is otherwise valid.");
|
|
33765
34059
|
}
|
|
33766
34060
|
if (user?.id == null) warnings.push("No authenticated user was supplied to validation; submit still requires API-key authenticated user context.");
|
|
33767
34061
|
const valid = errors.length === 0;
|
|
33768
|
-
const
|
|
33769
|
-
const autoPublishEligible = valid && revisionAllowsAutoPublish && input.reviewPolicy !== "always_review";
|
|
34062
|
+
const autoPublishEligible = valid && input.reviewPolicy !== "always_review";
|
|
33770
34063
|
const tagVocabulary = await commonsTagVocabulary();
|
|
33771
34064
|
return {
|
|
33772
34065
|
ok: true,
|
|
@@ -33804,7 +34097,10 @@ async function getCommonsEntity(idOrSlug) {
|
|
|
33804
34097
|
`,
|
|
33805
34098
|
args: [lookup, normalizeSlug(lookup)]
|
|
33806
34099
|
});
|
|
33807
|
-
|
|
34100
|
+
if (!result.rows[0]) return null;
|
|
34101
|
+
const entity = rowToEntity(result.rows[0]);
|
|
34102
|
+
await resolveRelatedLinksAgainstPublishedEntities([entity]);
|
|
34103
|
+
return entity;
|
|
33808
34104
|
}
|
|
33809
34105
|
async function classifyCommonsWrite(input, user) {
|
|
33810
34106
|
await ensureCommonsSchema();
|
|
@@ -33826,9 +34122,26 @@ async function submitCommonsEntity(input, user) {
|
|
|
33826
34122
|
entity: entity2,
|
|
33827
34123
|
applied: existingProposal.status === "accepted",
|
|
33828
34124
|
reviewReason: existingProposal.reviewReason,
|
|
34125
|
+
conflictNote: null,
|
|
34126
|
+
droppedImages: [],
|
|
33829
34127
|
publicUrl: entity2?.status === "published" ? publicUrlForSlug(entity2.slug) : null
|
|
33830
34128
|
};
|
|
33831
34129
|
}
|
|
34130
|
+
const hostedImages = await hostEntityImages({
|
|
34131
|
+
userId: Number(user.id),
|
|
34132
|
+
featuredImage: normalized.featuredImage,
|
|
34133
|
+
media: normalized.media
|
|
34134
|
+
});
|
|
34135
|
+
if (hostedImages.featuredImageUrl && normalized.featuredImage) {
|
|
34136
|
+
normalized.featuredImage = { ...normalized.featuredImage, url: hostedImages.featuredImageUrl };
|
|
34137
|
+
}
|
|
34138
|
+
if (hostedImages.mediaUrls.size || hostedImages.dropped.length) {
|
|
34139
|
+
const droppedUrls = new Set(hostedImages.dropped.map((item) => item.url));
|
|
34140
|
+
normalized.media = normalized.media.filter((item) => !droppedUrls.has(item.url)).map((item) => {
|
|
34141
|
+
const hostedUrl = hostedImages.mediaUrls.get(item.url);
|
|
34142
|
+
return hostedUrl ? { ...item, url: hostedUrl } : item;
|
|
34143
|
+
});
|
|
34144
|
+
}
|
|
33832
34145
|
const existing = await findEntityBySlugOrIdAnyStatus(normalized.slug, normalized.entityId);
|
|
33833
34146
|
const existingClaims = existing && normalized.claims !== void 0 ? await getCommonsClaimsByEntityId(existing.id) : [];
|
|
33834
34147
|
const safety = evaluatePublishSafety(normalized, existing);
|
|
@@ -33899,7 +34212,7 @@ async function submitCommonsEntity(input, user) {
|
|
|
33899
34212
|
...changedFieldPaths(existing, nextEntity),
|
|
33900
34213
|
...normalized.claims !== void 0 ? ["claims"] : []
|
|
33901
34214
|
]),
|
|
33902
|
-
normalized.changeSummary
|
|
34215
|
+
ledgerSummary(normalized.changeSummary, existing, safety.conflictNote),
|
|
33903
34216
|
existing ? JSON.stringify({
|
|
33904
34217
|
...entityLedgerSnapshot(existing),
|
|
33905
34218
|
...normalized.claims !== void 0 ? { claims: existingClaims } : {}
|
|
@@ -33922,6 +34235,8 @@ async function submitCommonsEntity(input, user) {
|
|
|
33922
34235
|
entity,
|
|
33923
34236
|
applied: shouldApply,
|
|
33924
34237
|
reviewReason: safety.reason,
|
|
34238
|
+
conflictNote: safety.conflictNote,
|
|
34239
|
+
droppedImages: hostedImages.dropped,
|
|
33925
34240
|
publicUrl: entity?.status === "published" && shouldApply ? publicUrlForSlug(entity.slug) : null
|
|
33926
34241
|
};
|
|
33927
34242
|
}
|
|
@@ -34060,8 +34375,9 @@ async function listCommonsNeedsLinks(filters = {}) {
|
|
|
34060
34375
|
});
|
|
34061
34376
|
const concepts = /* @__PURE__ */ new Map();
|
|
34062
34377
|
const queryTerms = tokenize(cleanFilters.query);
|
|
34063
|
-
|
|
34064
|
-
|
|
34378
|
+
const scanned = result.rows.map((row) => rowToEntity(row));
|
|
34379
|
+
await resolveRelatedLinksAgainstPublishedEntities(scanned);
|
|
34380
|
+
for (const entity of scanned) {
|
|
34065
34381
|
for (const link of entity.articleSections.relatedLinks) {
|
|
34066
34382
|
if (!relatedLinkNeedsPage(link)) continue;
|
|
34067
34383
|
const haystack = [link.title, link.summary, link.relationship, entity.title, entity.description, entity.tags.join(" ")].filter(Boolean).join(" ").toLowerCase();
|
|
@@ -34115,8 +34431,8 @@ function commonsDatabaseReport() {
|
|
|
34115
34431
|
directPublicHtmlWrites: false,
|
|
34116
34432
|
normalWorkflow: ["commons_prepare_entity", "commons_validate_entity", "commons_submit_entity"],
|
|
34117
34433
|
writesCreate: ["commons_proposals", "commons_ledger", "commons_entities when accepted"],
|
|
34118
|
-
safeAutoPublish: "
|
|
34119
|
-
conflictBehavior: "
|
|
34434
|
+
safeAutoPublish: "Any entity with title, description, featured image, and body/source evidence. Creates and edits both publish.",
|
|
34435
|
+
conflictBehavior: "Anyone may edit any entity. A missing or stale baseRevision still applies on top of the current revision; the divergence is recorded on the contribution ledger.",
|
|
34120
34436
|
billing: { validationCredits: 0, createCredits: LISTING_WRITE_RATE_POLICY.create.credits, editCredits: LISTING_WRITE_RATE_POLICY.edit.credits }
|
|
34121
34437
|
},
|
|
34122
34438
|
searchCorpus: ["title", "disambiguationName", "description", "tags", "keywords", "relatedEntities", "relationship claims", "bodyMarkdown", "sections", "media", "source metadata", "jsonLd"],
|
|
@@ -34438,19 +34754,23 @@ function normalizeClaimsForRepository(input) {
|
|
|
34438
34754
|
throw new CommonsRepositoryError("invalid_claims", error instanceof Error ? error.message : "Claims are invalid.");
|
|
34439
34755
|
}
|
|
34440
34756
|
}
|
|
34757
|
+
function ledgerSummary(changeSummary, existing, conflictNote) {
|
|
34758
|
+
const base = changeSummary || (existing ? "Updated Commons entity." : "Created Commons entity.");
|
|
34759
|
+
return conflictNote ? `${base} ${conflictNote}` : base;
|
|
34760
|
+
}
|
|
34441
34761
|
function evaluatePublishSafety(input, existing) {
|
|
34442
|
-
if (!input.featuredImage?.url) return { safe: false, reason: "A featured image is required before a Commons entity can publish." };
|
|
34762
|
+
if (!input.featuredImage?.url) return { safe: false, reason: "A featured image is required before a Commons entity can publish.", conflictNote: null };
|
|
34443
34763
|
if (!input.bodyMarkdown.trim() && !input.source.originalUrl && !input.source.resolvedUrl) {
|
|
34444
|
-
return { safe: false, reason: "A public entity needs article body text or source evidence before publishing." };
|
|
34764
|
+
return { safe: false, reason: "A public entity needs article body text or source evidence before publishing.", conflictNote: null };
|
|
34445
34765
|
}
|
|
34446
|
-
if (!existing) return { safe: true, reason: null };
|
|
34766
|
+
if (!existing) return { safe: true, reason: null, conflictNote: null };
|
|
34447
34767
|
if (input.baseRevision === void 0) {
|
|
34448
|
-
return { safe:
|
|
34768
|
+
return { safe: true, reason: null, conflictNote: `Applied without baseRevision on top of revision ${existing.revision}.` };
|
|
34449
34769
|
}
|
|
34450
34770
|
if (input.baseRevision !== existing.revision) {
|
|
34451
|
-
return { safe:
|
|
34771
|
+
return { safe: true, reason: null, conflictNote: `Applied on top of revision ${existing.revision}; this edit targeted revision ${input.baseRevision}.` };
|
|
34452
34772
|
}
|
|
34453
|
-
return { safe: true, reason: null };
|
|
34773
|
+
return { safe: true, reason: null, conflictNote: null };
|
|
34454
34774
|
}
|
|
34455
34775
|
function buildNextEntity(input) {
|
|
34456
34776
|
const current = input.existing;
|
|
@@ -34765,7 +35085,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
34765
35085
|
return documents;
|
|
34766
35086
|
}
|
|
34767
35087
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
34768
|
-
return `commons-index-${
|
|
35088
|
+
return `commons-index-${createHash15("sha256").update(`${entityId}
|
|
34769
35089
|
${documentType}
|
|
34770
35090
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
34771
35091
|
}
|
|
@@ -35100,6 +35420,44 @@ function sanitizeNeedsLinkFilters(filters) {
|
|
|
35100
35420
|
offset: clampNumber(filters.offset, 0, 1e4, 0)
|
|
35101
35421
|
};
|
|
35102
35422
|
}
|
|
35423
|
+
async function publishedEntityIndexFor(titles) {
|
|
35424
|
+
const index = /* @__PURE__ */ new Map();
|
|
35425
|
+
const slugKeys = uniqueStrings(titles.map((title) => normalizeNeedsLinkTitle(title))).filter(Boolean);
|
|
35426
|
+
const titleKeys = uniqueStrings(titles.map((title) => String(title || "").trim().toLowerCase())).filter(Boolean);
|
|
35427
|
+
if (!slugKeys.length && !titleKeys.length) return index;
|
|
35428
|
+
const result = await getDb().execute({
|
|
35429
|
+
sql: `
|
|
35430
|
+
SELECT id, slug, title
|
|
35431
|
+
FROM commons_entities
|
|
35432
|
+
WHERE status = 'published'
|
|
35433
|
+
AND (slug IN (${slugKeys.map(() => "?").join(", ") || "NULL"})
|
|
35434
|
+
OR lower(title) IN (${titleKeys.map(() => "?").join(", ") || "NULL"}))
|
|
35435
|
+
`,
|
|
35436
|
+
args: [...slugKeys, ...titleKeys]
|
|
35437
|
+
});
|
|
35438
|
+
for (const row of result.rows) {
|
|
35439
|
+
const record = row;
|
|
35440
|
+
const entry = { id: String(record.id), slug: String(record.slug), title: String(record.title) };
|
|
35441
|
+
index.set(entry.slug, entry);
|
|
35442
|
+
index.set(normalizeNeedsLinkTitle(entry.title), entry);
|
|
35443
|
+
}
|
|
35444
|
+
return index;
|
|
35445
|
+
}
|
|
35446
|
+
function unresolvedRelatedLinkTitles(entities) {
|
|
35447
|
+
return entities.flatMap((entity) => entity.articleSections.relatedLinks.filter((link) => relatedLinkNeedsPage(link)).map((link) => link.title));
|
|
35448
|
+
}
|
|
35449
|
+
async function resolveRelatedLinksAgainstPublishedEntities(entities) {
|
|
35450
|
+
const index = await publishedEntityIndexFor(unresolvedRelatedLinkTitles(entities));
|
|
35451
|
+
if (!index.size) return;
|
|
35452
|
+
for (const entity of entities) {
|
|
35453
|
+
entity.articleSections.relatedLinks = entity.articleSections.relatedLinks.map((link) => {
|
|
35454
|
+
if (!relatedLinkNeedsPage(link)) return link;
|
|
35455
|
+
const match = index.get(normalizeNeedsLinkTitle(link.title));
|
|
35456
|
+
if (!match || match.id === entity.id) return link;
|
|
35457
|
+
return { ...link, slug: match.slug, entityId: match.id, needsLink: false };
|
|
35458
|
+
});
|
|
35459
|
+
}
|
|
35460
|
+
}
|
|
35103
35461
|
function relatedLinkNeedsPage(link) {
|
|
35104
35462
|
if (link.needsLink === true) return true;
|
|
35105
35463
|
if (link.entityId || link.slug) return false;
|
|
@@ -35304,7 +35662,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
35304
35662
|
};
|
|
35305
35663
|
|
|
35306
35664
|
// src/api/commons-publication-repository.ts
|
|
35307
|
-
import { createHash as
|
|
35665
|
+
import { createHash as createHash16, randomUUID as randomUUID18 } from "crypto";
|
|
35308
35666
|
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
35309
35667
|
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
35310
35668
|
"admin",
|
|
@@ -35502,7 +35860,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
35502
35860
|
JSON.stringify(editionInput.articles),
|
|
35503
35861
|
html,
|
|
35504
35862
|
rendered.filename,
|
|
35505
|
-
|
|
35863
|
+
createHash16("sha256").update(html).digest("hex"),
|
|
35506
35864
|
rendered.articleCount,
|
|
35507
35865
|
rendered.wordCount,
|
|
35508
35866
|
Buffer.byteLength(html),
|
|
@@ -35939,6 +36297,28 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
35939
36297
|
return publicationError(c, error);
|
|
35940
36298
|
}
|
|
35941
36299
|
});
|
|
36300
|
+
var HostImageSchema = z32.object({
|
|
36301
|
+
sourceUrl: z32.string().url().optional(),
|
|
36302
|
+
imageBase64: z32.string().min(16).optional(),
|
|
36303
|
+
alt: z32.string().max(500).optional(),
|
|
36304
|
+
license: z32.string().max(240).optional(),
|
|
36305
|
+
attribution: z32.string().max(500).optional()
|
|
36306
|
+
}).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
|
|
36307
|
+
message: "Provide either sourceUrl or imageBase64."
|
|
36308
|
+
});
|
|
36309
|
+
commonsApp.post("/images", auth2, async (c) => {
|
|
36310
|
+
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
36311
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
36312
|
+
try {
|
|
36313
|
+
const hosted = await hostCommonsImage({ ...parsed.data, userId: Number(c.get("user").id) });
|
|
36314
|
+
return c.json({ ok: true, data: hosted }, hosted.reused ? 200 : 201);
|
|
36315
|
+
} catch (error) {
|
|
36316
|
+
if (error instanceof CommonsImageError) {
|
|
36317
|
+
return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
|
|
36318
|
+
}
|
|
36319
|
+
throw error;
|
|
36320
|
+
}
|
|
36321
|
+
});
|
|
35942
36322
|
commonsApp.get("/entities", async (c) => {
|
|
35943
36323
|
const result = await searchCommonsEntities(filtersFromQuery(c.req.query()));
|
|
35944
36324
|
return c.json({ ok: true, data: result });
|
|
@@ -35988,7 +36368,15 @@ commonsApp.post("/entities/propose", auth2, async (c) => {
|
|
|
35988
36368
|
if ("insufficient" in charged) return c.json(charged.insufficient, 402);
|
|
35989
36369
|
billing = charged;
|
|
35990
36370
|
const result = await submitCommonsEntity(parsed.data, c.get("user"));
|
|
35991
|
-
|
|
36371
|
+
if (!result.applied) {
|
|
36372
|
+
await refundListingWrite(billing, "Transparent Commons proposal was held for review instead of published.").catch(() => null);
|
|
36373
|
+
return c.json({
|
|
36374
|
+
ok: true,
|
|
36375
|
+
data: result,
|
|
36376
|
+
billing: { ...publicListingWriteBilling(billing), charged: false, refunded: !billing.duplicate }
|
|
36377
|
+
}, 202);
|
|
36378
|
+
}
|
|
36379
|
+
return c.json({ ok: true, data: result, billing: publicListingWriteBilling(billing) }, 201);
|
|
35992
36380
|
} catch (error) {
|
|
35993
36381
|
if (billing) await refundListingWrite(billing, "Transparent Commons write failed before persistence.").catch(() => null);
|
|
35994
36382
|
if (error instanceof CommonsRepositoryError) {
|
|
@@ -36143,12 +36531,12 @@ function xmlEscape(value) {
|
|
|
36143
36531
|
import { Hono as Hono32 } from "hono";
|
|
36144
36532
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
36145
36533
|
import { z as z34 } from "zod";
|
|
36146
|
-
import { createHash as
|
|
36534
|
+
import { createHash as createHash18 } from "crypto";
|
|
36147
36535
|
import Papa2 from "papaparse";
|
|
36148
36536
|
|
|
36149
36537
|
// src/api/analytics-entitlement.ts
|
|
36150
36538
|
import {
|
|
36151
|
-
createHash as
|
|
36539
|
+
createHash as createHash17,
|
|
36152
36540
|
createHmac as createHmac3,
|
|
36153
36541
|
randomBytes as randomBytes4,
|
|
36154
36542
|
timingSafeEqual as timingSafeEqual2
|
|
@@ -36363,7 +36751,7 @@ function dashboardCallbackUrl() {
|
|
|
36363
36751
|
async function createThorbitConnectUrl(userId) {
|
|
36364
36752
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
36365
36753
|
const state = randomBytes4(32).toString("base64url");
|
|
36366
|
-
const stateHash =
|
|
36754
|
+
const stateHash = createHash17("sha256").update(state).digest("hex");
|
|
36367
36755
|
await getAnalyticsPool().query(
|
|
36368
36756
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
36369
36757
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -36392,7 +36780,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
36392
36780
|
return parsed.data.entitlement;
|
|
36393
36781
|
}
|
|
36394
36782
|
async function consumeThorbitConnectCallback(input) {
|
|
36395
|
-
const stateHash =
|
|
36783
|
+
const stateHash = createHash17("sha256").update(input.state).digest("hex");
|
|
36396
36784
|
const client2 = await getAnalyticsPool().connect();
|
|
36397
36785
|
try {
|
|
36398
36786
|
await client2.query("BEGIN");
|
|
@@ -36868,7 +37256,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
|
|
|
36868
37256
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
36869
37257
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
36870
37258
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
36871
|
-
const suffix2 =
|
|
37259
|
+
const suffix2 = createHash18("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
36872
37260
|
const path5 = `Leads/person-${suffix2}`;
|
|
36873
37261
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
36874
37262
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -36967,9 +37355,9 @@ Submitted the published form.
|
|
|
36967
37355
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
36968
37356
|
});
|
|
36969
37357
|
const match = {
|
|
36970
|
-
...email ? { emailSha256:
|
|
37358
|
+
...email ? { emailSha256: createHash18("sha256").update(email).digest("hex") } : {},
|
|
36971
37359
|
...phone ? {
|
|
36972
|
-
phoneSha256:
|
|
37360
|
+
phoneSha256: createHash18("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
36973
37361
|
} : {}
|
|
36974
37362
|
};
|
|
36975
37363
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -37397,7 +37785,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
|
|
|
37397
37785
|
rejectedCount += 1;
|
|
37398
37786
|
continue;
|
|
37399
37787
|
}
|
|
37400
|
-
const digest2 =
|
|
37788
|
+
const digest2 = createHash18("sha256").update(
|
|
37401
37789
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
37402
37790
|
).digest("hex").slice(0, 16);
|
|
37403
37791
|
const path5 = `Leads/person-${digest2}`;
|
|
@@ -37848,13 +38236,13 @@ import { Hono as Hono33 } from "hono";
|
|
|
37848
38236
|
import { z as z35 } from "zod";
|
|
37849
38237
|
|
|
37850
38238
|
// src/api/scheduled-artifact-owner.ts
|
|
37851
|
-
import { createHash as
|
|
38239
|
+
import { createHash as createHash19 } from "crypto";
|
|
37852
38240
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
37853
|
-
return
|
|
38241
|
+
return createHash19("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
37854
38242
|
}
|
|
37855
38243
|
|
|
37856
38244
|
// src/api/scheduled-run-view-links.ts
|
|
37857
|
-
import { createHash as
|
|
38245
|
+
import { createHash as createHash20, randomBytes as randomBytes5, randomUUID as randomUUID19 } from "crypto";
|
|
37858
38246
|
var schemaReady2 = false;
|
|
37859
38247
|
async function ensureScheduledRunViewLinksSchema() {
|
|
37860
38248
|
if (schemaReady2) return;
|
|
@@ -37908,7 +38296,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
37908
38296
|
schemaReady2 = true;
|
|
37909
38297
|
}
|
|
37910
38298
|
function tokenHash2(token6) {
|
|
37911
|
-
return
|
|
38299
|
+
return createHash20("sha256").update(token6).digest("hex");
|
|
37912
38300
|
}
|
|
37913
38301
|
function mapRow(row) {
|
|
37914
38302
|
return {
|
|
@@ -37995,7 +38383,7 @@ async function revokeScheduledRunViewLink(ownerId, runId, shareId, now = /* @__P
|
|
|
37995
38383
|
}
|
|
37996
38384
|
|
|
37997
38385
|
// src/personal-authority/render.ts
|
|
37998
|
-
import { createHash as
|
|
38386
|
+
import { createHash as createHash21 } from "crypto";
|
|
37999
38387
|
function escapeHtml7(value) {
|
|
38000
38388
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38001
38389
|
"&": "&",
|
|
@@ -38055,13 +38443,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
38055
38443
|
html,
|
|
38056
38444
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38057
38445
|
bytes,
|
|
38058
|
-
sha256:
|
|
38446
|
+
sha256: createHash21("sha256").update(html).digest("hex"),
|
|
38059
38447
|
generatedAt: generatedAtIso
|
|
38060
38448
|
};
|
|
38061
38449
|
}
|
|
38062
38450
|
|
|
38063
38451
|
// src/personal-authority/render-v2.ts
|
|
38064
|
-
import { createHash as
|
|
38452
|
+
import { createHash as createHash22 } from "crypto";
|
|
38065
38453
|
var FONT_STACKS = Object.freeze({
|
|
38066
38454
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
38067
38455
|
"modern-sans": "Avenir Next, Avenir, Helvetica Neue, Arial, sans-serif",
|
|
@@ -38165,13 +38553,13 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
38165
38553
|
html,
|
|
38166
38554
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
38167
38555
|
bytes,
|
|
38168
|
-
sha256:
|
|
38556
|
+
sha256: createHash22("sha256").update(html).digest("hex"),
|
|
38169
38557
|
generatedAt: generatedAtIso
|
|
38170
38558
|
};
|
|
38171
38559
|
}
|
|
38172
38560
|
|
|
38173
38561
|
// src/newsroom-publisher/render.ts
|
|
38174
|
-
import { createHash as
|
|
38562
|
+
import { createHash as createHash23 } from "crypto";
|
|
38175
38563
|
function escapeHtml9(value) {
|
|
38176
38564
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
38177
38565
|
"&": "&",
|
|
@@ -38277,13 +38665,13 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
38277
38665
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
38278
38666
|
</style></head><body data-theme="${escapeHtml9(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
38279
38667
|
const bytes = Buffer.byteLength(html);
|
|
38280
|
-
const sha2563 =
|
|
38668
|
+
const sha2563 = createHash23("sha256").update(html).digest("hex");
|
|
38281
38669
|
const filename = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
38282
38670
|
return { html, filename, bytes, sha256: sha2563, generatedAt: generatedAtIso };
|
|
38283
38671
|
}
|
|
38284
38672
|
|
|
38285
38673
|
// src/blog-article/render.ts
|
|
38286
|
-
import { createHash as
|
|
38674
|
+
import { createHash as createHash24 } from "crypto";
|
|
38287
38675
|
function escapeHtml10(value) {
|
|
38288
38676
|
return value.replace(
|
|
38289
38677
|
/[&<>"']/g,
|
|
@@ -38371,7 +38759,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
38371
38759
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
38372
38760
|
</style></head><body data-theme="${escapeHtml10(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml10(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml10(input.eyebrow)}</span><h1>${escapeHtml10(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml10(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml10(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml10(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml10(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
38373
38761
|
const bytes = Buffer.byteLength(html);
|
|
38374
|
-
const sha2563 =
|
|
38762
|
+
const sha2563 = createHash24("sha256").update(html).digest("hex");
|
|
38375
38763
|
return {
|
|
38376
38764
|
html,
|
|
38377
38765
|
filename: "blog-article.html",
|
|
@@ -38674,7 +39062,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
38674
39062
|
import { Hono as Hono34 } from "hono";
|
|
38675
39063
|
|
|
38676
39064
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
38677
|
-
import { createHash as
|
|
39065
|
+
import { createHash as createHash25 } from "crypto";
|
|
38678
39066
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
38679
39067
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
38680
39068
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -38690,7 +39078,7 @@ function policy4() {
|
|
|
38690
39078
|
};
|
|
38691
39079
|
}
|
|
38692
39080
|
function runStorageSegment(runId) {
|
|
38693
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
39081
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash25("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
38694
39082
|
}
|
|
38695
39083
|
async function createScheduledRunArtifact(args) {
|
|
38696
39084
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -38797,7 +39185,7 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
38797
39185
|
import { Hono as Hono35 } from "hono";
|
|
38798
39186
|
|
|
38799
39187
|
// src/api/scheduler-integration-auth.ts
|
|
38800
|
-
import { createHash as
|
|
39188
|
+
import { createHash as createHash26, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
38801
39189
|
|
|
38802
39190
|
// src/api/service-connections.ts
|
|
38803
39191
|
import { randomUUID as randomUUID20 } from "crypto";
|
|
@@ -39095,7 +39483,7 @@ function signingSecret() {
|
|
|
39095
39483
|
return secret;
|
|
39096
39484
|
}
|
|
39097
39485
|
function schedulerIntegrationSignature(args) {
|
|
39098
|
-
const bodyHash =
|
|
39486
|
+
const bodyHash = createHash26("sha256").update(args.body).digest("hex");
|
|
39099
39487
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
39100
39488
|
${args.path}
|
|
39101
39489
|
${args.timestamp}
|
|
@@ -39287,7 +39675,7 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
39287
39675
|
}
|
|
39288
39676
|
|
|
39289
39677
|
// src/api/page-media-artifacts.ts
|
|
39290
|
-
import { createHash as
|
|
39678
|
+
import { createHash as createHash27, randomBytes as randomBytes6 } from "crypto";
|
|
39291
39679
|
import { homedir as homedir3 } from "os";
|
|
39292
39680
|
import { basename, extname, join as join8 } from "path";
|
|
39293
39681
|
import pLimit6 from "p-limit";
|
|
@@ -39434,7 +39822,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
39434
39822
|
finalUrl: checked.parsed.href,
|
|
39435
39823
|
width: dimensions?.width ?? null,
|
|
39436
39824
|
height: dimensions?.height ?? null,
|
|
39437
|
-
sha256:
|
|
39825
|
+
sha256: createHash27("sha256").update(bytes).digest("hex")
|
|
39438
39826
|
};
|
|
39439
39827
|
}
|
|
39440
39828
|
throw new Error("media_redirect_rejected");
|
|
@@ -39627,13 +40015,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
39627
40015
|
}
|
|
39628
40016
|
|
|
39629
40017
|
// src/api/page-diff.ts
|
|
39630
|
-
import { createHash as
|
|
40018
|
+
import { createHash as createHash28 } from "crypto";
|
|
39631
40019
|
import { diffLines } from "diff";
|
|
39632
40020
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
39633
40021
|
var MAX_DIFF_HUNKS = 200;
|
|
39634
40022
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
39635
40023
|
function sha256Hex(value) {
|
|
39636
|
-
return
|
|
40024
|
+
return createHash28("sha256").update(value).digest("hex");
|
|
39637
40025
|
}
|
|
39638
40026
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
39639
40027
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -39747,10 +40135,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
39747
40135
|
}
|
|
39748
40136
|
|
|
39749
40137
|
// src/api/scrape-image-sink.ts
|
|
39750
|
-
import { createHash as
|
|
40138
|
+
import { createHash as createHash29 } from "crypto";
|
|
39751
40139
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
39752
40140
|
function idempotencyKey2(userId, vault, input) {
|
|
39753
|
-
return `scrape-image-${
|
|
40141
|
+
return `scrape-image-${createHash29("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
39754
40142
|
}
|
|
39755
40143
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
39756
40144
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -40222,7 +40610,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
40222
40610
|
}
|
|
40223
40611
|
|
|
40224
40612
|
// src/api/connection-memory-import.ts
|
|
40225
|
-
import { createHash as
|
|
40613
|
+
import { createHash as createHash30 } from "crypto";
|
|
40226
40614
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
40227
40615
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
40228
40616
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -40252,7 +40640,7 @@ function canonicalJson(value) {
|
|
|
40252
40640
|
return JSON.stringify(value);
|
|
40253
40641
|
}
|
|
40254
40642
|
function sha2562(value) {
|
|
40255
|
-
return
|
|
40643
|
+
return createHash30("sha256").update(value).digest("hex");
|
|
40256
40644
|
}
|
|
40257
40645
|
function sensitiveKey(key) {
|
|
40258
40646
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -41263,7 +41651,7 @@ async function dbUsage(identity, plan) {
|
|
|
41263
41651
|
}
|
|
41264
41652
|
|
|
41265
41653
|
// src/api/nango-control.ts
|
|
41266
|
-
import { createHash as
|
|
41654
|
+
import { createHash as createHash32, randomUUID as randomUUID24 } from "crypto";
|
|
41267
41655
|
|
|
41268
41656
|
// src/api/slack-archive-analysis.ts
|
|
41269
41657
|
function isRecord2(value) {
|
|
@@ -41637,7 +42025,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
41637
42025
|
}
|
|
41638
42026
|
|
|
41639
42027
|
// src/api/main-nango-transport.ts
|
|
41640
|
-
import { createHash as
|
|
42028
|
+
import { createHash as createHash31, randomUUID as randomUUID23 } from "crypto";
|
|
41641
42029
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
41642
42030
|
|
|
41643
42031
|
// src/api/connected-tool-policy.ts
|
|
@@ -42030,7 +42418,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
42030
42418
|
});
|
|
42031
42419
|
const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
|
|
42032
42420
|
const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
|
|
42033
|
-
const revision =
|
|
42421
|
+
const revision = createHash31("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
42034
42422
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
42035
42423
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
42036
42424
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -42058,7 +42446,7 @@ async function callNangoToolDirect(args) {
|
|
|
42058
42446
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
42059
42447
|
});
|
|
42060
42448
|
const requestId = args.requestId?.trim() || randomUUID23();
|
|
42061
|
-
const idempotencyKey3 = `main-nango:${
|
|
42449
|
+
const idempotencyKey3 = `main-nango:${createHash31("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
42062
42450
|
const startedAt = /* @__PURE__ */ new Date();
|
|
42063
42451
|
const started = performance.now();
|
|
42064
42452
|
let result;
|
|
@@ -42083,7 +42471,7 @@ async function callNangoToolDirect(args) {
|
|
|
42083
42471
|
toolName: args.tool,
|
|
42084
42472
|
operationKind: args.operationKind ?? args.classification,
|
|
42085
42473
|
outcome: providerError ? "error" : "partial",
|
|
42086
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
42474
|
+
requestId: requestId.length <= 200 ? requestId : createHash31("sha256").update(requestId).digest("hex"),
|
|
42087
42475
|
startedAt: startedAt.toISOString(),
|
|
42088
42476
|
completedAt: completedAt.toISOString()
|
|
42089
42477
|
}
|
|
@@ -42125,7 +42513,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
42125
42513
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
42126
42514
|
protocolVersion: null,
|
|
42127
42515
|
schemaSource: "live_tools_list",
|
|
42128
|
-
schemaHash:
|
|
42516
|
+
schemaHash: createHash31("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
42129
42517
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
42130
42518
|
};
|
|
42131
42519
|
}
|
|
@@ -42849,8 +43237,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
42849
43237
|
return data.connection.actionsEnabled === true;
|
|
42850
43238
|
}
|
|
42851
43239
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
42852
|
-
const requestId = `main-connected-action:${
|
|
42853
|
-
const requestDigest =
|
|
43240
|
+
const requestId = `main-connected-action:${createHash32("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID24()).digest("hex")}`;
|
|
43241
|
+
const requestDigest = createHash32("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
42854
43242
|
if (mainOwnsIntegrations()) {
|
|
42855
43243
|
const selectedTool = tool?.trim();
|
|
42856
43244
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -43001,7 +43389,7 @@ function canonicalJson2(value) {
|
|
|
43001
43389
|
return JSON.stringify(value);
|
|
43002
43390
|
}
|
|
43003
43391
|
function projectedToolSchemaHash(tool) {
|
|
43004
|
-
return
|
|
43392
|
+
return createHash32("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
43005
43393
|
}
|
|
43006
43394
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
43007
43395
|
if (mainOwnsIntegrations()) {
|
|
@@ -43280,7 +43668,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
43280
43668
|
}
|
|
43281
43669
|
|
|
43282
43670
|
// src/api/resend-control.ts
|
|
43283
|
-
import { createHash as
|
|
43671
|
+
import { createHash as createHash33 } from "crypto";
|
|
43284
43672
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
43285
43673
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
43286
43674
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -43539,7 +43927,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
43539
43927
|
return isRecord5(data) ? data.result ?? data : data;
|
|
43540
43928
|
}
|
|
43541
43929
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
43542
|
-
const requestId = `main-resend-action:${
|
|
43930
|
+
const requestId = `main-resend-action:${createHash33("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
43543
43931
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
43544
43932
|
method: "POST",
|
|
43545
43933
|
headers: { "x-request-id": requestId },
|
|
@@ -46765,7 +47153,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
46765
47153
|
return c.json({ error: "Unauthorized" }, 401);
|
|
46766
47154
|
}
|
|
46767
47155
|
const startedAt = Date.now();
|
|
46768
|
-
const { drainQueue } = await import("./worker-
|
|
47156
|
+
const { drainQueue } = await import("./worker-S2WXELUC.js");
|
|
46769
47157
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
46770
47158
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
46771
47159
|
const unfinished = [];
|
|
@@ -46835,8 +47223,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46835
47223
|
finishExtractJob: finishExtractJob2,
|
|
46836
47224
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
46837
47225
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
46838
|
-
} = await import("./site-extract-repository-
|
|
46839
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
47226
|
+
} = await import("./site-extract-repository-CFMSWAVL.js");
|
|
47227
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-H7JEELY7.js");
|
|
46840
47228
|
const existing = await getExtractJob2(jobId2);
|
|
46841
47229
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
46842
47230
|
if (existing.status !== "failed") {
|
|
@@ -46863,7 +47251,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
46863
47251
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
46864
47252
|
let settlement = "already_settled_or_refunded";
|
|
46865
47253
|
if (job.billedMc == null && job.userId != null) {
|
|
46866
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
47254
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CFMSWAVL.js");
|
|
46867
47255
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
46868
47256
|
const successful = await countSuccessfulPages2(jobId2);
|
|
46869
47257
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -47048,4 +47436,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
47048
47436
|
export {
|
|
47049
47437
|
app
|
|
47050
47438
|
};
|
|
47051
|
-
//# sourceMappingURL=server-
|
|
47439
|
+
//# sourceMappingURL=server-NBGZJX4P.js.map
|