mcp-scraper 0.66.7 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +246 -41
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +56 -12
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.cjs +6 -0
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-F434YJPV.js → chunk-2L4C4DAZ.js} +1 -1
- package/dist/{chunk-LVXRWJJJ.js → chunk-2LXTOAKS.js} +1 -1
- package/dist/{chunk-CGOMGGOF.js → chunk-62LEXS5O.js} +2 -2
- package/dist/{chunk-HZDJ5UZ4.js → chunk-65FTMB7G.js} +25 -1
- package/dist/{chunk-U2UHJWFW.js → chunk-ES25GP6C.js} +1 -1
- package/dist/{chunk-7KB5QTHP.js → chunk-GT63KEUB.js} +1 -1
- package/dist/{chunk-O7ZC5Y3Z.js → chunk-IHDTONZV.js} +1 -1
- package/dist/{chunk-HWGQVP3Q.js → chunk-LCGTNWOT.js} +1 -1
- package/dist/{chunk-EVOBJYBE.js → chunk-LN6N3YLK.js} +1 -1
- package/dist/{chunk-JZPSG4NR.js → chunk-UL4ZKAWZ.js} +1 -1
- package/dist/{chunk-Z5QQEWXA.js → chunk-XOOKZYZG.js} +33 -13
- package/dist/{chunk-HBZENASD.js → chunk-Y46YNQMM.js} +1 -1
- package/dist/{db-MNNNIWVS.js → db-ZGZAKYBW.js} +1 -1
- package/dist/{extract-bundle-RW7CVIRF.js → extract-bundle-QQIYJSBW.js} +3 -3
- package/dist/index.cjs +6 -0
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-PZYE6TDP.js → lead-list-enrichment-repository-CXLVIV3R.js} +2 -2
- package/dist/{location-data-repository-EPCNGMOM.js → location-data-repository-G72M7DTN.js} +2 -2
- package/dist/{server-KSVJ4X4V.js → server-MFVYRB6I.js} +206 -48
- package/dist/{site-extract-repository-CZ7SVT3Z.js → site-extract-repository-7EP6UX6I.js} +2 -2
- package/dist/{worker-IBLGV272.js → worker-JGWV2LID.js} +5 -5
- package/package.json +1 -1
|
@@ -10,13 +10,13 @@ import {
|
|
|
10
10
|
queuedCommonsDocumentCount,
|
|
11
11
|
semanticCommonsEntityScores,
|
|
12
12
|
typeFromMime
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-LN6N3YLK.js";
|
|
14
14
|
import {
|
|
15
15
|
REQUIRED_LOCATION_STATE_CODES,
|
|
16
16
|
importHostedCensusPlacesCsv,
|
|
17
17
|
importHostedZipGroupsCsv,
|
|
18
18
|
listActiveHostedLocationDatasets
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-Y46YNQMM.js";
|
|
20
20
|
import {
|
|
21
21
|
cancelPendingLeadListEnrichmentRows,
|
|
22
22
|
claimLeadListEnrichmentOutbox,
|
|
@@ -43,7 +43,7 @@ import {
|
|
|
43
43
|
saveLeadListEnrichmentRowResult,
|
|
44
44
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
45
45
|
stableCanonicalJson
|
|
46
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-UL4ZKAWZ.js";
|
|
47
47
|
import {
|
|
48
48
|
ANALYTICS_CONTENT_SORTS,
|
|
49
49
|
AnalyticsRepositoryError,
|
|
@@ -108,7 +108,7 @@ import {
|
|
|
108
108
|
harvestProblemResponse,
|
|
109
109
|
publicErrorBoundary,
|
|
110
110
|
serializeHarvestProblem
|
|
111
|
-
} from "./chunk-
|
|
111
|
+
} from "./chunk-2LXTOAKS.js";
|
|
112
112
|
import "./chunk-OZJMVCDK.js";
|
|
113
113
|
import {
|
|
114
114
|
BACKUP_LAST_RESORT_ATTEMPTS,
|
|
@@ -127,7 +127,7 @@ import {
|
|
|
127
127
|
resolveKernelProxyId,
|
|
128
128
|
serpArrivalUrl,
|
|
129
129
|
withRuntimeUserAgent
|
|
130
|
-
} from "./chunk-
|
|
130
|
+
} from "./chunk-62LEXS5O.js";
|
|
131
131
|
import {
|
|
132
132
|
ApiErrorResponseSchema,
|
|
133
133
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -199,7 +199,7 @@ import {
|
|
|
199
199
|
resolveDeploymentProfile,
|
|
200
200
|
resolveLocalSourcebookSchemaType,
|
|
201
201
|
transcribeMediaUrl
|
|
202
|
-
} from "./chunk-
|
|
202
|
+
} from "./chunk-XOOKZYZG.js";
|
|
203
203
|
import {
|
|
204
204
|
auditImageUrls,
|
|
205
205
|
auditImages,
|
|
@@ -210,7 +210,7 @@ import {
|
|
|
210
210
|
sanitizeAttempts,
|
|
211
211
|
sanitizeHarvestResult,
|
|
212
212
|
sanitizeOutboundDiagnostics
|
|
213
|
-
} from "./chunk-
|
|
213
|
+
} from "./chunk-2L4C4DAZ.js";
|
|
214
214
|
import {
|
|
215
215
|
browserServiceApiKey,
|
|
216
216
|
browserServiceProfileName,
|
|
@@ -220,7 +220,7 @@ import {
|
|
|
220
220
|
recordKernelSession,
|
|
221
221
|
recordVendorUsage,
|
|
222
222
|
runWithCostContext
|
|
223
|
-
} from "./chunk-
|
|
223
|
+
} from "./chunk-ES25GP6C.js";
|
|
224
224
|
import {
|
|
225
225
|
listWorkflowDefinitions,
|
|
226
226
|
runWorkflow,
|
|
@@ -256,7 +256,7 @@ import {
|
|
|
256
256
|
} from "./chunk-GGZEC22A.js";
|
|
257
257
|
import {
|
|
258
258
|
PACKAGE_VERSION
|
|
259
|
-
} from "./chunk-
|
|
259
|
+
} from "./chunk-GT63KEUB.js";
|
|
260
260
|
import {
|
|
261
261
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
262
262
|
abandonExtractSettlement,
|
|
@@ -287,7 +287,7 @@ import {
|
|
|
287
287
|
setExtractJobTotal,
|
|
288
288
|
settleExtractJob,
|
|
289
289
|
terminalExtractJobStatus
|
|
290
|
-
} from "./chunk-
|
|
290
|
+
} from "./chunk-LCGTNWOT.js";
|
|
291
291
|
import {
|
|
292
292
|
createPrivateArtifact,
|
|
293
293
|
privateArtifactOwnerId,
|
|
@@ -451,7 +451,7 @@ import {
|
|
|
451
451
|
softDeleteInboxMessage,
|
|
452
452
|
upsertPageSnapshot,
|
|
453
453
|
verifyPassword
|
|
454
|
-
} from "./chunk-
|
|
454
|
+
} from "./chunk-65FTMB7G.js";
|
|
455
455
|
|
|
456
456
|
// src/blog/registry.ts
|
|
457
457
|
var posts = [
|
|
@@ -6312,7 +6312,8 @@ function firstJsonLdImage(value) {
|
|
|
6312
6312
|
}
|
|
6313
6313
|
return null;
|
|
6314
6314
|
}
|
|
6315
|
-
function
|
|
6315
|
+
function contentImages(html) {
|
|
6316
|
+
const images = [];
|
|
6316
6317
|
for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
|
|
6317
6318
|
const tag = match[0];
|
|
6318
6319
|
const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
|
|
@@ -6323,30 +6324,36 @@ function firstContentImage(html) {
|
|
|
6323
6324
|
const width = Number(attr("width") ?? 0);
|
|
6324
6325
|
const height = Number(attr("height") ?? 0);
|
|
6325
6326
|
if (width > 0 && width < 200 || height > 0 && height < 120) continue;
|
|
6326
|
-
|
|
6327
|
+
images.push(candidate);
|
|
6327
6328
|
}
|
|
6328
|
-
return
|
|
6329
|
+
return images;
|
|
6329
6330
|
}
|
|
6330
|
-
function
|
|
6331
|
+
function resolveFeaturedImageCandidates(input) {
|
|
6331
6332
|
const replay = parseWaybackReplayUrl(input.pageUrl);
|
|
6332
6333
|
const baseUrl = replay?.originalUrl ?? input.pageUrl;
|
|
6333
6334
|
const candidates = [
|
|
6334
6335
|
{ value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
|
|
6335
6336
|
{ value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
|
|
6336
6337
|
{ value: firstJsonLdImage(input.schema), source: "json-ld" },
|
|
6337
|
-
|
|
6338
|
+
...contentImages(input.html).map((value) => ({ value, source: "content-image" }))
|
|
6338
6339
|
];
|
|
6340
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6341
|
+
const resolved = [];
|
|
6339
6342
|
for (const candidate of candidates) {
|
|
6340
6343
|
if (!candidate.value) continue;
|
|
6341
6344
|
const url = safeHttpUrl(candidate.value, baseUrl);
|
|
6342
|
-
if (!url) continue;
|
|
6343
|
-
|
|
6345
|
+
if (!url || seen.has(url)) continue;
|
|
6346
|
+
seen.add(url);
|
|
6347
|
+
resolved.push({
|
|
6344
6348
|
url,
|
|
6345
6349
|
archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
|
|
6346
6350
|
source: candidate.source
|
|
6347
|
-
};
|
|
6351
|
+
});
|
|
6348
6352
|
}
|
|
6349
|
-
return
|
|
6353
|
+
return resolved;
|
|
6354
|
+
}
|
|
6355
|
+
function resolveFeaturedImage(input) {
|
|
6356
|
+
return resolveFeaturedImageCandidates(input)[0] ?? null;
|
|
6350
6357
|
}
|
|
6351
6358
|
|
|
6352
6359
|
// src/api/site-extractor.ts
|
|
@@ -9545,7 +9552,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9545
9552
|
});
|
|
9546
9553
|
}) : null;
|
|
9547
9554
|
const finalized = await step.run("finalize", async () => {
|
|
9548
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9555
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
|
|
9549
9556
|
const current = await getExtractJob(jobId2);
|
|
9550
9557
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9551
9558
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10505,7 +10512,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
10505
10512
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
10506
10513
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
10507
10514
|
if (mode === "hosted") {
|
|
10508
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
10515
|
+
const { getHostedZipGroups } = await import("./location-data-repository-G72M7DTN.js");
|
|
10509
10516
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
10510
10517
|
if (!hosted.dataset) {
|
|
10511
10518
|
throw new DirectoryLocationError(
|
|
@@ -10568,7 +10575,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10568
10575
|
const sourceUrl = censusStateUrl(state.fips);
|
|
10569
10576
|
const warnings = [];
|
|
10570
10577
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
10571
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
10578
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-G72M7DTN.js");
|
|
10572
10579
|
const hosted = await queryHostedLocationMarkets({
|
|
10573
10580
|
state: state.abbr,
|
|
10574
10581
|
minPopulation: options.minPopulation,
|
|
@@ -12055,7 +12062,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
|
|
|
12055
12062
|
async function settlePersistedLeadListEnrichment(jobId2) {
|
|
12056
12063
|
const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
|
|
12057
12064
|
const job = jobs.find((candidate) => candidate.id === jobId2);
|
|
12058
|
-
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12065
|
+
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
|
|
12059
12066
|
const current = job ?? await getLeadListEnrichmentJob2(jobId2);
|
|
12060
12067
|
if (!current) return null;
|
|
12061
12068
|
const rows = await listLeadListEnrichmentRows(jobId2);
|
|
@@ -12111,7 +12118,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
|
|
|
12111
12118
|
const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
|
|
12112
12119
|
const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
|
|
12113
12120
|
if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
|
|
12114
|
-
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12121
|
+
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
|
|
12115
12122
|
await failLeadListEnrichmentJob2({
|
|
12116
12123
|
jobId: job.id,
|
|
12117
12124
|
errorCode: "lead_list_all_rows_failed",
|
|
@@ -35066,7 +35073,9 @@ import { isIP } from "net";
|
|
|
35066
35073
|
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
35067
35074
|
var COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
35068
35075
|
var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
35069
|
-
var
|
|
35076
|
+
var COMMONS_IMAGE_PAGE_MAX_BYTES = 2 * 1024 * 1024;
|
|
35077
|
+
var COMMONS_IMAGE_MAX_PAGE_CANDIDATES = 12;
|
|
35078
|
+
var COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with imageBase64, an owner-authorized artifactId, a public HTTPS image URL, or a public HTTPS webpage containing a usable featured image. Replace featuredImage.url with the returned permanent URL, validate again, then retry commons_submit_entity with the same idempotency key.";
|
|
35070
35079
|
var CommonsImageError = class extends Error {
|
|
35071
35080
|
constructor(code, message, httpStatus = 400, retryable = false, retryGuidance = COMMONS_IMAGE_RETRY_GUIDANCE) {
|
|
35072
35081
|
super(message);
|
|
@@ -35139,6 +35148,9 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
35139
35148
|
if (url.protocol !== "https:") {
|
|
35140
35149
|
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
35141
35150
|
}
|
|
35151
|
+
if (url.username || url.password) {
|
|
35152
|
+
throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
|
|
35153
|
+
}
|
|
35142
35154
|
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
35143
35155
|
const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
|
|
35144
35156
|
if (!addresses.length) {
|
|
@@ -35149,7 +35161,28 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
35149
35161
|
}
|
|
35150
35162
|
return url;
|
|
35151
35163
|
}
|
|
35152
|
-
async function
|
|
35164
|
+
async function readResponseBytes(response, maxBytes) {
|
|
35165
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
35166
|
+
if (declaredLength > maxBytes) {
|
|
35167
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
35168
|
+
}
|
|
35169
|
+
if (!response.body) return Buffer.alloc(0);
|
|
35170
|
+
const reader = response.body.getReader();
|
|
35171
|
+
const chunks = [];
|
|
35172
|
+
let total = 0;
|
|
35173
|
+
while (true) {
|
|
35174
|
+
const { done, value } = await reader.read();
|
|
35175
|
+
if (done) break;
|
|
35176
|
+
total += value.byteLength;
|
|
35177
|
+
if (total > maxBytes) {
|
|
35178
|
+
await reader.cancel().catch(() => void 0);
|
|
35179
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
35180
|
+
}
|
|
35181
|
+
chunks.push(Buffer.from(value));
|
|
35182
|
+
}
|
|
35183
|
+
return Buffer.concat(chunks, total);
|
|
35184
|
+
}
|
|
35185
|
+
async function downloadPublicSource(sourceUrl) {
|
|
35153
35186
|
let current = sourceUrl;
|
|
35154
35187
|
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
35155
35188
|
const url = await assertPublicHttpsUrl(current);
|
|
@@ -35184,15 +35217,11 @@ async function downloadImage(sourceUrl) {
|
|
|
35184
35217
|
retryable
|
|
35185
35218
|
);
|
|
35186
35219
|
}
|
|
35187
|
-
|
|
35188
|
-
|
|
35189
|
-
|
|
35190
|
-
|
|
35191
|
-
|
|
35192
|
-
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
35193
|
-
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
35194
|
-
}
|
|
35195
|
-
return bytes;
|
|
35220
|
+
return {
|
|
35221
|
+
bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
|
|
35222
|
+
finalUrl: url.toString(),
|
|
35223
|
+
contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
|
|
35224
|
+
};
|
|
35196
35225
|
}
|
|
35197
35226
|
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
35198
35227
|
}
|
|
@@ -35205,6 +35234,114 @@ function decodeBase64Image(value) {
|
|
|
35205
35234
|
}
|
|
35206
35235
|
return bytes;
|
|
35207
35236
|
}
|
|
35237
|
+
function safeProvenanceUrl(rawUrl) {
|
|
35238
|
+
try {
|
|
35239
|
+
const url = new URL(rawUrl);
|
|
35240
|
+
url.username = "";
|
|
35241
|
+
url.password = "";
|
|
35242
|
+
url.search = "";
|
|
35243
|
+
url.hash = "";
|
|
35244
|
+
return url.toString();
|
|
35245
|
+
} catch {
|
|
35246
|
+
return null;
|
|
35247
|
+
}
|
|
35248
|
+
}
|
|
35249
|
+
function looksLikeHtml(bytes, declaredType) {
|
|
35250
|
+
if (declaredType === "text/html" || declaredType === "application/xhtml+xml") return true;
|
|
35251
|
+
const prefix = bytes.subarray(0, 512).toString("utf8").trimStart().toLowerCase();
|
|
35252
|
+
return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
|
|
35253
|
+
}
|
|
35254
|
+
async function resolveCommonsImageSource(input) {
|
|
35255
|
+
const supplied = [input.sourceUrl, input.imageBase64, input.artifactId].filter((value) => Boolean(value)).length;
|
|
35256
|
+
if (supplied === 0) {
|
|
35257
|
+
throw new CommonsImageError("image_source_missing", "Provide exactly one of sourceUrl, imageBase64, or artifactId.");
|
|
35258
|
+
}
|
|
35259
|
+
if (supplied !== 1) {
|
|
35260
|
+
throw new CommonsImageError("image_source_ambiguous", "Provide only one image source: sourceUrl, imageBase64, or artifactId.");
|
|
35261
|
+
}
|
|
35262
|
+
if (input.sourceUrl && input.sourceType) {
|
|
35263
|
+
throw new CommonsImageError("image_source_type_invalid", "sourceType applies only to imageBase64 or artifactId, not sourceUrl.");
|
|
35264
|
+
}
|
|
35265
|
+
if (input.imageBase64) {
|
|
35266
|
+
return {
|
|
35267
|
+
bytes: decodeBase64Image(input.imageBase64),
|
|
35268
|
+
sourceUrl: null,
|
|
35269
|
+
sourcePageUrl: null,
|
|
35270
|
+
sourceKind: input.sourceType ?? "uploaded",
|
|
35271
|
+
selectionSource: "provided_bytes"
|
|
35272
|
+
};
|
|
35273
|
+
}
|
|
35274
|
+
if (input.artifactId) {
|
|
35275
|
+
const bytes = await readOwnedImageSourceArtifact({ artifactId: input.artifactId, ownerId: String(input.userId) });
|
|
35276
|
+
if (!bytes) {
|
|
35277
|
+
throw new CommonsImageError("image_artifact_not_found", "The image artifact was not found, expired, or is not owned by this account.", 422);
|
|
35278
|
+
}
|
|
35279
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
35280
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
35281
|
+
}
|
|
35282
|
+
return {
|
|
35283
|
+
bytes,
|
|
35284
|
+
sourceUrl: null,
|
|
35285
|
+
sourcePageUrl: null,
|
|
35286
|
+
sourceKind: input.sourceType ?? "artifact",
|
|
35287
|
+
selectionSource: "artifact"
|
|
35288
|
+
};
|
|
35289
|
+
}
|
|
35290
|
+
const requestedUrl = String(input.sourceUrl);
|
|
35291
|
+
const downloaded = await downloadPublicSource(requestedUrl);
|
|
35292
|
+
if (sniffImageFormat(downloaded.bytes)) {
|
|
35293
|
+
return {
|
|
35294
|
+
bytes: downloaded.bytes,
|
|
35295
|
+
sourceUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
35296
|
+
sourcePageUrl: null,
|
|
35297
|
+
sourceKind: "direct_url",
|
|
35298
|
+
selectionSource: "direct_url"
|
|
35299
|
+
};
|
|
35300
|
+
}
|
|
35301
|
+
if (!looksLikeHtml(downloaded.bytes, downloaded.contentType)) {
|
|
35302
|
+
throw new CommonsImageError(
|
|
35303
|
+
"image_format_unsupported",
|
|
35304
|
+
"The supplied URL returned neither a supported image nor an HTML webpage with a discoverable image.",
|
|
35305
|
+
415
|
|
35306
|
+
);
|
|
35307
|
+
}
|
|
35308
|
+
if (downloaded.bytes.length > COMMONS_IMAGE_PAGE_MAX_BYTES) {
|
|
35309
|
+
throw new CommonsImageError("image_page_too_large", `Webpage exceeds the ${COMMONS_IMAGE_PAGE_MAX_BYTES} byte discovery limit.`, 413);
|
|
35310
|
+
}
|
|
35311
|
+
const html = downloaded.bytes.toString("utf8");
|
|
35312
|
+
const document2 = loadHtmlDocument(html);
|
|
35313
|
+
const candidates = resolveFeaturedImageCandidates({
|
|
35314
|
+
html,
|
|
35315
|
+
meta: document2.meta,
|
|
35316
|
+
schema: document2.schema,
|
|
35317
|
+
pageUrl: downloaded.finalUrl
|
|
35318
|
+
}).slice(0, COMMONS_IMAGE_MAX_PAGE_CANDIDATES);
|
|
35319
|
+
if (!candidates.length) {
|
|
35320
|
+
throw new CommonsImageError("page_image_not_found", "The webpage did not expose a usable featured or content image.", 422);
|
|
35321
|
+
}
|
|
35322
|
+
let sawRetryableFailure = false;
|
|
35323
|
+
for (const candidate of candidates) {
|
|
35324
|
+
try {
|
|
35325
|
+
const image = await downloadPublicSource(candidate.url);
|
|
35326
|
+
if (!sniffImageFormat(image.bytes)) continue;
|
|
35327
|
+
return {
|
|
35328
|
+
bytes: image.bytes,
|
|
35329
|
+
sourceUrl: safeProvenanceUrl(image.finalUrl),
|
|
35330
|
+
sourcePageUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
35331
|
+
sourceKind: "webpage",
|
|
35332
|
+
selectionSource: candidate.source
|
|
35333
|
+
};
|
|
35334
|
+
} catch (error) {
|
|
35335
|
+
if (error instanceof CommonsImageError && error.retryable) sawRetryableFailure = true;
|
|
35336
|
+
}
|
|
35337
|
+
}
|
|
35338
|
+
throw new CommonsImageError(
|
|
35339
|
+
"page_image_candidates_unusable",
|
|
35340
|
+
"The webpage exposed image candidates, but none returned supported public image bytes.",
|
|
35341
|
+
422,
|
|
35342
|
+
sawRetryableFailure
|
|
35343
|
+
);
|
|
35344
|
+
}
|
|
35208
35345
|
function commonsImageBlobStore() {
|
|
35209
35346
|
const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
|
|
35210
35347
|
return token6 ? new VercelBlobStore(token6) : getBlobStore();
|
|
@@ -35236,6 +35373,9 @@ async function findCommonsHostedImageByUrl(rawUrl) {
|
|
|
35236
35373
|
bytes: Number(row.bytes),
|
|
35237
35374
|
digest: String(row.digest),
|
|
35238
35375
|
sourceUrl: row.source_url ? String(row.source_url) : null,
|
|
35376
|
+
sourcePageUrl: null,
|
|
35377
|
+
sourceKind: "registered",
|
|
35378
|
+
selectionSource: "registered",
|
|
35239
35379
|
reused: true
|
|
35240
35380
|
};
|
|
35241
35381
|
}
|
|
@@ -35243,10 +35383,12 @@ var hostOverride = null;
|
|
|
35243
35383
|
async function hostCommonsImage(input) {
|
|
35244
35384
|
if (hostOverride) return hostOverride(input);
|
|
35245
35385
|
await ensureCommonsImageSchema();
|
|
35246
|
-
if (
|
|
35247
|
-
|
|
35386
|
+
if (input.sourceUrl && !input.imageBase64 && !input.artifactId && !input.sourceType) {
|
|
35387
|
+
const registered6 = await findCommonsHostedImageByUrl(input.sourceUrl);
|
|
35388
|
+
if (registered6) return registered6;
|
|
35248
35389
|
}
|
|
35249
|
-
const
|
|
35390
|
+
const resolved = await resolveCommonsImageSource(input);
|
|
35391
|
+
const bytes = resolved.bytes;
|
|
35250
35392
|
const contentType = sniffImageFormat(bytes);
|
|
35251
35393
|
if (!contentType) {
|
|
35252
35394
|
throw new CommonsImageError(
|
|
@@ -35268,7 +35410,10 @@ async function hostCommonsImage(input) {
|
|
|
35268
35410
|
contentType: String(reusable.content_type),
|
|
35269
35411
|
bytes: Number(reusable.bytes),
|
|
35270
35412
|
digest: digest2,
|
|
35271
|
-
sourceUrl:
|
|
35413
|
+
sourceUrl: resolved.sourceUrl,
|
|
35414
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
35415
|
+
sourceKind: resolved.sourceKind,
|
|
35416
|
+
selectionSource: resolved.selectionSource,
|
|
35272
35417
|
reused: true
|
|
35273
35418
|
};
|
|
35274
35419
|
}
|
|
@@ -35297,14 +35442,25 @@ async function hostCommonsImage(input) {
|
|
|
35297
35442
|
stored.url,
|
|
35298
35443
|
contentType,
|
|
35299
35444
|
bytes.length,
|
|
35300
|
-
|
|
35445
|
+
resolved.sourceUrl,
|
|
35301
35446
|
input.alt ?? null,
|
|
35302
35447
|
input.license ?? null,
|
|
35303
35448
|
input.attribution ?? null,
|
|
35304
35449
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
35305
35450
|
]
|
|
35306
35451
|
});
|
|
35307
|
-
return {
|
|
35452
|
+
return {
|
|
35453
|
+
id,
|
|
35454
|
+
url: stored.url,
|
|
35455
|
+
contentType,
|
|
35456
|
+
bytes: bytes.length,
|
|
35457
|
+
digest: digest2,
|
|
35458
|
+
sourceUrl: resolved.sourceUrl,
|
|
35459
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
35460
|
+
sourceKind: resolved.sourceKind,
|
|
35461
|
+
selectionSource: resolved.selectionSource,
|
|
35462
|
+
reused: false
|
|
35463
|
+
};
|
|
35308
35464
|
}
|
|
35309
35465
|
async function hostEntityImages(input) {
|
|
35310
35466
|
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
@@ -38738,11 +38894,13 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
38738
38894
|
var HostImageSchema = z33.object({
|
|
38739
38895
|
sourceUrl: z33.string().url().optional(),
|
|
38740
38896
|
imageBase64: z33.string().min(16).optional(),
|
|
38897
|
+
artifactId: z33.string().min(1).max(500).optional(),
|
|
38898
|
+
sourceType: z33.enum(["uploaded", "ai_generated"]).optional(),
|
|
38741
38899
|
alt: z33.string().max(500).optional(),
|
|
38742
38900
|
license: z33.string().max(240).optional(),
|
|
38743
38901
|
attribution: z33.string().max(500).optional()
|
|
38744
|
-
}).strict().refine((body) =>
|
|
38745
|
-
message: "Provide
|
|
38902
|
+
}).strict().refine((body) => [body.sourceUrl, body.imageBase64, body.artifactId].filter(Boolean).length === 1, {
|
|
38903
|
+
message: "Provide exactly one of sourceUrl, imageBase64, or artifactId."
|
|
38746
38904
|
});
|
|
38747
38905
|
commonsApp.post("/images", auth2, async (c) => {
|
|
38748
38906
|
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
@@ -49896,7 +50054,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
49896
50054
|
return c.json({ error: "Unauthorized" }, 401);
|
|
49897
50055
|
}
|
|
49898
50056
|
const startedAt = Date.now();
|
|
49899
|
-
const { drainQueue } = await import("./worker-
|
|
50057
|
+
const { drainQueue } = await import("./worker-JGWV2LID.js");
|
|
49900
50058
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
49901
50059
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
49902
50060
|
const unfinished = [];
|
|
@@ -49974,8 +50132,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
49974
50132
|
finishExtractJob: finishExtractJob2,
|
|
49975
50133
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
49976
50134
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
49977
|
-
} = await import("./site-extract-repository-
|
|
49978
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
50135
|
+
} = await import("./site-extract-repository-7EP6UX6I.js");
|
|
50136
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
|
|
49979
50137
|
const existing = await getExtractJob2(jobId2);
|
|
49980
50138
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
49981
50139
|
if (existing.status !== "failed") {
|
|
@@ -50002,7 +50160,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
50002
50160
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
50003
50161
|
let settlement = "already_settled_or_refunded";
|
|
50004
50162
|
if (job.billedMc == null && job.userId != null) {
|
|
50005
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
50163
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-7EP6UX6I.js");
|
|
50006
50164
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
50007
50165
|
const successful = await countSuccessfulPages2(jobId2);
|
|
50008
50166
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -28,10 +28,10 @@ import {
|
|
|
28
28
|
setExtractJobTotal,
|
|
29
29
|
settleExtractJob,
|
|
30
30
|
terminalExtractJobStatus
|
|
31
|
-
} from "./chunk-
|
|
31
|
+
} from "./chunk-LCGTNWOT.js";
|
|
32
32
|
import "./chunk-DNM65UCK.js";
|
|
33
33
|
import "./chunk-AYJHI2FK.js";
|
|
34
|
-
import "./chunk-
|
|
34
|
+
import "./chunk-65FTMB7G.js";
|
|
35
35
|
export {
|
|
36
36
|
abandonExtractSettlement,
|
|
37
37
|
claimFailedExtractJobForRefinalize,
|
|
@@ -5,19 +5,19 @@ import {
|
|
|
5
5
|
harvestProblemEnvelope,
|
|
6
6
|
harvestProblemResponse,
|
|
7
7
|
serializeHarvestProblem
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-2LXTOAKS.js";
|
|
9
9
|
import "./chunk-OZJMVCDK.js";
|
|
10
10
|
import {
|
|
11
11
|
harvest
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-62LEXS5O.js";
|
|
13
13
|
import {
|
|
14
14
|
sanitizeAttempts,
|
|
15
15
|
sanitizeHarvestResult
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-2L4C4DAZ.js";
|
|
17
17
|
import {
|
|
18
18
|
browserServiceApiKey,
|
|
19
19
|
runWithCostContext
|
|
20
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-ES25GP6C.js";
|
|
21
21
|
import "./chunk-77LQV73B.js";
|
|
22
22
|
import {
|
|
23
23
|
MC_COSTS,
|
|
@@ -30,7 +30,7 @@ import {
|
|
|
30
30
|
debitMc,
|
|
31
31
|
failJob,
|
|
32
32
|
listHarvestAttempts
|
|
33
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-65FTMB7G.js";
|
|
34
34
|
|
|
35
35
|
// src/api/worker.ts
|
|
36
36
|
var MAX_CONCURRENT = 2;
|