mcp-scraper 0.66.7 → 0.67.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/README.md +1 -1
  3. package/dist/bin/api-server.cjs +246 -41
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +2 -2
  8. package/dist/bin/mcp-scraper-install.js +2 -2
  9. package/dist/bin/mcp-stdio-server.cjs +56 -12
  10. package/dist/bin/mcp-stdio-server.js +6 -6
  11. package/dist/bin/paa-harvest.cjs +6 -0
  12. package/dist/bin/paa-harvest.js +3 -3
  13. package/dist/{chunk-F434YJPV.js → chunk-2L4C4DAZ.js} +1 -1
  14. package/dist/{chunk-LVXRWJJJ.js → chunk-2LXTOAKS.js} +1 -1
  15. package/dist/{chunk-CGOMGGOF.js → chunk-62LEXS5O.js} +2 -2
  16. package/dist/{chunk-HZDJ5UZ4.js → chunk-65FTMB7G.js} +25 -1
  17. package/dist/{chunk-U2UHJWFW.js → chunk-ES25GP6C.js} +1 -1
  18. package/dist/{chunk-7KB5QTHP.js → chunk-GT63KEUB.js} +1 -1
  19. package/dist/{chunk-O7ZC5Y3Z.js → chunk-IHDTONZV.js} +1 -1
  20. package/dist/{chunk-HWGQVP3Q.js → chunk-LCGTNWOT.js} +1 -1
  21. package/dist/{chunk-EVOBJYBE.js → chunk-LN6N3YLK.js} +1 -1
  22. package/dist/{chunk-JZPSG4NR.js → chunk-UL4ZKAWZ.js} +1 -1
  23. package/dist/{chunk-Z5QQEWXA.js → chunk-XOOKZYZG.js} +33 -13
  24. package/dist/{chunk-HBZENASD.js → chunk-Y46YNQMM.js} +1 -1
  25. package/dist/{db-MNNNIWVS.js → db-ZGZAKYBW.js} +1 -1
  26. package/dist/{extract-bundle-RW7CVIRF.js → extract-bundle-QQIYJSBW.js} +3 -3
  27. package/dist/index.cjs +6 -0
  28. package/dist/index.js +3 -3
  29. package/dist/{lead-list-enrichment-repository-PZYE6TDP.js → lead-list-enrichment-repository-CXLVIV3R.js} +2 -2
  30. package/dist/{location-data-repository-EPCNGMOM.js → location-data-repository-G72M7DTN.js} +2 -2
  31. package/dist/{server-KSVJ4X4V.js → server-MFVYRB6I.js} +206 -48
  32. package/dist/{site-extract-repository-CZ7SVT3Z.js → site-extract-repository-7EP6UX6I.js} +2 -2
  33. package/dist/{worker-IBLGV272.js → worker-JGWV2LID.js} +5 -5
  34. package/package.json +1 -1
@@ -10,13 +10,13 @@ import {
10
10
  queuedCommonsDocumentCount,
11
11
  semanticCommonsEntityScores,
12
12
  typeFromMime
13
- } from "./chunk-EVOBJYBE.js";
13
+ } from "./chunk-LN6N3YLK.js";
14
14
  import {
15
15
  REQUIRED_LOCATION_STATE_CODES,
16
16
  importHostedCensusPlacesCsv,
17
17
  importHostedZipGroupsCsv,
18
18
  listActiveHostedLocationDatasets
19
- } from "./chunk-HBZENASD.js";
19
+ } from "./chunk-Y46YNQMM.js";
20
20
  import {
21
21
  cancelPendingLeadListEnrichmentRows,
22
22
  claimLeadListEnrichmentOutbox,
@@ -43,7 +43,7 @@ import {
43
43
  saveLeadListEnrichmentRowResult,
44
44
  setInitializingLeadListEnrichmentConcurrencyLock,
45
45
  stableCanonicalJson
46
- } from "./chunk-JZPSG4NR.js";
46
+ } from "./chunk-UL4ZKAWZ.js";
47
47
  import {
48
48
  ANALYTICS_CONTENT_SORTS,
49
49
  AnalyticsRepositoryError,
@@ -108,7 +108,7 @@ import {
108
108
  harvestProblemResponse,
109
109
  publicErrorBoundary,
110
110
  serializeHarvestProblem
111
- } from "./chunk-LVXRWJJJ.js";
111
+ } from "./chunk-2LXTOAKS.js";
112
112
  import "./chunk-OZJMVCDK.js";
113
113
  import {
114
114
  BACKUP_LAST_RESORT_ATTEMPTS,
@@ -127,7 +127,7 @@ import {
127
127
  resolveKernelProxyId,
128
128
  serpArrivalUrl,
129
129
  withRuntimeUserAgent
130
- } from "./chunk-CGOMGGOF.js";
130
+ } from "./chunk-62LEXS5O.js";
131
131
  import {
132
132
  ApiErrorResponseSchema,
133
133
  ArtifactTemplatePresetKeySchema,
@@ -199,7 +199,7 @@ import {
199
199
  resolveDeploymentProfile,
200
200
  resolveLocalSourcebookSchemaType,
201
201
  transcribeMediaUrl
202
- } from "./chunk-Z5QQEWXA.js";
202
+ } from "./chunk-XOOKZYZG.js";
203
203
  import {
204
204
  auditImageUrls,
205
205
  auditImages,
@@ -210,7 +210,7 @@ import {
210
210
  sanitizeAttempts,
211
211
  sanitizeHarvestResult,
212
212
  sanitizeOutboundDiagnostics
213
- } from "./chunk-F434YJPV.js";
213
+ } from "./chunk-2L4C4DAZ.js";
214
214
  import {
215
215
  browserServiceApiKey,
216
216
  browserServiceProfileName,
@@ -220,7 +220,7 @@ import {
220
220
  recordKernelSession,
221
221
  recordVendorUsage,
222
222
  runWithCostContext
223
- } from "./chunk-U2UHJWFW.js";
223
+ } from "./chunk-ES25GP6C.js";
224
224
  import {
225
225
  listWorkflowDefinitions,
226
226
  runWorkflow,
@@ -256,7 +256,7 @@ import {
256
256
  } from "./chunk-GGZEC22A.js";
257
257
  import {
258
258
  PACKAGE_VERSION
259
- } from "./chunk-7KB5QTHP.js";
259
+ } from "./chunk-GT63KEUB.js";
260
260
  import {
261
261
  SITE_EXTRACT_ARTIFACT_PREFIX,
262
262
  abandonExtractSettlement,
@@ -287,7 +287,7 @@ import {
287
287
  setExtractJobTotal,
288
288
  settleExtractJob,
289
289
  terminalExtractJobStatus
290
- } from "./chunk-HWGQVP3Q.js";
290
+ } from "./chunk-LCGTNWOT.js";
291
291
  import {
292
292
  createPrivateArtifact,
293
293
  privateArtifactOwnerId,
@@ -451,7 +451,7 @@ import {
451
451
  softDeleteInboxMessage,
452
452
  upsertPageSnapshot,
453
453
  verifyPassword
454
- } from "./chunk-HZDJ5UZ4.js";
454
+ } from "./chunk-65FTMB7G.js";
455
455
 
456
456
  // src/blog/registry.ts
457
457
  var posts = [
@@ -6312,7 +6312,8 @@ function firstJsonLdImage(value) {
6312
6312
  }
6313
6313
  return null;
6314
6314
  }
6315
- function firstContentImage(html) {
6315
+ function contentImages(html) {
6316
+ const images = [];
6316
6317
  for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
6317
6318
  const tag = match[0];
6318
6319
  const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
@@ -6323,30 +6324,36 @@ function firstContentImage(html) {
6323
6324
  const width = Number(attr("width") ?? 0);
6324
6325
  const height = Number(attr("height") ?? 0);
6325
6326
  if (width > 0 && width < 200 || height > 0 && height < 120) continue;
6326
- return candidate;
6327
+ images.push(candidate);
6327
6328
  }
6328
- return null;
6329
+ return images;
6329
6330
  }
6330
- function resolveFeaturedImage(input) {
6331
+ function resolveFeaturedImageCandidates(input) {
6331
6332
  const replay = parseWaybackReplayUrl(input.pageUrl);
6332
6333
  const baseUrl = replay?.originalUrl ?? input.pageUrl;
6333
6334
  const candidates = [
6334
6335
  { value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
6335
6336
  { value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
6336
6337
  { value: firstJsonLdImage(input.schema), source: "json-ld" },
6337
- { value: firstContentImage(input.html), source: "content-image" }
6338
+ ...contentImages(input.html).map((value) => ({ value, source: "content-image" }))
6338
6339
  ];
6340
+ const seen = /* @__PURE__ */ new Set();
6341
+ const resolved = [];
6339
6342
  for (const candidate of candidates) {
6340
6343
  if (!candidate.value) continue;
6341
6344
  const url = safeHttpUrl(candidate.value, baseUrl);
6342
- if (!url) continue;
6343
- return {
6345
+ if (!url || seen.has(url)) continue;
6346
+ seen.add(url);
6347
+ resolved.push({
6344
6348
  url,
6345
6349
  archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
6346
6350
  source: candidate.source
6347
- };
6351
+ });
6348
6352
  }
6349
- return null;
6353
+ return resolved;
6354
+ }
6355
+ function resolveFeaturedImage(input) {
6356
+ return resolveFeaturedImageCandidates(input)[0] ?? null;
6350
6357
  }
6351
6358
 
6352
6359
  // src/api/site-extractor.ts
@@ -9545,7 +9552,7 @@ var siteExtractFn = inngest.createFunction(
9545
9552
  });
9546
9553
  }) : null;
9547
9554
  const finalized = await step.run("finalize", async () => {
9548
- const { assembleExtractArtifacts } = await import("./extract-bundle-RW7CVIRF.js");
9555
+ const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
9549
9556
  const current = await getExtractJob(jobId2);
9550
9557
  if (!current) throw new Error("extract job disappeared before finalization");
9551
9558
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10505,7 +10512,7 @@ async function resolveUsZipsPath(requestedPath) {
10505
10512
  async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
10506
10513
  const mode = resolvedLocationDataMode({ zipSource: sourceMode });
10507
10514
  if (mode === "hosted") {
10508
- const { getHostedZipGroups } = await import("./location-data-repository-EPCNGMOM.js");
10515
+ const { getHostedZipGroups } = await import("./location-data-repository-G72M7DTN.js");
10509
10516
  const hosted = await getHostedZipGroups(stateAbbr);
10510
10517
  if (!hosted.dataset) {
10511
10518
  throw new DirectoryLocationError(
@@ -10568,7 +10575,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
10568
10575
  const sourceUrl = censusStateUrl(state.fips);
10569
10576
  const warnings = [];
10570
10577
  if (resolvedLocationDataMode(config) === "hosted") {
10571
- const { queryHostedLocationMarkets } = await import("./location-data-repository-EPCNGMOM.js");
10578
+ const { queryHostedLocationMarkets } = await import("./location-data-repository-G72M7DTN.js");
10572
10579
  const hosted = await queryHostedLocationMarkets({
10573
10580
  state: state.abbr,
10574
10581
  minPopulation: options.minPopulation,
@@ -12055,7 +12062,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
12055
12062
  async function settlePersistedLeadListEnrichment(jobId2) {
12056
12063
  const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
12057
12064
  const job = jobs.find((candidate) => candidate.id === jobId2);
12058
- const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-PZYE6TDP.js");
12065
+ const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
12059
12066
  const current = job ?? await getLeadListEnrichmentJob2(jobId2);
12060
12067
  if (!current) return null;
12061
12068
  const rows = await listLeadListEnrichmentRows(jobId2);
@@ -12111,7 +12118,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
12111
12118
  const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
12112
12119
  const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
12113
12120
  if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
12114
- const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-PZYE6TDP.js");
12121
+ const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-CXLVIV3R.js");
12115
12122
  await failLeadListEnrichmentJob2({
12116
12123
  jobId: job.id,
12117
12124
  errorCode: "lead_list_all_rows_failed",
@@ -35066,7 +35073,9 @@ import { isIP } from "net";
35066
35073
  var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
35067
35074
  var COMMONS_IMAGE_MAX_REDIRECTS = 3;
35068
35075
  var COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
35069
- var COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with a different stable public HTTPS image URL or imageBase64, replace featuredImage.url with the returned URL, validate again, then retry commons_submit_entity with the same idempotency key.";
35076
+ var COMMONS_IMAGE_PAGE_MAX_BYTES = 2 * 1024 * 1024;
35077
+ var COMMONS_IMAGE_MAX_PAGE_CANDIDATES = 12;
35078
+ var COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with imageBase64, an owner-authorized artifactId, a public HTTPS image URL, or a public HTTPS webpage containing a usable featured image. Replace featuredImage.url with the returned permanent URL, validate again, then retry commons_submit_entity with the same idempotency key.";
35070
35079
  var CommonsImageError = class extends Error {
35071
35080
  constructor(code, message, httpStatus = 400, retryable = false, retryGuidance = COMMONS_IMAGE_RETRY_GUIDANCE) {
35072
35081
  super(message);
@@ -35139,6 +35148,9 @@ async function assertPublicHttpsUrl(rawUrl) {
35139
35148
  if (url.protocol !== "https:") {
35140
35149
  throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
35141
35150
  }
35151
+ if (url.username || url.password) {
35152
+ throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
35153
+ }
35142
35154
  const hostname = url.hostname.replace(/^\[|\]$/g, "");
35143
35155
  const addresses = isIP(hostname) ? [{ address: hostname }] : await dnsLookup(hostname, { all: true }).catch(() => []);
35144
35156
  if (!addresses.length) {
@@ -35149,7 +35161,28 @@ async function assertPublicHttpsUrl(rawUrl) {
35149
35161
  }
35150
35162
  return url;
35151
35163
  }
35152
- async function downloadImage(sourceUrl) {
35164
+ async function readResponseBytes(response, maxBytes) {
35165
+ const declaredLength = Number(response.headers.get("content-length") ?? "0");
35166
+ if (declaredLength > maxBytes) {
35167
+ throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
35168
+ }
35169
+ if (!response.body) return Buffer.alloc(0);
35170
+ const reader = response.body.getReader();
35171
+ const chunks = [];
35172
+ let total = 0;
35173
+ while (true) {
35174
+ const { done, value } = await reader.read();
35175
+ if (done) break;
35176
+ total += value.byteLength;
35177
+ if (total > maxBytes) {
35178
+ await reader.cancel().catch(() => void 0);
35179
+ throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
35180
+ }
35181
+ chunks.push(Buffer.from(value));
35182
+ }
35183
+ return Buffer.concat(chunks, total);
35184
+ }
35185
+ async function downloadPublicSource(sourceUrl) {
35153
35186
  let current = sourceUrl;
35154
35187
  for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
35155
35188
  const url = await assertPublicHttpsUrl(current);
@@ -35184,15 +35217,11 @@ async function downloadImage(sourceUrl) {
35184
35217
  retryable
35185
35218
  );
35186
35219
  }
35187
- const declaredLength = Number(response.headers.get("content-length") ?? "0");
35188
- if (declaredLength > COMMONS_IMAGE_MAX_BYTES) {
35189
- throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
35190
- }
35191
- const bytes = Buffer.from(await response.arrayBuffer());
35192
- if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
35193
- throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
35194
- }
35195
- return bytes;
35220
+ return {
35221
+ bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
35222
+ finalUrl: url.toString(),
35223
+ contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
35224
+ };
35196
35225
  }
35197
35226
  throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
35198
35227
  }
@@ -35205,6 +35234,114 @@ function decodeBase64Image(value) {
35205
35234
  }
35206
35235
  return bytes;
35207
35236
  }
35237
+ function safeProvenanceUrl(rawUrl) {
35238
+ try {
35239
+ const url = new URL(rawUrl);
35240
+ url.username = "";
35241
+ url.password = "";
35242
+ url.search = "";
35243
+ url.hash = "";
35244
+ return url.toString();
35245
+ } catch {
35246
+ return null;
35247
+ }
35248
+ }
35249
+ function looksLikeHtml(bytes, declaredType) {
35250
+ if (declaredType === "text/html" || declaredType === "application/xhtml+xml") return true;
35251
+ const prefix = bytes.subarray(0, 512).toString("utf8").trimStart().toLowerCase();
35252
+ return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
35253
+ }
35254
+ async function resolveCommonsImageSource(input) {
35255
+ const supplied = [input.sourceUrl, input.imageBase64, input.artifactId].filter((value) => Boolean(value)).length;
35256
+ if (supplied === 0) {
35257
+ throw new CommonsImageError("image_source_missing", "Provide exactly one of sourceUrl, imageBase64, or artifactId.");
35258
+ }
35259
+ if (supplied !== 1) {
35260
+ throw new CommonsImageError("image_source_ambiguous", "Provide only one image source: sourceUrl, imageBase64, or artifactId.");
35261
+ }
35262
+ if (input.sourceUrl && input.sourceType) {
35263
+ throw new CommonsImageError("image_source_type_invalid", "sourceType applies only to imageBase64 or artifactId, not sourceUrl.");
35264
+ }
35265
+ if (input.imageBase64) {
35266
+ return {
35267
+ bytes: decodeBase64Image(input.imageBase64),
35268
+ sourceUrl: null,
35269
+ sourcePageUrl: null,
35270
+ sourceKind: input.sourceType ?? "uploaded",
35271
+ selectionSource: "provided_bytes"
35272
+ };
35273
+ }
35274
+ if (input.artifactId) {
35275
+ const bytes = await readOwnedImageSourceArtifact({ artifactId: input.artifactId, ownerId: String(input.userId) });
35276
+ if (!bytes) {
35277
+ throw new CommonsImageError("image_artifact_not_found", "The image artifact was not found, expired, or is not owned by this account.", 422);
35278
+ }
35279
+ if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
35280
+ throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
35281
+ }
35282
+ return {
35283
+ bytes,
35284
+ sourceUrl: null,
35285
+ sourcePageUrl: null,
35286
+ sourceKind: input.sourceType ?? "artifact",
35287
+ selectionSource: "artifact"
35288
+ };
35289
+ }
35290
+ const requestedUrl = String(input.sourceUrl);
35291
+ const downloaded = await downloadPublicSource(requestedUrl);
35292
+ if (sniffImageFormat(downloaded.bytes)) {
35293
+ return {
35294
+ bytes: downloaded.bytes,
35295
+ sourceUrl: safeProvenanceUrl(downloaded.finalUrl),
35296
+ sourcePageUrl: null,
35297
+ sourceKind: "direct_url",
35298
+ selectionSource: "direct_url"
35299
+ };
35300
+ }
35301
+ if (!looksLikeHtml(downloaded.bytes, downloaded.contentType)) {
35302
+ throw new CommonsImageError(
35303
+ "image_format_unsupported",
35304
+ "The supplied URL returned neither a supported image nor an HTML webpage with a discoverable image.",
35305
+ 415
35306
+ );
35307
+ }
35308
+ if (downloaded.bytes.length > COMMONS_IMAGE_PAGE_MAX_BYTES) {
35309
+ throw new CommonsImageError("image_page_too_large", `Webpage exceeds the ${COMMONS_IMAGE_PAGE_MAX_BYTES} byte discovery limit.`, 413);
35310
+ }
35311
+ const html = downloaded.bytes.toString("utf8");
35312
+ const document2 = loadHtmlDocument(html);
35313
+ const candidates = resolveFeaturedImageCandidates({
35314
+ html,
35315
+ meta: document2.meta,
35316
+ schema: document2.schema,
35317
+ pageUrl: downloaded.finalUrl
35318
+ }).slice(0, COMMONS_IMAGE_MAX_PAGE_CANDIDATES);
35319
+ if (!candidates.length) {
35320
+ throw new CommonsImageError("page_image_not_found", "The webpage did not expose a usable featured or content image.", 422);
35321
+ }
35322
+ let sawRetryableFailure = false;
35323
+ for (const candidate of candidates) {
35324
+ try {
35325
+ const image = await downloadPublicSource(candidate.url);
35326
+ if (!sniffImageFormat(image.bytes)) continue;
35327
+ return {
35328
+ bytes: image.bytes,
35329
+ sourceUrl: safeProvenanceUrl(image.finalUrl),
35330
+ sourcePageUrl: safeProvenanceUrl(downloaded.finalUrl),
35331
+ sourceKind: "webpage",
35332
+ selectionSource: candidate.source
35333
+ };
35334
+ } catch (error) {
35335
+ if (error instanceof CommonsImageError && error.retryable) sawRetryableFailure = true;
35336
+ }
35337
+ }
35338
+ throw new CommonsImageError(
35339
+ "page_image_candidates_unusable",
35340
+ "The webpage exposed image candidates, but none returned supported public image bytes.",
35341
+ 422,
35342
+ sawRetryableFailure
35343
+ );
35344
+ }
35208
35345
  function commonsImageBlobStore() {
35209
35346
  const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
35210
35347
  return token6 ? new VercelBlobStore(token6) : getBlobStore();
@@ -35236,6 +35373,9 @@ async function findCommonsHostedImageByUrl(rawUrl) {
35236
35373
  bytes: Number(row.bytes),
35237
35374
  digest: String(row.digest),
35238
35375
  sourceUrl: row.source_url ? String(row.source_url) : null,
35376
+ sourcePageUrl: null,
35377
+ sourceKind: "registered",
35378
+ selectionSource: "registered",
35239
35379
  reused: true
35240
35380
  };
35241
35381
  }
@@ -35243,10 +35383,12 @@ var hostOverride = null;
35243
35383
  async function hostCommonsImage(input) {
35244
35384
  if (hostOverride) return hostOverride(input);
35245
35385
  await ensureCommonsImageSchema();
35246
- if (!input.sourceUrl && !input.imageBase64) {
35247
- throw new CommonsImageError("image_source_missing", "Provide either sourceUrl or imageBase64.");
35386
+ if (input.sourceUrl && !input.imageBase64 && !input.artifactId && !input.sourceType) {
35387
+ const registered6 = await findCommonsHostedImageByUrl(input.sourceUrl);
35388
+ if (registered6) return registered6;
35248
35389
  }
35249
- const bytes = input.imageBase64 ? decodeBase64Image(input.imageBase64) : await downloadImage(String(input.sourceUrl));
35390
+ const resolved = await resolveCommonsImageSource(input);
35391
+ const bytes = resolved.bytes;
35250
35392
  const contentType = sniffImageFormat(bytes);
35251
35393
  if (!contentType) {
35252
35394
  throw new CommonsImageError(
@@ -35268,7 +35410,10 @@ async function hostCommonsImage(input) {
35268
35410
  contentType: String(reusable.content_type),
35269
35411
  bytes: Number(reusable.bytes),
35270
35412
  digest: digest2,
35271
- sourceUrl: reusable.source_url ? String(reusable.source_url) : null,
35413
+ sourceUrl: resolved.sourceUrl,
35414
+ sourcePageUrl: resolved.sourcePageUrl,
35415
+ sourceKind: resolved.sourceKind,
35416
+ selectionSource: resolved.selectionSource,
35272
35417
  reused: true
35273
35418
  };
35274
35419
  }
@@ -35297,14 +35442,25 @@ async function hostCommonsImage(input) {
35297
35442
  stored.url,
35298
35443
  contentType,
35299
35444
  bytes.length,
35300
- input.sourceUrl ?? null,
35445
+ resolved.sourceUrl,
35301
35446
  input.alt ?? null,
35302
35447
  input.license ?? null,
35303
35448
  input.attribution ?? null,
35304
35449
  (/* @__PURE__ */ new Date()).toISOString()
35305
35450
  ]
35306
35451
  });
35307
- return { id, url: stored.url, contentType, bytes: bytes.length, digest: digest2, sourceUrl: input.sourceUrl ?? null, reused: false };
35452
+ return {
35453
+ id,
35454
+ url: stored.url,
35455
+ contentType,
35456
+ bytes: bytes.length,
35457
+ digest: digest2,
35458
+ sourceUrl: resolved.sourceUrl,
35459
+ sourcePageUrl: resolved.sourcePageUrl,
35460
+ sourceKind: resolved.sourceKind,
35461
+ selectionSource: resolved.selectionSource,
35462
+ reused: false
35463
+ };
35308
35464
  }
35309
35465
  async function hostEntityImages(input) {
35310
35466
  const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
@@ -38738,11 +38894,13 @@ commonsApp.get("/publications/:subdomain", async (c) => {
38738
38894
  var HostImageSchema = z33.object({
38739
38895
  sourceUrl: z33.string().url().optional(),
38740
38896
  imageBase64: z33.string().min(16).optional(),
38897
+ artifactId: z33.string().min(1).max(500).optional(),
38898
+ sourceType: z33.enum(["uploaded", "ai_generated"]).optional(),
38741
38899
  alt: z33.string().max(500).optional(),
38742
38900
  license: z33.string().max(240).optional(),
38743
38901
  attribution: z33.string().max(500).optional()
38744
- }).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
38745
- message: "Provide either sourceUrl or imageBase64."
38902
+ }).strict().refine((body) => [body.sourceUrl, body.imageBase64, body.artifactId].filter(Boolean).length === 1, {
38903
+ message: "Provide exactly one of sourceUrl, imageBase64, or artifactId."
38746
38904
  });
38747
38905
  commonsApp.post("/images", auth2, async (c) => {
38748
38906
  const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
@@ -49896,7 +50054,7 @@ app.get("/cron/tick", async (c) => {
49896
50054
  return c.json({ error: "Unauthorized" }, 401);
49897
50055
  }
49898
50056
  const startedAt = Date.now();
49899
- const { drainQueue } = await import("./worker-IBLGV272.js");
50057
+ const { drainQueue } = await import("./worker-JGWV2LID.js");
49900
50058
  const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
49901
50059
  const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
49902
50060
  const unfinished = [];
@@ -49974,8 +50132,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
49974
50132
  finishExtractJob: finishExtractJob2,
49975
50133
  terminalExtractJobStatus: terminalExtractJobStatus2,
49976
50134
  extractJobLimitInfo: extractJobLimitInfo2
49977
- } = await import("./site-extract-repository-CZ7SVT3Z.js");
49978
- const { assembleExtractArtifacts } = await import("./extract-bundle-RW7CVIRF.js");
50135
+ } = await import("./site-extract-repository-7EP6UX6I.js");
50136
+ const { assembleExtractArtifacts } = await import("./extract-bundle-QQIYJSBW.js");
49979
50137
  const existing = await getExtractJob2(jobId2);
49980
50138
  if (!existing) return c.json({ error: "job not found" }, 404);
49981
50139
  if (existing.status !== "failed") {
@@ -50002,7 +50160,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
50002
50160
  if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
50003
50161
  let settlement = "already_settled_or_refunded";
50004
50162
  if (job.billedMc == null && job.userId != null) {
50005
- const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CZ7SVT3Z.js");
50163
+ const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-7EP6UX6I.js");
50006
50164
  const heldMc = Number(job.options.heldMc ?? 0);
50007
50165
  const successful = await countSuccessfulPages2(jobId2);
50008
50166
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -28,10 +28,10 @@ import {
28
28
  setExtractJobTotal,
29
29
  settleExtractJob,
30
30
  terminalExtractJobStatus
31
- } from "./chunk-HWGQVP3Q.js";
31
+ } from "./chunk-LCGTNWOT.js";
32
32
  import "./chunk-DNM65UCK.js";
33
33
  import "./chunk-AYJHI2FK.js";
34
- import "./chunk-HZDJ5UZ4.js";
34
+ import "./chunk-65FTMB7G.js";
35
35
  export {
36
36
  abandonExtractSettlement,
37
37
  claimFailedExtractJobForRefinalize,
@@ -5,19 +5,19 @@ import {
5
5
  harvestProblemEnvelope,
6
6
  harvestProblemResponse,
7
7
  serializeHarvestProblem
8
- } from "./chunk-LVXRWJJJ.js";
8
+ } from "./chunk-2LXTOAKS.js";
9
9
  import "./chunk-OZJMVCDK.js";
10
10
  import {
11
11
  harvest
12
- } from "./chunk-CGOMGGOF.js";
12
+ } from "./chunk-62LEXS5O.js";
13
13
  import {
14
14
  sanitizeAttempts,
15
15
  sanitizeHarvestResult
16
- } from "./chunk-F434YJPV.js";
16
+ } from "./chunk-2L4C4DAZ.js";
17
17
  import {
18
18
  browserServiceApiKey,
19
19
  runWithCostContext
20
- } from "./chunk-U2UHJWFW.js";
20
+ } from "./chunk-ES25GP6C.js";
21
21
  import "./chunk-77LQV73B.js";
22
22
  import {
23
23
  MC_COSTS,
@@ -30,7 +30,7 @@ import {
30
30
  debitMc,
31
31
  failJob,
32
32
  listHarvestAttempts
33
- } from "./chunk-HZDJ5UZ4.js";
33
+ } from "./chunk-65FTMB7G.js";
34
34
 
35
35
  // src/api/worker.ts
36
36
  var MAX_CONCURRENT = 2;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mcp-scraper",
3
- "version": "0.66.7",
3
+ "version": "0.67.0",
4
4
  "description": "MCP server for MCP Scraper web intelligence tools",
5
5
  "type": "module",
6
6
  "main": "./dist/index.cjs",