mcp-scraper 0.66.7 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +246 -41
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +56 -12
- package/dist/bin/mcp-stdio-server.js +6 -6
- package/dist/bin/paa-harvest.cjs +6 -0
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-F434YJPV.js → chunk-2L4C4DAZ.js} +1 -1
- package/dist/{chunk-LVXRWJJJ.js → chunk-2LXTOAKS.js} +1 -1
- package/dist/{chunk-CGOMGGOF.js → chunk-62LEXS5O.js} +2 -2
- package/dist/{chunk-HZDJ5UZ4.js → chunk-65FTMB7G.js} +25 -1
- package/dist/{chunk-U2UHJWFW.js → chunk-ES25GP6C.js} +1 -1
- package/dist/{chunk-7KB5QTHP.js → chunk-GT63KEUB.js} +1 -1
- package/dist/{chunk-O7ZC5Y3Z.js → chunk-IHDTONZV.js} +1 -1
- package/dist/{chunk-HWGQVP3Q.js → chunk-LCGTNWOT.js} +1 -1
- package/dist/{chunk-EVOBJYBE.js → chunk-LN6N3YLK.js} +1 -1
- package/dist/{chunk-JZPSG4NR.js → chunk-UL4ZKAWZ.js} +1 -1
- package/dist/{chunk-Z5QQEWXA.js → chunk-XOOKZYZG.js} +33 -13
- package/dist/{chunk-HBZENASD.js → chunk-Y46YNQMM.js} +1 -1
- package/dist/{db-MNNNIWVS.js → db-ZGZAKYBW.js} +1 -1
- package/dist/{extract-bundle-RW7CVIRF.js → extract-bundle-QQIYJSBW.js} +3 -3
- package/dist/index.cjs +6 -0
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-PZYE6TDP.js → lead-list-enrichment-repository-CXLVIV3R.js} +2 -2
- package/dist/{location-data-repository-EPCNGMOM.js → location-data-repository-G72M7DTN.js} +2 -2
- package/dist/{server-KSVJ4X4V.js → server-MFVYRB6I.js} +206 -48
- package/dist/{site-extract-repository-CZ7SVT3Z.js → site-extract-repository-7EP6UX6I.js} +2 -2
- package/dist/{worker-IBLGV272.js → worker-JGWV2LID.js} +5 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,13 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.67.0] - 2026-08-26
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `commons_host_image` now accepts pasted or AI-generated bytes, owner-scoped image artifacts, direct image URLs, and webpage URLs through one workflow. Webpage inputs try Open Graph, Twitter, JSON-LD, and meaningful content images, return source-selection provenance, and omit temporary URL query credentials from durable image metadata.
|
|
12
|
+
- `mcpscraper.dev` now publishes its complete generated LLM discovery profile and bounded semantic authority-retrieval endpoints from the site-owned Next.js authority layer.
|
|
13
|
+
|
|
7
14
|
## [0.66.7] - 2026-08-26
|
|
8
15
|
|
|
9
16
|
### Fixed
|
package/README.md
CHANGED
|
@@ -159,7 +159,7 @@ Build the branded one-click bundle:
|
|
|
159
159
|
npm run build:mcpb
|
|
160
160
|
```
|
|
161
161
|
|
|
162
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
162
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.67.0`, SHA-256 `03cd764c2ec1741ae0a64373f91324a8f46b523d4bf2ace8d9845f8c45f8d9e4`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
163
163
|
|
|
164
164
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
165
165
|
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -174,6 +174,12 @@ function publicErrorType(code) {
|
|
|
174
174
|
case "image_redirect_limit":
|
|
175
175
|
case "image_base64_invalid":
|
|
176
176
|
case "image_source_missing":
|
|
177
|
+
case "image_source_ambiguous":
|
|
178
|
+
case "image_source_type_invalid":
|
|
179
|
+
case "image_artifact_not_found":
|
|
180
|
+
case "image_page_too_large":
|
|
181
|
+
case "page_image_not_found":
|
|
182
|
+
case "page_image_candidates_unusable":
|
|
177
183
|
case "image_format_unsupported":
|
|
178
184
|
case "image_storage_unconfigured":
|
|
179
185
|
case "featured_image_unhostable":
|
|
@@ -270,7 +276,19 @@ function publicErrorMessage(code) {
|
|
|
270
276
|
case "image_base64_invalid":
|
|
271
277
|
return "The supplied inline image bytes are invalid.";
|
|
272
278
|
case "image_source_missing":
|
|
273
|
-
return "Provide
|
|
279
|
+
return "Provide exactly one of a public image or webpage URL, inline image bytes, or an owner-scoped image artifact.";
|
|
280
|
+
case "image_source_ambiguous":
|
|
281
|
+
return "Provide only one image source: sourceUrl, imageBase64, or artifactId.";
|
|
282
|
+
case "image_source_type_invalid":
|
|
283
|
+
return "Use sourceType only with inline image bytes or an owner-scoped image artifact.";
|
|
284
|
+
case "image_artifact_not_found":
|
|
285
|
+
return "The image artifact was not found, expired, or does not belong to this account.";
|
|
286
|
+
case "image_page_too_large":
|
|
287
|
+
return "The webpage is too large for bounded Commons image discovery.";
|
|
288
|
+
case "page_image_not_found":
|
|
289
|
+
return "The webpage did not expose a usable featured or content image.";
|
|
290
|
+
case "page_image_candidates_unusable":
|
|
291
|
+
return "The webpage exposed image candidates, but none returned supported public image bytes.";
|
|
274
292
|
case "image_format_unsupported":
|
|
275
293
|
return "Commons accepts JPEG, PNG, GIF, or WebP image bytes.";
|
|
276
294
|
case "image_storage_unconfigured":
|
|
@@ -350,6 +368,12 @@ var init_errors = __esm({
|
|
|
350
368
|
"image_redirect_limit",
|
|
351
369
|
"image_base64_invalid",
|
|
352
370
|
"image_source_missing",
|
|
371
|
+
"image_source_ambiguous",
|
|
372
|
+
"image_source_type_invalid",
|
|
373
|
+
"image_artifact_not_found",
|
|
374
|
+
"image_page_too_large",
|
|
375
|
+
"page_image_not_found",
|
|
376
|
+
"page_image_candidates_unusable",
|
|
353
377
|
"image_format_unsupported",
|
|
354
378
|
"image_storage_unconfigured",
|
|
355
379
|
"featured_image_unhostable"
|
|
@@ -11394,7 +11418,8 @@ function firstJsonLdImage(value) {
|
|
|
11394
11418
|
}
|
|
11395
11419
|
return null;
|
|
11396
11420
|
}
|
|
11397
|
-
function
|
|
11421
|
+
function contentImages(html) {
|
|
11422
|
+
const images = [];
|
|
11398
11423
|
for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
|
|
11399
11424
|
const tag = match[0];
|
|
11400
11425
|
const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
|
|
@@ -11405,30 +11430,36 @@ function firstContentImage(html) {
|
|
|
11405
11430
|
const width = Number(attr("width") ?? 0);
|
|
11406
11431
|
const height = Number(attr("height") ?? 0);
|
|
11407
11432
|
if (width > 0 && width < 200 || height > 0 && height < 120) continue;
|
|
11408
|
-
|
|
11433
|
+
images.push(candidate);
|
|
11409
11434
|
}
|
|
11410
|
-
return
|
|
11435
|
+
return images;
|
|
11411
11436
|
}
|
|
11412
|
-
function
|
|
11437
|
+
function resolveFeaturedImageCandidates(input) {
|
|
11413
11438
|
const replay = parseWaybackReplayUrl(input.pageUrl);
|
|
11414
11439
|
const baseUrl = replay?.originalUrl ?? input.pageUrl;
|
|
11415
11440
|
const candidates = [
|
|
11416
11441
|
{ value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
|
|
11417
11442
|
{ value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
|
|
11418
11443
|
{ value: firstJsonLdImage(input.schema), source: "json-ld" },
|
|
11419
|
-
|
|
11444
|
+
...contentImages(input.html).map((value) => ({ value, source: "content-image" }))
|
|
11420
11445
|
];
|
|
11446
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11447
|
+
const resolved = [];
|
|
11421
11448
|
for (const candidate of candidates) {
|
|
11422
11449
|
if (!candidate.value) continue;
|
|
11423
11450
|
const url = safeHttpUrl2(candidate.value, baseUrl);
|
|
11424
|
-
if (!url) continue;
|
|
11425
|
-
|
|
11451
|
+
if (!url || seen.has(url)) continue;
|
|
11452
|
+
seen.add(url);
|
|
11453
|
+
resolved.push({
|
|
11426
11454
|
url,
|
|
11427
11455
|
archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
|
|
11428
11456
|
source: candidate.source
|
|
11429
|
-
};
|
|
11457
|
+
});
|
|
11430
11458
|
}
|
|
11431
|
-
return
|
|
11459
|
+
return resolved;
|
|
11460
|
+
}
|
|
11461
|
+
function resolveFeaturedImage(input) {
|
|
11462
|
+
return resolveFeaturedImageCandidates(input)[0] ?? null;
|
|
11432
11463
|
}
|
|
11433
11464
|
var import_p_limit, WAYBACK_HOSTS, MAX_CDX_RESPONSE_BYTES, MAX_CDX_ROWS, MAX_ARCHIVE_URL_CHARS, MAX_SELECTED_TIMELINE_CELLS, TIMELINE_CDX_CONCURRENCY;
|
|
11434
11465
|
var init_wayback = __esm({
|
|
@@ -50207,7 +50238,7 @@ var PACKAGE_VERSION;
|
|
|
50207
50238
|
var init_version = __esm({
|
|
50208
50239
|
"src/version.ts"() {
|
|
50209
50240
|
"use strict";
|
|
50210
|
-
PACKAGE_VERSION = "0.
|
|
50241
|
+
PACKAGE_VERSION = "0.67.0";
|
|
50211
50242
|
}
|
|
50212
50243
|
});
|
|
50213
50244
|
|
|
@@ -50278,9 +50309,10 @@ seam is noted so you can chain them.
|
|
|
50278
50309
|
guidance, recommended sections, optional sections, and heading guidance before any write occurs.
|
|
50279
50310
|
- Host a featured image -> **commons_host_image**. The normal write sequence is **commons_prepare_entity ->
|
|
50280
50311
|
compose the payload -> commons_host_image -> commons_validate_entity -> commons_submit_entity**. Put the
|
|
50281
|
-
permanent returned URL in featuredImage.url. Use imageBase64 for
|
|
50282
|
-
|
|
50283
|
-
|
|
50312
|
+
permanent returned URL in featuredImage.url. Use imageBase64 for pasted or AI-generated bytes, artifactId for an
|
|
50313
|
+
owner-scoped MCP Scraper image artifact, or sourceUrl for either a public HTTPS image or a webpage containing one.
|
|
50314
|
+
Webpage discovery tries Open Graph, Twitter, JSON-LD, then meaningful content images. Never pass a caller-local
|
|
50315
|
+
path, opaque chat attachment reference, or private/authenticated URL.
|
|
50284
50316
|
- Validate a composed entity without writing -> **commons_validate_entity**. Use it after image hosting and before
|
|
50285
50317
|
submit. Validation does not fetch or store image bytes; an external/unregistered featured image is reported in
|
|
50286
50318
|
imageDiagnostics as not publishable until commons_host_image returns its permanent Commons URL. Validation also
|
|
@@ -52031,7 +52063,7 @@ var init_contracts = __esm({
|
|
|
52031
52063
|
});
|
|
52032
52064
|
|
|
52033
52065
|
// src/mcp/mcp-tool-schemas.ts
|
|
52034
|
-
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, HarvestPaaHostedInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, CommonsFeaturedImageDiagnosticsOutputSchema, CommonsValidateEntityOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
52066
|
+
var import_zod45, WEBSITE_URL_OR_DOMAIN_ERROR, WebsiteUrlOrDomainSchema, HarvestPaaInputSchema, HarvestPaaHostedInputSchema, ExtractUrlBaseInputSchema, ExtractUrlInputSchema, ExtractUrlLocalInputSchema, DiffPageBaseInputSchema, DiffPageInputSchema, DiffPageLocalInputSchema, MapSiteUrlsInputSchema, MapWaybackSnapshotsInputSchema, ExtractSiteInputSchema, AnalyzeSiteSimilarityInputSchema, AuditSiteInputSchema, CheckSiteExportInputSchema, SiteExportReadInputSchema, SiteExportImageInputSchema, ArchiveReadInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, RedditTrendingInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, LeadScalarSchema2, LeadRowSchema, LeadColumnMapSchema2, LeadRowsImportSourceSchema, LeadCsvTextImportSourceSchema, LeadUploadImportSourceSchema, LeadListUploadStartInputSchema, LeadListImportInputSchema, LeadRowsEnrichmentSourceSchema, ImportedLeadListSourceSchema, LeadListEnrichInputSchema, LeadListEnrichStatusInputSchema, LocationMarketsInputSchema, CommonsSearchEntitiesInputSchema, CommonsGetEntityInputSchema, CommonsGetEntityLinksetInputSchema, CommonsFeaturedImageInputSchema, CommonsMediaInputSchema, CommonsCitationInputSchema, CommonsSourceInputSchema, CommonsRelatedLinkInputSchema, CommonsClaimInputSchema, CommonsPrepareEntityInputSchema, CommonsSubmitEntityInputSchema, CommonsValidateEntityInputSchema, CommonsGetEntityLedgerInputSchema, CommonsHostImageInputSchema, CommonsHostImageOutputSchema, CommonsGetProposalInputSchema, CommonsSaveFilterInputSchema, CommonsListFiltersInputSchema, CommonsListNeedsLinksInputSchema, CommonsGenericOutputSchema, CommonsFeaturedImageDiagnosticsOutputSchema, CommonsValidateEntityOutputSchema, DirectoryWorkflowStatusInputSchema, LocalSourcebookSubmitInputSchema, LocalSourcebookCategorySchema, LocalSourcebookSchemaTypeInputSchema, LocalSourcebookTagCandidateObjectSchema, LocalSourcebookTagDecisionObjectSchema, LocalSourcebookIdentityObjectSchema, GetLocalSourcebookContractInputSchema, ListLocalSourcebookTagsInputSchema, ResolveLocalSourcebookTagsInputSchema, PrepareLocalSourcebookWriteInputSchema, ValidateLocalSourcebookWriteInputSchema, LocalSourcebookCaptureInputSchema, LocalSourcebookSubmissionStatusInputSchema, LocalSourcebookRefreshInputSchema, LocalSourcebookOutputSchema, ArtifactPointerOutputSchema, EditorialReadingRoomSiteSchema, EditorialReadingRoomImageSchema, EditorialReadingRoomArticleSchema, EditorialReadingRoomGuideInputSchema, EditorialReadingRoomGuideOutputSchema, CreateEditorialReadingRoomInputSchema, EditorialReadingRoomArtifactSchema, CreateEditorialReadingRoomOutputSchema, RenewEditorialReadingRoomDownloadInputSchema, RenewEditorialReadingRoomDownloadOutputSchema, CommonsPublicationSubdomainSchema, CommonsPreparePublicationInputSchema, CommonsValidatePublicationInputSchema, CommonsClaimPublicationInputSchema, CommonsPublishEditorialInputSchema, CommonsUpdateEditorialArticleInputSchema, CommonsGetPublicationInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryCsvArtifactOutput, DirectoryWorkflowOutputSchema, LeadSuggestedColumnOutputSchema, LeadColumnMapSuggestionOutputSchema, LeadArtifactOutputSchema, LeadListUploadStartOutputSchema, LeadListImportOutputSchema, LeadCandidateOutputSchema, LeadProgressOutputSchema, LeadBillingOutputSchema, LeadAssociatedPersonSourceOutputSchema, LeadAssociatedPersonOutputSchema, LeadSampleRowOutputSchema, LeadListEnrichmentOutputSchema, LocationDatasetProvenanceOutput, LocationMarketsOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, PageMediaAssetOutput, PageMediaArtifactOutput, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, CheckSiteExportOutputSchema, SiteExportReadOutputSchema, SiteExportImageOutputSchema, ArchiveEntryOutputSchema, ArchiveReadOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, WaybackCaptureOutputSchema, MapWaybackSnapshotsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, RedditTrendingOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, TranscriptSignalOutput, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, TestServiceConnectionInputSchema, TestServiceConnectionOutputSchema, ReadServiceConnectionInputSchema, ReadServiceConnectionOutputSchema, MetaAdCreativeMediaInputSchema, MetaAdCreativeMediaOutputSchema, ImportServiceConnectionToMemoryInputSchema, ImportServiceConnectionToMemoryOutputSchema, DescribeServiceConnectionToolInputSchema, DescribeServiceConnectionToolOutputSchema, ConnectedDataContinuationSchema, ExportConnectedServiceDataInputSchema, ConnectedDataArtifactSchema, ExportConnectedServiceDataOutputSchema, SearchConsoleTableColumnSchema, SearchConsoleTableFilterSchema, ExportSearchConsoleTableDataInputSchema, ExportSearchConsoleTableDataOutputSchema, RenewConnectedDataExportDownloadInputSchema, RenewConnectedDataExportDownloadOutputSchema, CallServiceConnectionActionInputSchema, CallServiceConnectionActionOutputSchema, SetScheduledActionConnectionsInputSchema, SetScheduledActionConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GmailSearchContactsInputSchema, GmailSearchContactsOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema, ZoomCreateMeetingInputSchema, ZoomCreateMeetingOutputSchema;
|
|
52035
52067
|
var init_mcp_tool_schemas = __esm({
|
|
52036
52068
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
52037
52069
|
"use strict";
|
|
@@ -52444,7 +52476,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52444
52476
|
idOrSlug: import_zod45.z.string().trim().min(1).max(180).describe("Transparent Commons TPW-Q entity id or /wiki/ slug whose governed RFC 9264 linkset should be read.")
|
|
52445
52477
|
};
|
|
52446
52478
|
CommonsFeaturedImageInputSchema = import_zod45.z.object({
|
|
52447
|
-
url: import_zod45.z.string().url().describe("For a publish-ready entity, use the permanent URL returned by commons_host_image.
|
|
52479
|
+
url: import_zod45.z.string().url().describe("For a publish-ready entity, use the permanent URL returned by commons_host_image. Validation reports any unregistered external URL as not ready. Do not use a chat attachment reference, caller-local path, private URL, or HTML page URL here; pass the original image or webpage source to commons_host_image first."),
|
|
52448
52480
|
alt: import_zod45.z.string().trim().max(500).optional(),
|
|
52449
52481
|
caption: import_zod45.z.string().trim().max(1e3).optional(),
|
|
52450
52482
|
sourceUrl: import_zod45.z.string().url().optional(),
|
|
@@ -52543,7 +52575,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52543
52575
|
entityId: import_zod45.z.string().trim().max(80).optional().describe("Existing Transparent Public Wiki entity id when proposing an edit. New entities normally omit this and receive a TPW-Q id."),
|
|
52544
52576
|
entityType: import_zod45.z.string().trim().max(120).default("PublicArticle").describe("Backend Wikidata-style type. Prefer precise entity classes such as SoftwareApplication, Organization, Person, Event, Place, Taxon, ScienceConcept, MathConcept, TechArticle, or PublicArticle; the public article structure should match the selected type."),
|
|
52545
52577
|
disambiguationName: import_zod45.z.string().trim().max(240).optional().describe("Clarifying name used when the concept could be confused with another entity."),
|
|
52546
|
-
featuredImage: CommonsFeaturedImageInputSchema.optional().describe("Required for auto-published public entities. First call commons_host_image
|
|
52578
|
+
featuredImage: CommonsFeaturedImageInputSchema.optional().describe("Required for auto-published public entities. First call commons_host_image with attachment bytes, an image artifact, a direct image URL, or a webpage URL, then place its returned permanent URL here. Validation does not fetch external images and reports unregistered URLs as not publishable. Submit retains a compatibility fallback for public URLs. The image is also added to the media manifest if absent."),
|
|
52547
52579
|
source: CommonsSourceInputSchema.optional().describe("Source provenance. Store original URL, source byline, and origin canonical here; canonical does not replace rights review."),
|
|
52548
52580
|
tags: import_zod45.z.array(import_zod45.z.string().trim().min(1).max(80)).max(24).optional().describe("Standardized topic tags. Use existing/searchable concepts when possible."),
|
|
52549
52581
|
keywords: import_zod45.z.array(import_zod45.z.string().trim().min(1).max(120)).max(50).optional().describe("SEO and retrieval keywords."),
|
|
@@ -52605,12 +52637,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
52605
52637
|
idOrSlug: import_zod45.z.string().trim().min(1).max(180).describe("Published entity id or /wiki/ slug whose contribution ledger should be read.")
|
|
52606
52638
|
};
|
|
52607
52639
|
CommonsHostImageInputSchema = {
|
|
52608
|
-
sourceUrl: import_zod45.z.string().url().optional().describe("
|
|
52609
|
-
imageBase64: import_zod45.z.string().min(16).optional().describe("Base64 image bytes for an image
|
|
52640
|
+
sourceUrl: import_zod45.z.string().url().optional().describe("Public HTTPS direct image URL or webpage URL. A direct image is hosted as-is; an HTML page is inspected for Open Graph, Twitter, JSON-LD, and meaningful content images, then the first usable candidate is hosted. Temporary signed URLs are consumed immediately and query credentials are not retained as provenance."),
|
|
52641
|
+
imageBase64: import_zod45.z.string().min(16).optional().describe("Base64 image bytes for an image already available to the caller, including a pasted attachment or AI-generated image. Use sourceType to preserve whether it was uploaded or generated. Keep the complete serialized request within the client transport limit."),
|
|
52642
|
+
artifactId: import_zod45.z.string().trim().min(1).max(500).optional().describe("Owner-scoped image artifact returned by an MCP Scraper media or screenshot workflow. The server reauthorizes ownership and reads the bytes directly; do not construct an artifact ID."),
|
|
52643
|
+
sourceType: import_zod45.z.enum(["uploaded", "ai_generated"]).optional().describe("Provenance for imageBase64 or artifactId. Use ai_generated only when an image model created the supplied bytes; otherwise use uploaded or omit it."),
|
|
52610
52644
|
alt: import_zod45.z.string().trim().max(500).optional().describe("Alt text describing the image for readers who cannot see it."),
|
|
52611
52645
|
license: import_zod45.z.string().trim().max(240).optional().describe("License or usage terms of the original image, preserved with the hosted copy."),
|
|
52612
52646
|
attribution: import_zod45.z.string().trim().max(500).optional().describe("Credit line for the original photographer, publication, or archive.")
|
|
52613
52647
|
};
|
|
52648
|
+
CommonsHostImageOutputSchema = {
|
|
52649
|
+
ok: import_zod45.z.boolean(),
|
|
52650
|
+
data: import_zod45.z.object({
|
|
52651
|
+
id: import_zod45.z.string().describe("Stable Commons image ID."),
|
|
52652
|
+
url: import_zod45.z.string().url().describe("Permanent Commons URL to use as featuredImage.url."),
|
|
52653
|
+
contentType: import_zod45.z.enum(["image/jpeg", "image/png", "image/gif", "image/webp"]),
|
|
52654
|
+
bytes: import_zod45.z.number().int().positive(),
|
|
52655
|
+
digest: import_zod45.z.string(),
|
|
52656
|
+
sourceUrl: import_zod45.z.string().url().nullable().describe("Sanitized direct asset URL that supplied the hosted bytes, when applicable."),
|
|
52657
|
+
sourcePageUrl: import_zod45.z.string().url().nullable().describe("Sanitized webpage URL from which the image was discovered, when applicable."),
|
|
52658
|
+
sourceKind: import_zod45.z.enum(["uploaded", "ai_generated", "artifact", "direct_url", "webpage", "registered"]),
|
|
52659
|
+
selectionSource: import_zod45.z.enum(["provided_bytes", "artifact", "direct_url", "og:image", "twitter:image", "json-ld", "content-image", "registered"]),
|
|
52660
|
+
reused: import_zod45.z.boolean().describe("True when identical image bytes were already stored.")
|
|
52661
|
+
}).optional(),
|
|
52662
|
+
error: import_zod45.z.string().optional(),
|
|
52663
|
+
message: import_zod45.z.string().optional()
|
|
52664
|
+
};
|
|
52614
52665
|
CommonsGetProposalInputSchema = {
|
|
52615
52666
|
proposalId: import_zod45.z.string().trim().min(1).max(120).describe("Proposal id returned by commons_submit_entity, in the form commons-proposal-<uuid>.")
|
|
52616
52667
|
};
|
|
@@ -56597,9 +56648,9 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
56597
56648
|
}, async (input) => executor.commonsGetEntityLedger(input));
|
|
56598
56649
|
server.registerTool("commons_host_image", {
|
|
56599
56650
|
title: "Host a Transparent Commons Image",
|
|
56600
|
-
description: "
|
|
56651
|
+
description: "Turn one supplied image source into a permanent Transparent Commons image before validation and submission. Provide exactly one of imageBase64 for a pasted or AI-generated image, artifactId for an owner-scoped MCP Scraper image artifact, or sourceUrl for either a direct public HTTPS image or a webpage containing the desired image. Webpages are inspected in deterministic order for Open Graph, Twitter, JSON-LD, and meaningful content images; unusable candidates fall through to the next candidate. Returns the permanent URL for featuredImage.url plus source and selection provenance. JPEG, PNG, GIF, and WebP are accepted up to 10 MB; SVG is rejected. Identical bytes are stored once. Never pass a caller-local path or private/authenticated URL.",
|
|
56601
56652
|
inputSchema: CommonsHostImageInputSchema,
|
|
56602
|
-
outputSchema: recordOutputSchema("commons_host_image",
|
|
56653
|
+
outputSchema: recordOutputSchema("commons_host_image", CommonsHostImageOutputSchema),
|
|
56603
56654
|
annotations: {
|
|
56604
56655
|
title: "Host a Transparent Commons Image",
|
|
56605
56656
|
readOnlyHint: false,
|
|
@@ -72162,6 +72213,9 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
72162
72213
|
if (url.protocol !== "https:") {
|
|
72163
72214
|
throw new CommonsImageError("image_url_not_https", "Commons image sources must be https URLs.");
|
|
72164
72215
|
}
|
|
72216
|
+
if (url.username || url.password) {
|
|
72217
|
+
throw new CommonsImageError("image_url_invalid", "Commons image source URLs cannot contain embedded credentials.");
|
|
72218
|
+
}
|
|
72165
72219
|
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
72166
72220
|
const addresses = (0, import_node_net2.isIP)(hostname) ? [{ address: hostname }] : await (0, import_promises16.lookup)(hostname, { all: true }).catch(() => []);
|
|
72167
72221
|
if (!addresses.length) {
|
|
@@ -72172,7 +72226,28 @@ async function assertPublicHttpsUrl(rawUrl) {
|
|
|
72172
72226
|
}
|
|
72173
72227
|
return url;
|
|
72174
72228
|
}
|
|
72175
|
-
async function
|
|
72229
|
+
async function readResponseBytes(response, maxBytes) {
|
|
72230
|
+
const declaredLength = Number(response.headers.get("content-length") ?? "0");
|
|
72231
|
+
if (declaredLength > maxBytes) {
|
|
72232
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
72233
|
+
}
|
|
72234
|
+
if (!response.body) return Buffer.alloc(0);
|
|
72235
|
+
const reader = response.body.getReader();
|
|
72236
|
+
const chunks = [];
|
|
72237
|
+
let total = 0;
|
|
72238
|
+
while (true) {
|
|
72239
|
+
const { done, value } = await reader.read();
|
|
72240
|
+
if (done) break;
|
|
72241
|
+
total += value.byteLength;
|
|
72242
|
+
if (total > maxBytes) {
|
|
72243
|
+
await reader.cancel().catch(() => void 0);
|
|
72244
|
+
throw new CommonsImageError("image_too_large", `Source exceeds the ${maxBytes} byte limit.`, 413);
|
|
72245
|
+
}
|
|
72246
|
+
chunks.push(Buffer.from(value));
|
|
72247
|
+
}
|
|
72248
|
+
return Buffer.concat(chunks, total);
|
|
72249
|
+
}
|
|
72250
|
+
async function downloadPublicSource(sourceUrl) {
|
|
72176
72251
|
let current = sourceUrl;
|
|
72177
72252
|
for (let redirect = 0; redirect <= COMMONS_IMAGE_MAX_REDIRECTS; redirect += 1) {
|
|
72178
72253
|
const url = await assertPublicHttpsUrl(current);
|
|
@@ -72207,15 +72282,11 @@ async function downloadImage(sourceUrl) {
|
|
|
72207
72282
|
retryable
|
|
72208
72283
|
);
|
|
72209
72284
|
}
|
|
72210
|
-
|
|
72211
|
-
|
|
72212
|
-
|
|
72213
|
-
|
|
72214
|
-
|
|
72215
|
-
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
72216
|
-
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
72217
|
-
}
|
|
72218
|
-
return bytes;
|
|
72285
|
+
return {
|
|
72286
|
+
bytes: await readResponseBytes(response, COMMONS_IMAGE_MAX_BYTES),
|
|
72287
|
+
finalUrl: url.toString(),
|
|
72288
|
+
contentType: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase()
|
|
72289
|
+
};
|
|
72219
72290
|
}
|
|
72220
72291
|
throw new CommonsImageError("image_redirect_limit", "Image source exceeded the redirect limit.");
|
|
72221
72292
|
}
|
|
@@ -72228,6 +72299,114 @@ function decodeBase64Image(value) {
|
|
|
72228
72299
|
}
|
|
72229
72300
|
return bytes;
|
|
72230
72301
|
}
|
|
72302
|
+
function safeProvenanceUrl(rawUrl) {
|
|
72303
|
+
try {
|
|
72304
|
+
const url = new URL(rawUrl);
|
|
72305
|
+
url.username = "";
|
|
72306
|
+
url.password = "";
|
|
72307
|
+
url.search = "";
|
|
72308
|
+
url.hash = "";
|
|
72309
|
+
return url.toString();
|
|
72310
|
+
} catch {
|
|
72311
|
+
return null;
|
|
72312
|
+
}
|
|
72313
|
+
}
|
|
72314
|
+
function looksLikeHtml(bytes, declaredType) {
|
|
72315
|
+
if (declaredType === "text/html" || declaredType === "application/xhtml+xml") return true;
|
|
72316
|
+
const prefix = bytes.subarray(0, 512).toString("utf8").trimStart().toLowerCase();
|
|
72317
|
+
return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
|
|
72318
|
+
}
|
|
72319
|
+
async function resolveCommonsImageSource(input) {
|
|
72320
|
+
const supplied = [input.sourceUrl, input.imageBase64, input.artifactId].filter((value) => Boolean(value)).length;
|
|
72321
|
+
if (supplied === 0) {
|
|
72322
|
+
throw new CommonsImageError("image_source_missing", "Provide exactly one of sourceUrl, imageBase64, or artifactId.");
|
|
72323
|
+
}
|
|
72324
|
+
if (supplied !== 1) {
|
|
72325
|
+
throw new CommonsImageError("image_source_ambiguous", "Provide only one image source: sourceUrl, imageBase64, or artifactId.");
|
|
72326
|
+
}
|
|
72327
|
+
if (input.sourceUrl && input.sourceType) {
|
|
72328
|
+
throw new CommonsImageError("image_source_type_invalid", "sourceType applies only to imageBase64 or artifactId, not sourceUrl.");
|
|
72329
|
+
}
|
|
72330
|
+
if (input.imageBase64) {
|
|
72331
|
+
return {
|
|
72332
|
+
bytes: decodeBase64Image(input.imageBase64),
|
|
72333
|
+
sourceUrl: null,
|
|
72334
|
+
sourcePageUrl: null,
|
|
72335
|
+
sourceKind: input.sourceType ?? "uploaded",
|
|
72336
|
+
selectionSource: "provided_bytes"
|
|
72337
|
+
};
|
|
72338
|
+
}
|
|
72339
|
+
if (input.artifactId) {
|
|
72340
|
+
const bytes = await readOwnedImageSourceArtifact({ artifactId: input.artifactId, ownerId: String(input.userId) });
|
|
72341
|
+
if (!bytes) {
|
|
72342
|
+
throw new CommonsImageError("image_artifact_not_found", "The image artifact was not found, expired, or is not owned by this account.", 422);
|
|
72343
|
+
}
|
|
72344
|
+
if (bytes.length > COMMONS_IMAGE_MAX_BYTES) {
|
|
72345
|
+
throw new CommonsImageError("image_too_large", `Image exceeds the ${COMMONS_IMAGE_MAX_BYTES} byte limit.`, 413);
|
|
72346
|
+
}
|
|
72347
|
+
return {
|
|
72348
|
+
bytes,
|
|
72349
|
+
sourceUrl: null,
|
|
72350
|
+
sourcePageUrl: null,
|
|
72351
|
+
sourceKind: input.sourceType ?? "artifact",
|
|
72352
|
+
selectionSource: "artifact"
|
|
72353
|
+
};
|
|
72354
|
+
}
|
|
72355
|
+
const requestedUrl = String(input.sourceUrl);
|
|
72356
|
+
const downloaded = await downloadPublicSource(requestedUrl);
|
|
72357
|
+
if (sniffImageFormat(downloaded.bytes)) {
|
|
72358
|
+
return {
|
|
72359
|
+
bytes: downloaded.bytes,
|
|
72360
|
+
sourceUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
72361
|
+
sourcePageUrl: null,
|
|
72362
|
+
sourceKind: "direct_url",
|
|
72363
|
+
selectionSource: "direct_url"
|
|
72364
|
+
};
|
|
72365
|
+
}
|
|
72366
|
+
if (!looksLikeHtml(downloaded.bytes, downloaded.contentType)) {
|
|
72367
|
+
throw new CommonsImageError(
|
|
72368
|
+
"image_format_unsupported",
|
|
72369
|
+
"The supplied URL returned neither a supported image nor an HTML webpage with a discoverable image.",
|
|
72370
|
+
415
|
|
72371
|
+
);
|
|
72372
|
+
}
|
|
72373
|
+
if (downloaded.bytes.length > COMMONS_IMAGE_PAGE_MAX_BYTES) {
|
|
72374
|
+
throw new CommonsImageError("image_page_too_large", `Webpage exceeds the ${COMMONS_IMAGE_PAGE_MAX_BYTES} byte discovery limit.`, 413);
|
|
72375
|
+
}
|
|
72376
|
+
const html = downloaded.bytes.toString("utf8");
|
|
72377
|
+
const document2 = loadHtmlDocument(html);
|
|
72378
|
+
const candidates = resolveFeaturedImageCandidates({
|
|
72379
|
+
html,
|
|
72380
|
+
meta: document2.meta,
|
|
72381
|
+
schema: document2.schema,
|
|
72382
|
+
pageUrl: downloaded.finalUrl
|
|
72383
|
+
}).slice(0, COMMONS_IMAGE_MAX_PAGE_CANDIDATES);
|
|
72384
|
+
if (!candidates.length) {
|
|
72385
|
+
throw new CommonsImageError("page_image_not_found", "The webpage did not expose a usable featured or content image.", 422);
|
|
72386
|
+
}
|
|
72387
|
+
let sawRetryableFailure = false;
|
|
72388
|
+
for (const candidate of candidates) {
|
|
72389
|
+
try {
|
|
72390
|
+
const image = await downloadPublicSource(candidate.url);
|
|
72391
|
+
if (!sniffImageFormat(image.bytes)) continue;
|
|
72392
|
+
return {
|
|
72393
|
+
bytes: image.bytes,
|
|
72394
|
+
sourceUrl: safeProvenanceUrl(image.finalUrl),
|
|
72395
|
+
sourcePageUrl: safeProvenanceUrl(downloaded.finalUrl),
|
|
72396
|
+
sourceKind: "webpage",
|
|
72397
|
+
selectionSource: candidate.source
|
|
72398
|
+
};
|
|
72399
|
+
} catch (error) {
|
|
72400
|
+
if (error instanceof CommonsImageError && error.retryable) sawRetryableFailure = true;
|
|
72401
|
+
}
|
|
72402
|
+
}
|
|
72403
|
+
throw new CommonsImageError(
|
|
72404
|
+
"page_image_candidates_unusable",
|
|
72405
|
+
"The webpage exposed image candidates, but none returned supported public image bytes.",
|
|
72406
|
+
422,
|
|
72407
|
+
sawRetryableFailure
|
|
72408
|
+
);
|
|
72409
|
+
}
|
|
72231
72410
|
function commonsImageBlobStore() {
|
|
72232
72411
|
const token6 = process.env.COMMONS_IMAGE_READ_WRITE_TOKEN?.trim() || process.env.COMMONS_IMAGE_BLOB_READ_WRITE_TOKEN?.trim() || process.env.BLOB_READ_WRITE_TOKEN?.trim();
|
|
72233
72412
|
return token6 ? new VercelBlobStore(token6) : getBlobStore();
|
|
@@ -72259,16 +72438,21 @@ async function findCommonsHostedImageByUrl(rawUrl) {
|
|
|
72259
72438
|
bytes: Number(row.bytes),
|
|
72260
72439
|
digest: String(row.digest),
|
|
72261
72440
|
sourceUrl: row.source_url ? String(row.source_url) : null,
|
|
72441
|
+
sourcePageUrl: null,
|
|
72442
|
+
sourceKind: "registered",
|
|
72443
|
+
selectionSource: "registered",
|
|
72262
72444
|
reused: true
|
|
72263
72445
|
};
|
|
72264
72446
|
}
|
|
72265
72447
|
async function hostCommonsImage(input) {
|
|
72266
72448
|
if (hostOverride) return hostOverride(input);
|
|
72267
72449
|
await ensureCommonsImageSchema();
|
|
72268
|
-
if (
|
|
72269
|
-
|
|
72450
|
+
if (input.sourceUrl && !input.imageBase64 && !input.artifactId && !input.sourceType) {
|
|
72451
|
+
const registered6 = await findCommonsHostedImageByUrl(input.sourceUrl);
|
|
72452
|
+
if (registered6) return registered6;
|
|
72270
72453
|
}
|
|
72271
|
-
const
|
|
72454
|
+
const resolved = await resolveCommonsImageSource(input);
|
|
72455
|
+
const bytes = resolved.bytes;
|
|
72272
72456
|
const contentType = sniffImageFormat(bytes);
|
|
72273
72457
|
if (!contentType) {
|
|
72274
72458
|
throw new CommonsImageError(
|
|
@@ -72290,7 +72474,10 @@ async function hostCommonsImage(input) {
|
|
|
72290
72474
|
contentType: String(reusable.content_type),
|
|
72291
72475
|
bytes: Number(reusable.bytes),
|
|
72292
72476
|
digest: digest2,
|
|
72293
|
-
sourceUrl:
|
|
72477
|
+
sourceUrl: resolved.sourceUrl,
|
|
72478
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
72479
|
+
sourceKind: resolved.sourceKind,
|
|
72480
|
+
selectionSource: resolved.selectionSource,
|
|
72294
72481
|
reused: true
|
|
72295
72482
|
};
|
|
72296
72483
|
}
|
|
@@ -72319,14 +72506,25 @@ async function hostCommonsImage(input) {
|
|
|
72319
72506
|
stored.url,
|
|
72320
72507
|
contentType,
|
|
72321
72508
|
bytes.length,
|
|
72322
|
-
|
|
72509
|
+
resolved.sourceUrl,
|
|
72323
72510
|
input.alt ?? null,
|
|
72324
72511
|
input.license ?? null,
|
|
72325
72512
|
input.attribution ?? null,
|
|
72326
72513
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
72327
72514
|
]
|
|
72328
72515
|
});
|
|
72329
|
-
return {
|
|
72516
|
+
return {
|
|
72517
|
+
id,
|
|
72518
|
+
url: stored.url,
|
|
72519
|
+
contentType,
|
|
72520
|
+
bytes: bytes.length,
|
|
72521
|
+
digest: digest2,
|
|
72522
|
+
sourceUrl: resolved.sourceUrl,
|
|
72523
|
+
sourcePageUrl: resolved.sourcePageUrl,
|
|
72524
|
+
sourceKind: resolved.sourceKind,
|
|
72525
|
+
selectionSource: resolved.selectionSource,
|
|
72526
|
+
reused: false
|
|
72527
|
+
};
|
|
72330
72528
|
}
|
|
72331
72529
|
async function hostEntityImages(input) {
|
|
72332
72530
|
const rewrite = { mediaUrls: /* @__PURE__ */ new Map(), dropped: [] };
|
|
@@ -72368,7 +72566,7 @@ async function hostEntityImages(input) {
|
|
|
72368
72566
|
}
|
|
72369
72567
|
return rewrite;
|
|
72370
72568
|
}
|
|
72371
|
-
var import_node_crypto46, import_promises16, import_node_net2, COMMONS_IMAGE_MAX_BYTES, COMMONS_IMAGE_MAX_REDIRECTS, COMMONS_IMAGE_FETCH_TIMEOUT_MS, COMMONS_IMAGE_RETRY_GUIDANCE, CommonsImageError, imageSchemaReady, imageSchemaDb, hostOverride;
|
|
72569
|
+
var import_node_crypto46, import_promises16, import_node_net2, COMMONS_IMAGE_MAX_BYTES, COMMONS_IMAGE_MAX_REDIRECTS, COMMONS_IMAGE_FETCH_TIMEOUT_MS, COMMONS_IMAGE_PAGE_MAX_BYTES, COMMONS_IMAGE_MAX_PAGE_CANDIDATES, COMMONS_IMAGE_RETRY_GUIDANCE, CommonsImageError, imageSchemaReady, imageSchemaDb, hostOverride;
|
|
72372
72570
|
var init_commons_image_store = __esm({
|
|
72373
72571
|
"src/api/commons-image-store.ts"() {
|
|
72374
72572
|
"use strict";
|
|
@@ -72377,10 +72575,15 @@ var init_commons_image_store = __esm({
|
|
|
72377
72575
|
import_node_net2 = require("net");
|
|
72378
72576
|
init_blob_store();
|
|
72379
72577
|
init_db();
|
|
72578
|
+
init_image_source_artifacts();
|
|
72579
|
+
init_html_document();
|
|
72580
|
+
init_wayback();
|
|
72380
72581
|
COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
72381
72582
|
COMMONS_IMAGE_MAX_REDIRECTS = 3;
|
|
72382
72583
|
COMMONS_IMAGE_FETCH_TIMEOUT_MS = 2e4;
|
|
72383
|
-
|
|
72584
|
+
COMMONS_IMAGE_PAGE_MAX_BYTES = 2 * 1024 * 1024;
|
|
72585
|
+
COMMONS_IMAGE_MAX_PAGE_CANDIDATES = 12;
|
|
72586
|
+
COMMONS_IMAGE_RETRY_GUIDANCE = "Call commons_host_image with imageBase64, an owner-authorized artifactId, a public HTTPS image URL, or a public HTTPS webpage containing a usable featured image. Replace featuredImage.url with the returned permanent URL, validate again, then retry commons_submit_entity with the same idempotency key.";
|
|
72384
72587
|
CommonsImageError = class extends Error {
|
|
72385
72588
|
constructor(code, message, httpStatus = 400, retryable = false, retryGuidance = COMMONS_IMAGE_RETRY_GUIDANCE) {
|
|
72386
72589
|
super(message);
|
|
@@ -75926,11 +76129,13 @@ var init_commons_routes = __esm({
|
|
|
75926
76129
|
HostImageSchema = import_zod54.z.object({
|
|
75927
76130
|
sourceUrl: import_zod54.z.string().url().optional(),
|
|
75928
76131
|
imageBase64: import_zod54.z.string().min(16).optional(),
|
|
76132
|
+
artifactId: import_zod54.z.string().min(1).max(500).optional(),
|
|
76133
|
+
sourceType: import_zod54.z.enum(["uploaded", "ai_generated"]).optional(),
|
|
75929
76134
|
alt: import_zod54.z.string().max(500).optional(),
|
|
75930
76135
|
license: import_zod54.z.string().max(240).optional(),
|
|
75931
76136
|
attribution: import_zod54.z.string().max(500).optional()
|
|
75932
|
-
}).strict().refine((body) =>
|
|
75933
|
-
message: "Provide
|
|
76137
|
+
}).strict().refine((body) => [body.sourceUrl, body.imageBase64, body.artifactId].filter(Boolean).length === 1, {
|
|
76138
|
+
message: "Provide exactly one of sourceUrl, imageBase64, or artifactId."
|
|
75934
76139
|
});
|
|
75935
76140
|
commonsApp.post("/images", auth2, async (c) => {
|
|
75936
76141
|
const parsed = HostImageSchema.safeParse(await c.req.json().catch(() => ({})));
|
package/dist/bin/api-server.js
CHANGED
|
@@ -17,9 +17,9 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
22
|
-
import("../db-
|
|
20
|
+
import("../server-MFVYRB6I.js"),
|
|
21
|
+
import("../worker-JGWV2LID.js"),
|
|
22
|
+
import("../db-ZGZAKYBW.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
25
25
|
try {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
// release-message.json
|
|
5
5
|
var release_message_default = {
|
|
6
|
-
message: "Transparent Commons now
|
|
6
|
+
message: "Transparent Commons can now turn pasted images, generated image bytes, stored image artifacts, direct image URLs, or ordinary webpage URLs into one permanent Commons-hosted image before validation and publishing."
|
|
7
7
|
};
|
|
8
8
|
|
|
9
9
|
// src/install-terminal.ts
|
|
@@ -112,7 +112,7 @@ function renderInstallTerminal(options) {
|
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
// src/version.ts
|
|
115
|
-
var PACKAGE_VERSION = "0.
|
|
115
|
+
var PACKAGE_VERSION = "0.67.0";
|
|
116
116
|
|
|
117
117
|
// bin/mcp-scraper-install.ts
|
|
118
118
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-IHDTONZV.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-GT63KEUB.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|