mcp-scraper 0.66.6 → 0.67.0

package/dist/{chunk-VMWVEKZT.js → chunk-XOOKZYZG.js}

@@ -8,13 +8,13 @@ import {
 } from "./chunk-PGJQDMC2.js";
 import {
   sanitizeOutboundDiagnostics
-} from "./chunk-3EYALCYK.js";
+} from "./chunk-2L4C4DAZ.js";
 import {
   browserServiceProfileName,
   browserServiceProfileSaveChanges,
   recordVendorUsage,
   vendorCostUsd
-} from "./chunk-3TKKJQB4.js";
+} from "./chunk-ES25GP6C.js";
 import {
   buildSerpEmailQuery,
   classifyEmailScope,
@@ -25,7 +25,7 @@ import {
 } from "./chunk-VXLU74YZ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-NLEAYKOC.js";
+} from "./chunk-GT63KEUB.js";
 import {
   createPrivateArtifact,
   privateArtifactOwnerId,
@@ -45,7 +45,7 @@ import {
   normalizePublicErrorCode,
   publicErrorMessage,
   sanitizeVendorName
-} from "./chunk-55B3T5L2.js";
+} from "./chunk-65FTMB7G.js";
 
 // src/harvest-timeout.ts
 var SCRAPE_FUNCTION_MAX_MS = 8e5;
@@ -388,11 +388,21 @@ seam is noted so you can chain them.
 - Plan a new or edited entity -> **commons_prepare_entity**. This is the Commons equivalent of
   prepare-memory-write: it returns the live entity profile contract, duplicate candidates, tag reuse/review/create
   guidance, recommended sections, optional sections, and heading guidance before any write occurs.
-- Validate a composed entity without writing -> **commons_validate_entity**. Use it after composing and before submit
-  to catch missing featured images, missing source/body evidence, duplicate candidates, and
-  unsupported placeholder sections.
+- Host a featured image -> **commons_host_image**. The normal write sequence is **commons_prepare_entity ->
+  compose the payload -> commons_host_image -> commons_validate_entity -> commons_submit_entity**. Put the
+  permanent returned URL in featuredImage.url. Use imageBase64 for pasted or AI-generated bytes, artifactId for an
+  owner-scoped MCP Scraper image artifact, or sourceUrl for either a public HTTPS image or a webpage containing one.
+  Webpage discovery tries Open Graph, Twitter, JSON-LD, then meaningful content images. Never pass a caller-local
+  path, opaque chat attachment reference, or private/authenticated URL.
+- Validate a composed entity without writing -> **commons_validate_entity**. Use it after image hosting and before
+  submit. Validation does not fetch or store image bytes; an external/unregistered featured image is reported in
+  imageDiagnostics as not publishable until commons_host_image returns its permanent Commons URL. Validation also
+  catches missing source/body evidence, duplicate candidates, and unsupported placeholder sections.
 - Create or edit a public wiki entity -> **commons_submit_entity**. This is a governed proposal write,
-  not direct rendered HTML editing. Normal workflow is prepare -> validate -> submit. Pick the closest entity type
+  not direct rendered HTML editing. Submit retains defensive auto-hosting for a stable direct public HTTPS image,
+  but image ingress failure returns an actionable tool error: host different bytes or a different stable source,
+  replace featuredImage.url, validate again, and retry the same intended write with the same idempotency key.
+  Pick the closest entity type
   first: SoftwareApplication, Organization, Person, Event, Place, Taxon, ScienceConcept, MathConcept, TechArticle,
   Trail, or PublicArticle.
   Creating a new entity costs 10 Credits; editing an existing entity costs 2 Credits. Preparation, validation,
@@ -7496,7 +7506,7 @@ var CommonsGetEntityLinksetInputSchema = {
   idOrSlug: z6.string().trim().min(1).max(180).describe("Transparent Commons TPW-Q entity id or /wiki/ slug whose governed RFC 9264 linkset should be read.")
 };
 var CommonsFeaturedImageInputSchema = z6.object({
-  url: z6.string().url().describe("Required public image URL for a publishable entity. Use extract_url includeFeaturedImage or preserved media when available."),
+  url: z6.string().url().describe("For a publish-ready entity, use the permanent URL returned by commons_host_image. Validation reports any unregistered external URL as not ready. Do not use a chat attachment reference, caller-local path, private URL, or HTML page URL here; pass the original image or webpage source to commons_host_image first."),
   alt: z6.string().trim().max(500).optional(),
   caption: z6.string().trim().max(1e3).optional(),
   sourceUrl: z6.string().url().optional(),
@@ -7588,14 +7598,14 @@ var CommonsPrepareEntityInputSchema = {
   maxCandidates: z6.number().int().min(1).max(20).default(8).describe("Maximum duplicate candidates to return.")
 };
 var CommonsSubmitEntityInputSchema = {
-  idempotencyKey: z6.string().trim().min(8).max(200).describe("Required unique opaque ID for this intended Commons write. Reuse only when retrying the same write after a timeout; use a new value for each intentional create or edit."),
+  idempotencyKey: z6.string().trim().min(8).max(200).describe("Required unique opaque ID for this intended Commons write. Reuse it when retrying the same intended write after a timeout or after correcting a pre-persistence image-ingress failure; use a new value for each intentional create or edit."),
   title: z6.string().trim().min(3).max(180).describe("Public page title. This is also used for duplicate slug/entity checks when slug is omitted."),
   description: z6.string().trim().min(8).max(600).describe("Short article summary and schema.org description."),
   slug: z6.string().trim().max(180).optional().describe("Optional URL slug. Omit to derive one from title."),
   entityId: z6.string().trim().max(80).optional().describe("Existing Transparent Public Wiki entity id when proposing an edit. New entities normally omit this and receive a TPW-Q id."),
   entityType: z6.string().trim().max(120).default("PublicArticle").describe("Backend Wikidata-style type. Prefer precise entity classes such as SoftwareApplication, Organization, Person, Event, Place, Taxon, ScienceConcept, MathConcept, TechArticle, or PublicArticle; the public article structure should match the selected type."),
   disambiguationName: z6.string().trim().max(240).optional().describe("Clarifying name used when the concept could be confused with another entity."),
-  featuredImage: CommonsFeaturedImageInputSchema.optional().describe("Required for auto-published public entities. The image is also added to the media manifest if absent."),
+  featuredImage: CommonsFeaturedImageInputSchema.optional().describe("Required for auto-published public entities. First call commons_host_image with attachment bytes, an image artifact, a direct image URL, or a webpage URL, then place its returned permanent URL here. Validation does not fetch external images and reports unregistered URLs as not publishable. Submit retains a compatibility fallback for public URLs. The image is also added to the media manifest if absent."),
   source: CommonsSourceInputSchema.optional().describe("Source provenance. Store original URL, source byline, and origin canonical here; canonical does not replace rights review."),
   tags: z6.array(z6.string().trim().min(1).max(80)).max(24).optional().describe("Standardized topic tags. Use existing/searchable concepts when possible."),
   keywords: z6.array(z6.string().trim().min(1).max(120)).max(50).optional().describe("SEO and retrieval keywords."),
@@ -7657,12 +7667,31 @@ var CommonsGetEntityLedgerInputSchema = {
   idOrSlug: z6.string().trim().min(1).max(180).describe("Published entity id or /wiki/ slug whose contribution ledger should be read.")
 };
 var CommonsHostImageInputSchema = {
-  sourceUrl: z6.string().url().optional().describe("Public https image URL to download and host. The wiki never serves a third-party URL directly, so pass the original source here rather than putting it on the entity."),
-  imageBase64: z6.string().min(16).optional().describe("Base64 image bytes for an image you already hold. Use for images under about 3 MB; larger files should be published from a URL."),
+  sourceUrl: z6.string().url().optional().describe("Public HTTPS direct image URL or webpage URL. A direct image is hosted as-is; an HTML page is inspected for Open Graph, Twitter, JSON-LD, and meaningful content images, then the first usable candidate is hosted. Temporary signed URLs are consumed immediately and query credentials are not retained as provenance."),
+  imageBase64: z6.string().min(16).optional().describe("Base64 image bytes for an image already available to the caller, including a pasted attachment or AI-generated image. Use sourceType to preserve whether it was uploaded or generated. Keep the complete serialized request within the client transport limit."),
+  artifactId: z6.string().trim().min(1).max(500).optional().describe("Owner-scoped image artifact returned by an MCP Scraper media or screenshot workflow. The server reauthorizes ownership and reads the bytes directly; do not construct an artifact ID."),
+  sourceType: z6.enum(["uploaded", "ai_generated"]).optional().describe("Provenance for imageBase64 or artifactId. Use ai_generated only when an image model created the supplied bytes; otherwise use uploaded or omit it."),
   alt: z6.string().trim().max(500).optional().describe("Alt text describing the image for readers who cannot see it."),
   license: z6.string().trim().max(240).optional().describe("License or usage terms of the original image, preserved with the hosted copy."),
   attribution: z6.string().trim().max(500).optional().describe("Credit line for the original photographer, publication, or archive.")
 };
+var CommonsHostImageOutputSchema = {
+  ok: z6.boolean(),
+  data: z6.object({
+    id: z6.string().describe("Stable Commons image ID."),
+    url: z6.string().url().describe("Permanent Commons URL to use as featuredImage.url."),
+    contentType: z6.enum(["image/jpeg", "image/png", "image/gif", "image/webp"]),
+    bytes: z6.number().int().positive(),
+    digest: z6.string(),
+    sourceUrl: z6.string().url().nullable().describe("Sanitized direct asset URL that supplied the hosted bytes, when applicable."),
+    sourcePageUrl: z6.string().url().nullable().describe("Sanitized webpage URL from which the image was discovered, when applicable."),
+    sourceKind: z6.enum(["uploaded", "ai_generated", "artifact", "direct_url", "webpage", "registered"]),
+    selectionSource: z6.enum(["provided_bytes", "artifact", "direct_url", "og:image", "twitter:image", "json-ld", "content-image", "registered"]),
+    reused: z6.boolean().describe("True when identical image bytes were already stored.")
+  }).optional(),
+  error: z6.string().optional(),
+  message: z6.string().optional()
+};
 var CommonsGetProposalInputSchema = {
   proposalId: z6.string().trim().min(1).max(120).describe("Proposal id returned by commons_submit_entity, in the form commons-proposal-<uuid>.")
 };
@@ -7690,6 +7719,30 @@ var CommonsGenericOutputSchema = {
   error: z6.string().optional(),
   message: z6.string().optional()
 };
+var CommonsFeaturedImageDiagnosticsOutputSchema = z6.object({
+  state: z6.enum(["missing", "external_unhosted", "commons_hosted"]).describe("Storage-readiness state for the candidate featured image."),
+  url: z6.string().url().nullable().describe("Normalized candidate featured-image URL, or null when none was supplied."),
+  commonsImageId: z6.string().nullable().describe("Registered Commons image ID when metadata proves the image is hosted, otherwise null."),
+  readyForPublish: z6.boolean().describe("Whether Commons storage metadata proves this featured image is ready to publish."),
+  submitWillAttemptAutoHost: z6.boolean().describe("Whether submit will use its compatibility fallback to try hosting this external image."),
+  requiredAction: z6.string().nullable().describe("Action the caller must take before submit, or null when the image is ready.")
+}).describe("Featured-image storage readiness determined without fetching or storing external bytes.");
+var CommonsValidateEntityOutputSchema = {
+  ok: z6.boolean(),
+  data: z6.object({
+    ok: z6.literal(true),
+    valid: z6.boolean(),
+    publishable: z6.boolean(),
+    autoPublishEligible: z6.boolean(),
+    errors: z6.array(z6.string()),
+    warnings: z6.array(z6.string()),
+    imageDiagnostics: z6.object({
+      featuredImage: CommonsFeaturedImageDiagnosticsOutputSchema
+    }).describe("Read-only image readiness and recovery guidance for this payload.")
+  }).passthrough().optional(),
+  error: z6.string().optional(),
+  message: z6.string().optional()
+};
 var DirectoryWorkflowStatusInputSchema = {
   jobId: z6.string().trim().min(1).describe("The jobId returned by directory_workflow. Poll until status is complete, partial, empty, or failed.")
 };
@@ -11527,9 +11580,9 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => executor.commonsPrepareEntity(input));
   server.registerTool("commons_validate_entity", {
     title: "Transparent Commons Validate Entity",
-    description: "Validate a proposed Transparent Commons entity payload without writing. Checks publishable basics, featured image, source/body evidence, existing-entity conflict state, heading profile alignment, and unsupported placeholder sections. Call this after composing the page and before commons_submit_entity.",
+    description: "Read-only validation for a proposed Transparent Commons entity payload. Call it after commons_prepare_entity, composing the page, and commons_host_image. It does not fetch or store image bytes: an external or otherwise unregistered featured image remains not publishable until commons_host_image returns the permanent Commons URL. Inspect imageDiagnostics for readiness and the required action before commons_submit_entity. Also checks source/body evidence, existing-entity conflict state, heading profile alignment, and unsupported placeholder sections.",
     inputSchema: CommonsValidateEntityInputSchema,
-    outputSchema: recordOutputSchema("commons_validate_entity", CommonsGenericOutputSchema),
+    outputSchema: recordOutputSchema("commons_validate_entity", CommonsValidateEntityOutputSchema),
     annotations: {
       title: "Transparent Commons Validate Entity",
       readOnlyHint: true,
@@ -11540,7 +11593,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => executor.commonsValidateEntity(input));
   server.registerTool("commons_submit_entity", {
     title: "Transparent Commons Governed Entity Write",
-    description: "Create or propose an edit to a Transparent Commons public wiki entity through the governed MCP Scraper write plane. A new entity costs 10 Credits; an edit to an existing entity costs 2 Credits. Preparation and validation are free, and idempotent retries are not charged twice. This never edits rendered HTML directly. Normal workflow is commons_prepare_entity, compose, commons_validate_entity, then submit. commons_prepare_entity returns the entity profile contract and section guidance to compose against. Transparent Commons is an open wiki: any account may edit any entity, and every edit publishes and is recorded on the public contribution ledger. Publishing requires a featured image and body/source evidence. Pass baseRevision when editing so the ledger can show whether the edit was written on top of a newer revision. Requires idempotencyKey.",
+    description: "Charged governed write that creates or proposes an edit to a Transparent Commons public wiki entity. A new entity costs 10 Credits; an edit costs 2 Credits. Preparation, image hosting, and validation are free, and idempotent retries are not charged twice. Normal workflow is commons_prepare_entity, compose the page, commons_host_image, commons_validate_entity, then submit with the returned permanent featuredImage.url. Submit retains a compatibility fallback that auto-hosts a stable direct public HTTPS image URL; if image ingress fails, it returns an actionable image tool error so you can host different bytes or a different stable source, replace the URL, validate again, and retry the same intended write with the same idempotencyKey. This never edits rendered HTML directly. Transparent Commons is an open wiki: any account may edit any entity, and every applied edit is recorded on the public contribution ledger. Publishing requires a ready featured image and body/source evidence. Pass baseRevision when editing so the ledger can show whether the edit was written on top of a newer revision. Requires idempotencyKey.",
     inputSchema: CommonsSubmitEntityInputSchema,
     outputSchema: recordOutputSchema("commons_submit_entity", CommonsGenericOutputSchema),
     annotations: {
@@ -11566,9 +11619,9 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => executor.commonsGetEntityLedger(input));
   server.registerTool("commons_host_image", {
     title: "Host a Transparent Commons Image",
-    description: "Host an image from a public https URL or base64 bytes, returning a permanent Commons URL. Commons never serves a third-party image URL \u2014 commons_submit_entity hosts external images automatically, so use this only to host one before or independently of a write. JPEG, PNG, GIF, WebP up to 10 MB; SVG rejected. Identical bytes are stored once.",
+    description: "Turn one supplied image source into a permanent Transparent Commons image before validation and submission. Provide exactly one of imageBase64 for a pasted or AI-generated image, artifactId for an owner-scoped MCP Scraper image artifact, or sourceUrl for either a direct public HTTPS image or a webpage containing the desired image. Webpages are inspected in deterministic order for Open Graph, Twitter, JSON-LD, and meaningful content images; unusable candidates fall through to the next candidate. Returns the permanent URL for featuredImage.url plus source and selection provenance. JPEG, PNG, GIF, and WebP are accepted up to 10 MB; SVG is rejected. Identical bytes are stored once. Never pass a caller-local path or private/authenticated URL.",
     inputSchema: CommonsHostImageInputSchema,
-    outputSchema: recordOutputSchema("commons_host_image", CommonsGenericOutputSchema),
+    outputSchema: recordOutputSchema("commons_host_image", CommonsHostImageOutputSchema),
     annotations: {
       title: "Host a Transparent Commons Image",
       readOnlyHint: false,

package/dist/{chunk-PMCT2YQM.js → chunk-Y46YNQMM.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-RJ7JVYKU.js";
 import {
   getDb
-} from "./chunk-55B3T5L2.js";
+} from "./chunk-65FTMB7G.js";
 
 // src/api/location-data-repository.ts
 import { createHash, randomUUID } from "crypto";

package/dist/{db-4RVZ3NYZ.js → db-ZGZAKYBW.js}

@@ -126,7 +126,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-55B3T5L2.js";
+} from "./chunk-65FTMB7G.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,

package/dist/{extract-bundle-EA5URLXN.js → extract-bundle-QQIYJSBW.js}

@@ -4,7 +4,7 @@ import {
   downloadAsset,
   embedCommonsTexts,
   publicizeExtractionFailure
-} from "./chunk-DBWWIK7O.js";
+} from "./chunk-LN6N3YLK.js";
 import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
@@ -18,12 +18,12 @@ import {
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-KXRWY7DZ.js";
+} from "./chunk-LCGTNWOT.js";
 import "./chunk-DNM65UCK.js";
 import "./chunk-AYJHI2FK.js";
 import {
   getDb
-} from "./chunk-55B3T5L2.js";
+} from "./chunk-65FTMB7G.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";

package/dist/index.cjs

@@ -310,7 +310,26 @@ var PUBLIC_ERROR_CODES = [
   "site_export_not_found",
   "site_export_format_unavailable",
   "site_export_read_failed",
-  "site_export_image_not_found"
+  "site_export_image_not_found",
+  "image_url_invalid",
+  "image_url_not_https",
+  "image_host_unresolvable",
+  "image_host_blocked",
+  "image_redirect_invalid",
+  "image_source_unavailable",
+  "image_too_large",
+  "image_redirect_limit",
+  "image_base64_invalid",
+  "image_source_missing",
+  "image_source_ambiguous",
+  "image_source_type_invalid",
+  "image_artifact_not_found",
+  "image_page_too_large",
+  "page_image_not_found",
+  "page_image_candidates_unusable",
+  "image_format_unsupported",
+  "image_storage_unconfigured",
+  "featured_image_unhostable"
 ];
 var PUBLIC_ERROR_TYPES = [
   "request_aborted",

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-K7Y27ND3.js";
+} from "./chunk-62LEXS5O.js";
 import {
   recordVendorUsage
-} from "./chunk-3TKKJQB4.js";
+} from "./chunk-ES25GP6C.js";
 import "./chunk-77LQV73B.js";
-import "./chunk-55B3T5L2.js";
+import "./chunk-65FTMB7G.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{lead-list-enrichment-repository-IBXNWM6H.js → lead-list-enrichment-repository-CXLVIV3R.js}

@@ -30,8 +30,8 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-ABYBPODM.js";
-import "./chunk-55B3T5L2.js";
+} from "./chunk-UL4ZKAWZ.js";
+import "./chunk-65FTMB7G.js";
 export {
   LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
   LEAD_LIST_RUNNING_STALE_MINUTES,

package/dist/{location-data-repository-WHHQUCUI.js → location-data-repository-G72M7DTN.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-PMCT2YQM.js";
+} from "./chunk-Y46YNQMM.js";
 import "./chunk-RJ7JVYKU.js";
-import "./chunk-55B3T5L2.js";
+import "./chunk-65FTMB7G.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,