mcp-scraper 0.35.1 → 0.37.0

package/dist/site-extract-repository-I3VM6WXN.js

@@ -0,0 +1,62 @@
+import {
+  abandonExtractSettlement,
+  claimFailedExtractJobForRefinalize,
+  completeExtractJob,
+  countSuccessfulPages,
+  createExtractJob,
+  createOrGetExtractJob,
+  extractJobLimitInfo,
+  failExtractJob,
+  failStaleRunningExtractJob,
+  failUnfundedExtractJob,
+  finishExtractJob,
+  getExtractJob,
+  getExtractJobByIdempotencyKey,
+  getExtractedImageLinks,
+  getExtractedPages,
+  getExtractedUrls,
+  listExtractJobs,
+  listFundedPendingExtractJobs,
+  listStaleRunningExtractJobs,
+  listUnsettledExtractJobs,
+  markExtractJobDispatchAttempt,
+  recordExtractJobDispatchFailure,
+  recordExtractSettlementFailure,
+  saveExtractPages,
+  setExtractJobTotal,
+  settleExtractJob,
+  terminalExtractJobStatus
+} from "./chunk-Q35WZJJK.js";
+import "./chunk-PWPUKR5U.js";
+import "./chunk-44HZLHDV.js";
+import "./chunk-CSCD2HNS.js";
+export {
+  abandonExtractSettlement,
+  claimFailedExtractJobForRefinalize,
+  completeExtractJob,
+  countSuccessfulPages,
+  createExtractJob,
+  createOrGetExtractJob,
+  extractJobLimitInfo,
+  failExtractJob,
+  failStaleRunningExtractJob,
+  failUnfundedExtractJob,
+  finishExtractJob,
+  getExtractJob,
+  getExtractJobByIdempotencyKey,
+  getExtractedImageLinks,
+  getExtractedPages,
+  getExtractedUrls,
+  listExtractJobs,
+  listFundedPendingExtractJobs,
+  listStaleRunningExtractJobs,
+  listUnsettledExtractJobs,
+  markExtractJobDispatchAttempt,
+  recordExtractJobDispatchFailure,
+  recordExtractSettlementFailure,
+  saveExtractPages,
+  setExtractJobTotal,
+  settleExtractJob,
+  terminalExtractJobStatus
+};
+//# sourceMappingURL=site-extract-repository-I3VM6WXN.js.map

package/dist/site-extract-repository-I3VM6WXN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/{worker-645BZPEK.js → worker-TTFXPPDK.js}

@@ -3,21 +3,21 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-3HBPKR5G.js";
+} from "./chunk-FSAXLDB3.js";
 import {
   harvest
-} from "./chunk-XVVNKASZ.js";
+} from "./chunk-LOPKN3YL.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-4ZB3X6BQ.js";
+} from "./chunk-MA5JBAUZ.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-CB5C3BPB.js";
 import {
   MC_COSTS,
   serpActualCostMc
-} from "./chunk-BWXLTWF7.js";
-import "./chunk-M2S27J6Z.js";
+} from "./chunk-PWPUKR5U.js";
+import "./chunk-44HZLHDV.js";
 import {
   claimPendingJob,
   completeJob,
@@ -25,7 +25,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-62DQAWPF.js";
+} from "./chunk-CSCD2HNS.js";
 
 // src/api/webhook.ts
 async function deliverWebhook(url, payload, retries = 3) {
@@ -139,4 +139,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-645BZPEK.js.map
+//# sourceMappingURL=worker-TTFXPPDK.js.map

package/docs/hosted-location-data.md

@@ -0,0 +1,108 @@
+# Hosted location data
+
+Production directory workflows and `location_markets` read versioned location snapshots from Turso. They do not read a server-local CSV and do not call Census during a customer request.
+
+Two datasets are active independently:
+
+- `us_census_places:<STATE>` stores every Census place and its 2020–2025 population estimates for one state.
+- `us_zip_groups` stores normalized city ZIP and county groups for all imported states.
+
+Each import receives an immutable dataset ID. An active pointer is switched only after every normalized row is durable, so readers see either the old complete version or the new complete version. Responses include both active dataset IDs, source URLs, and refresh timestamps.
+
+## Required environment
+
+```dotenv
+TURSO_DATABASE_URL=...
+TURSO_AUTH_TOKEN=...
+MCP_SCRAPER_LOCATION_ADMIN_KEY=...
+```
+
+`MCP_SCRAPER_USZIPS_CSV_URL` may hold the default public HTTPS ZIP source. The import endpoint also accepts a source URL explicitly. `MCP_SCRAPER_USZIPS_CSV_PATH` remains available only for local tests and local CLI runs.
+
+ZIP imports are fail-closed: a request URL must exactly match `MCP_SCRAPER_USZIPS_CSV_URL` or one entry in `MCP_SCRAPER_LOCATION_IMPORT_SOURCE_ALLOWLIST`. Redirects are rejected, private-network targets are rejected, and response bytes are streamed through a 64 MiB in-flight limit. Census sync does not accept a URL; it constructs the trusted `www2.census.gov` state URL server-side.
+
+## Publish the nationwide GeoNames source
+
+The production ZIP/county snapshot is derived from the official GeoNames US postal-code archive at `https://download.geonames.org/export/zip/US.zip`. GeoNames postal-code data is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/); retain attribution to [GeoNames](https://www.geonames.org/) anywhere this derived dataset is redistributed.
+
+Publish a normalized, immutable public CSV to Vercel Blob:
+
+```bash
+export LOCATION_DATA_READ_WRITE_TOKEN='...'
+npm run locations:publish-geonames
+```
+
+`BLOB_READ_WRITE_TOKEN` is also accepted for projects that use their default
+public Blob store. Production uses a dedicated public store connected with the
+`LOCATION_DATA_` environment-variable prefix so this dataset cannot collide
+with private connected-data artifacts.
+
+The publisher downloads the direct HTTPS ZIP without following redirects, applies status, timeout, compressed-size, and extracted-size limits, extracts only `US.txt`, and converts the documented tab-separated fields to `state_abbr,zipcode,county,city`. It then runs the same production CSV parser and nationwide gate used by the import route: all 50 states plus DC and at least 25,000 distinct five-digit ZIPs must be present.
+
+The Blob pathname includes the normalized CSV's SHA-256 digest. Random suffixes and overwrites are disabled, so a published URL is an immutable snapshot. The command prints only non-secret counts, hashes, paths, and public URLs; copy its `url` value into `MCP_SCRAPER_USZIPS_CSV_URL`. A repeat publication of byte-identical data intentionally fails instead of replacing the existing content-addressed object.
+
+## Seed and refresh
+
+Bootstrap the nationwide ZIP snapshot plus all 50 states and DC in one command:
+
+```bash
+export MCP_SCRAPER_LOCATION_ADMIN_KEY='...'
+export MCP_SCRAPER_USZIPS_CSV_URL='https://...blob.vercel-storage.com/location-data/geonames/us/geonames-us-<sha256>.csv'
+npm run locations:sync
+```
+
+The bootstrap uses three concurrent state requests by default, retries retryable HTTP/network failures up to three times, prints a per-state result and final failed-state list, and exits nonzero if any dataset failed. It never prints the admin key or ZIP source URL. Useful bounded refreshes include:
+
+```bash
+# Refresh only selected Census snapshots and leave the active ZIP version unchanged.
+npm run locations:sync -- --skip-zips --states=TN,CO,CA
+
+# Point the operator at a preview API and lower concurrency.
+npm run locations:sync -- --base-url=https://preview.example.com --concurrency=2
+```
+
+The individual admin calls below remain available for targeted recovery.
+
+Sync one state's trusted Census snapshot:
+
+```bash
+curl -sS https://mcpscraper.dev/locations/admin/sync-census \
+  -H "x-location-admin-key: $MCP_SCRAPER_LOCATION_ADMIN_KEY" \
+  -H 'content-type: application/json' \
+  --data '{"state":"TN"}'
+```
+
+Import the nationwide ZIP/county CSV from a public HTTPS object:
+
+```bash
+curl -sS https://mcpscraper.dev/locations/admin/import \
+  -H "x-location-admin-key: $MCP_SCRAPER_LOCATION_ADMIN_KEY" \
+  -H 'content-type: application/json' \
+  --data '{"sourceUrl":"https://data.example.com/uszips.csv"}'
+```
+
+The ZIP import accepts the common header aliases `state_abbr|state|state_id`, `zipcode|zip|zip_code`, `city|primary_city`, and `county|county_name`. A replacement must cover all 50 states plus DC and contain at least 25,000 distinct five-digit ZIPs. Unsupported state codes and malformed rows are counted as rejected rows in the dataset record. A partial or undersized snapshot is rejected without changing the active pointer. Re-importing byte-identical active data is idempotent.
+
+Census imports verify that every place row's `STATE` FIPS matches the requested state before activation. `/locations/status` reports missing population states, required state count, ZIP coverage, rejected rows, and readiness; `available` becomes true only when all 51 population snapshots and a nationwide ZIP snapshot are active.
+
+## Read and verify
+
+Dataset readiness requires an API key:
+
+```bash
+curl -sS https://mcpscraper.dev/locations/status \
+  -H "x-api-key: $MCP_SCRAPER_API_KEY"
+```
+
+Query hosted markets directly:
+
+```bash
+curl -sS 'https://mcpscraper.dev/locations/markets?state=TN&populationYear=2025&minPopulation=100000&maxResults=25&includeZipGroups=true' \
+  -H "x-api-key: $MCP_SCRAPER_API_KEY"
+```
+
+The same read is exposed to agents as `location_markets`. Filters are applied against the hosted population and ZIP rows before the population-sorted result limit. If a required state snapshot or ZIP snapshot has not been activated, the API returns a typed `503` instead of silently falling back to a local path or a live Census fetch.
+
+## Operational follow-up
+
+The active-pointer design prevents partial data from becoming visible and preserves old dataset rows for rollback. A future maintenance pass should add an explicit cross-process import lease, an admin rollback/history endpoint, and retention cleanup for superseded snapshot rows. These do not weaken current read isolation or coverage gates, but they will make concurrent operator refreshes and long-term storage management more ergonomic.

package/docs/mcp-tool-craft-lint.generated.md

@@ -1,9 +1,9 @@
 # MCP Tool Craft Lint
 
-Generated: 2026-07-26T13:48:25.492Z
-Manifest: `/Users/vilovieta/.codex/worktrees/mcp-scraper-query-fanout-transport-fix/docs/mcp-tool-manifest.generated.json`
+Generated: 2026-07-28T15:41:51.205Z
+Manifest: `/private/tmp/mcp-scraper-zoom-transcript/docs/mcp-tool-manifest.generated.json`
 
-Tools: 167
+Tools: 170
 Checks per tool: 7
 Failing tools: 0
 
@@ -68,6 +68,7 @@ Failing tools: 0
 | `describe_service_connection_tool` | 7/7 | none |
 | `diff_page` | 7/7 | none |
 | `directory_workflow` | 7/7 | none |
+| `directory_workflow_status` | 7/7 | none |
 | `export_connected_service_data` | 7/7 | none |
 | `export_search_console_table_data` | 7/7 | none |
 | `extract_site` | 7/7 | none |
@@ -103,7 +104,9 @@ Failing tools: 0
 | `list-shared-with-me` | 7/7 | none |
 | `list-vaults` | 7/7 | none |
 | `list-webhooks` | 7/7 | none |
+| `location_markets` | 7/7 | none |
 | `map_site_urls` | 7/7 | none |
+| `map_wayback_snapshots` | 7/7 | none |
 | `maps_place_intel` | 7/7 | none |
 | `maps_search` | 7/7 | none |
 | `memory-backlinks` | 7/7 | none |