mcp-scraper 0.77.0 → 0.79.0

package/CHANGELOG.md

@@ -4,6 +4,59 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 
 ## [Unreleased]
 
+## [0.79.0] - 2026-08-28
+
+### Added
+
+- Added Scheduling as the canonical Personal Assistant setup surface, with connection readiness for Gmail, Calendar, Zoom, browser profiles, Memory, SMS, and email; exact schedule confirmation; approval and spend review; run history; and explicit watch/takeover states.
+- Added owner-scoped browser profiles that can hold multiple independently verified login bindings, while every browser schedule grant selects one exact profile, login, domain, and action set.
+- Added immutable schedule revisions, readiness receipts, append-only activation records, additive legacy schedule projection, and single-owner occurrence transition receipts so migration cannot silently infer browser authority or double-dispatch work.
+- Added Kernel and private-Mac browser runtime boundaries with collision-resistant tenant namespaces, per-owner concurrency ceilings, bounded sessions, explicit and timeout cleanup, owner-qualified account deletion, and provider deletion readback.
+- Added an owner-controlled Personal Assistant that brings SMS/MMS, Gmail, Google Calendar, Zoom, browser work, reminders, and Memory context packets into one governed workflow with immutable plans, approval checkpoints, spend limits, and durable receipts.
+- Added Twilio number discovery, owned-number attachment, purchase and registration previews, Messaging Service readiness, signed inbound and delivery webhooks, safe MMS ingestion, deterministic opt-out handling, single and reviewed bulk messaging, and reconciliation for unknown provider outcomes.
+- Added immutable, revisioned Memory context packets with source and attachment provenance, Gmail full-message imports, MMS media metadata, lifecycle controls, and readback verification against the selected vault.
+- Added a portable Codex harness and command-ledger runtime for restart-safe assistant work, bounded wake packets, signed worker claims, short-lived tool leases, approval suspension and resume, deterministic reminders, and governed Gmail, Calendar, Zoom, and browser execution.
+- Added the authenticated `/api/v1/assistant` owner API, signed private worker routes, an `/assistant` web workspace, 15 public assistant MCP tools, two owner-scoped resources, and the `assistant_setup` prompt. Private webhook, Cron, database, credential, and worker operations remain outside the public contract.
+- Added a sanitized legacy-data salvage and migration path that preserves explicit scoped consent and inbound evidence, quarantines unresolved ownership or provider identifiers, rejects inferred consent and legacy send-ready claims, and supports deterministic rollback from a disposable staging store.
+
+### Changed
+
+- **Breaking — Personal Assistant scheduler ownership:** Personal Assistant schedules and model-assisted work now use the main command ledger and portable harness instead of the legacy Mastra/Inngest scheduled-action path. Environments adopting the assistant must provision the dedicated `assistant` database schema and signed worker boundary; legacy Scheduler remains only for unrelated maintenance workloads.
+- **Public contract:** Owner authority now comes only from authenticated server context. Assistant request bodies use opaque references and cannot supply owner IDs, provider identifiers, credentials, signed URLs, SQL, or private worker operations; mutations require bounded idempotency and return receipt-based status for safe readback.
+- Hosted browser setup and takeover remain disabled in production until MCP Scraper owns an opaque HTTP/WebSocket proxy or the browser provider supplies an equivalent opaque embed contract; the provider-iframe transport remains test-only.
+- Release verification now pins the immutable Personal Assistant artifact commit and proves it is an ancestor of the release-metadata commit, avoiding a self-referential `HEAD` requirement while still rejecting divergent or unavailable source identities.
+- The Personal Assistant is implemented and fixture-tested, and its source, generated contracts, web UI, Memory packages, migration utilities, harness bundle, and operating skills are packaged in this repository. Production activation still requires the release deployment, database migration, provider callbacks, and an approved spare-Mac worker installation.
+- Personal Assistant Mac workers now support an explicit private staging profile with an isolated authenticated Codex home, a self-contained pinned runtime outside macOS protected folders, launchd supervision, live task submission, restart recovery, and a verified prior-artifact rollback target; FileVault, firewall, and backup remain production-promotion gates.
+
+### Fixed
+
+- Mac harness installation now fails closed until the named LaunchAgent owns an initialized app-server heartbeat, and task submission retains final assistant messages delivered through incremental completion events instead of reporting a completed turn as empty.
+
+### Security
+
+- Browser takeover now uses owner-authenticated, one-time claims with view-only leases by default, exact control upgrades, stale/replay rejection, and sanitized persistence that excludes provider profile names, session IDs, CDP endpoints, cookies, local profile paths, and credentials.
+- Added a fail-closed five-level authority model that intersects provider capability, connection state, scoped grants, consent, suppression, registration, quiet hours, immutable approval bindings, spend ceilings, and short-lived leases before any external or destructive action.
+- Added deterministic `STOP`, `START`, and `HELP` handling, append-only consent and suppression evidence, per-recipient bulk-send rechecks, frozen audience digests, abuse ceilings, kill switches, and internal do-not-contact precedence; inbound activity and imported data never imply outbound consent.
+- Twilio webhooks now verify signatures against the exact raw request and canonical public URL before tenant lookup, deduplicate callbacks, keep attachment bytes and provider identifiers private, and preserve uncertain side effects for reconciliation instead of blindly retrying.
+- Twilio inbound webhooks now commit an owner-encrypted SMS/MMS envelope before acknowledgement, require the exact Messaging Service, enforce durable authenticated quotas, serialize only within each sender lane, and route unknown senders to web setup without exposing their message to the assistant.
+- Assistant logs, wake packets, MCP/API results, and Memory references exclude secrets, database access, signed URLs, raw provider identifiers, attachment bytes, and full recipient identifiers; provider and retrieved content remains explicitly untrusted data rather than executable authority.
+
+## [0.78.0] - 2026-08-28
+
+### Changed
+
+- Made `maxQuestions` an explicit target count rather than a traversal-depth control, with separate discovery and material-completeness diagnostics.
+- Preserved complete People Also Ask, AI Overview, and organic-result link provenance in JSON, structured MCP output, and CSV while classifying plain links and Google redirect links explicitly.
+- Resolved opaque Google `/goto` targets through bounded concurrent manual-redirect requests with active-browser interception as a fallback, without following publisher destinations and without dropping unresolved material.
+- Aligned the bounded PAA production-provider canary with the public `maxQuestions` contract and made Bright Data the default test provider.
+
+### Fixed
+
+- Distinguished a healthy three-pass PAA frontier exhaustion from failed, disappeared, or timed-out browser interactions; interrupted jobs now renew fenced leases, resume automatically on the same job ID, and stop after a bounded recovery budget.
+- Checkpointed the initial SERP material before PAA expansion so a worker interruption retains already captured AI Overview citations, organic rankings, entities, forums, videos, and local results.
+- Required an exact visible AI Overview label before accepting an overview container, preventing ordinary sponsored blocks from being misclassified as AI answers.
+- Kept synchronous and durable PAA results consistent for question identifiers, all answer sources, SERP modules, link-resolution status, and completeness diagnostics.
+
 ## [0.77.0] - 2026-08-28
 
 ### Fixed
@@ -1419,14 +1472,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 - Write actions remain unavailable until the account owner explicitly enables them.
 - Provider-specific connection data is normalized into one agent-facing contract.
 
-[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.76.0...HEAD
-[0.76.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.1...v0.76.0
-[0.75.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.0...v0.75.1
-[0.75.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.74.0...v0.75.0
-[0.74.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.8...v0.74.0
-[0.73.8]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.7...v0.73.8
-[0.73.7]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.6...v0.73.7
-[0.73.6]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.5...v0.73.6
+[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.5...HEAD
 [0.73.5]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.4...v0.73.5
 [0.73.4]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.3...v0.73.4
 [0.73.3]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.2...v0.73.3

package/README.md

@@ -23,7 +23,6 @@ Use the MCPB Desktop Extension for the branded Claude Desktop install, or use th
 MCP Scraper ships one stdio entrypoint plus human-facing helper CLIs:
 
 - `mcp-scraper` — the single MCP server with every tool: live web intelligence (SERP, PAA, site extraction, YouTube, Facebook ads and organic video transcripts, Maps, directory, rank tracker blueprint, credits) plus agent-controlled direct/no-proxy hosted browser sessions (screenshots, clicks, typing, scrolling, watch URLs, replay links, MP4 replay download, and saved profile/login setup for authenticated AI visibility workflows). It is context-aware: in a human terminal it prints the branded ASCII install card; in an MCP client it runs as a protocol-clean stdio server. This is the entrypoint used by the MCPB Desktop Extension.
-- `mcp-scraper-core` — a restricted stdio entrypoint for Thorbit that registers exactly 34 direct public-web scraping and analysis tools. It excludes MCP Scraper integrations and connected-account actions, Scheduler, Memory, browser-agent, analytics, publishing, Commons, lead-list, Local Sourcebook, and generic workflow surfaces. The normal `mcp-scraper` entrypoint remains full-surface.
 - `mcp-scraper-install` — explicit alias for the human-facing terminal installer card with the branded ASCII intro and copyable install commands.
 - `mcp-scraper-cli` — a human-facing CLI for setup checks, AI-agent config generation, workflow prompts, SEO workflow runs, and HTML reports. This command is safe to print because it is not an MCP stdio server.
 
@@ -174,7 +173,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.77.0`, SHA-256 `d59c2c55eeff7889a78d0fcddfb93d8062003e2ebd18f8dc8a03a73d262fc25a`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.79.0`, SHA-256 `c4728bfebff747f752f5f8b0e4188d45c46f794e2756238b71f951661ed048b2`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 
@@ -336,7 +335,7 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
 
 All MCP tools return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step, plus readable text content for compatibility. Runtime `tools/list` omits output schemas so strict clients can register the complete catalog; the generated developer manifest retains every canonical output schema for validation and typed SDK generation. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
 
-The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 339 tools: 238 scraper, browser, workflow, billing, and connected-service tools plus 101 durable-memory tools. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray analytics, attribution, CRM policy and receipt, campaign, export, and scheduled-report tools. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
+The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 354 tools: 253 scraper, browser, workflow, billing, connected-service, and personal-assistant tools plus 101 durable-memory tools. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; governed personal-assistant commands, messaging, approvals, grants, number setup, and execution readback; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray analytics, attribution, CRM policy and receipt, campaign, export, and scheduled-report tools. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
 
 For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone memory service, not the public stdio execution path. Direct `mcp-memory` OAuth and stdio clients continue to use `memory.mcpscraper.dev` and must be verified as a separate dependent release surface.
 

package/dist/{analytics-repository-AUTJK4HK.js → analytics-repository-VEUDEPWI.js}

@@ -85,9 +85,9 @@ import {
   updateAnalyticsPixel,
   upsertAnalyticsAdSpend,
   upsertAnalyticsHostGroup
-} from "./chunk-IBQNOZ7E.js";
-import "./chunk-PJEEKOUM.js";
-import "./chunk-WEFPBAAG.js";
+} from "./chunk-DUWZQSSJ.js";
+import "./chunk-Q5L2OCOQ.js";
+import "./chunk-OU4FV2RP.js";
 export {
   ANALYTICS_CONTENT_SORTS,
   AnalyticsRepositoryError,

package/dist/bin/api-server.js

@@ -15,17 +15,18 @@ function loadDotEnv() {
 }
 loadDotEnv();
 async function main() {
-  const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
+  const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-EKRNKKLU.js"),
-    import("../worker-DR6R7J46.js"),
-    import("../db-566MOHHD.js")
+    import("../server-QJUBOH6L.js"),
+    import("../worker-VU6JCCLG.js"),
+    import("../db-TA567JLA.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");
   try {
+    await personalAssistantProductionStartup;
     await migrate();
     if (process.env.ANALYTICS_DATABASE_URL) {
-      const { migrateAnalytics } = await import("../analytics-repository-AUTJK4HK.js");
+      const { migrateAnalytics } = await import("../analytics-repository-VEUDEPWI.js");
       await migrateAnalytics();
     }
     startWorker();
@@ -34,7 +35,7 @@ async function main() {
       console.log(`[server] admin auth: ${process.env.ADMIN_KEY ? "configured" : "not configured"}`);
     });
   } catch (err) {
-    console.error("[startup] migration failed", err);
+    console.error("[startup] server preflight failed", err instanceof Error ? err.name : "unknown_error");
     process.exit(1);
   }
 }

package/dist/bin/mcp-scraper-cli.js

@@ -6,14 +6,14 @@ import {
   openWorkflowReport,
   runWorkflow,
   workflowOutputBaseDir
-} from "../chunk-KBTUPRTF.js";
+} from "../chunk-VEQPA65D.js";
 import "../chunk-RJ7JVYKU.js";
-import "../chunk-77LQV73B.js";
+import "../chunk-OWF2JJKN.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-7HF3T7P2.js";
+} from "../chunk-ATIVTEML.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-core.js

@@ -1,20 +1,20 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-IP7UDXMU.js";
-import "../chunk-DTJ56S5B.js";
+} from "../chunk-Q54QVYJ2.js";
+import "../chunk-2JF4ECSJ.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
-import "../chunk-7RQULQF3.js";
-import "../chunk-6W4ADSWE.js";
+import "../chunk-B6L73AO4.js";
+import "../chunk-VFCUGTCF.js";
 import "../chunk-OM7HVEJ3.js";
-import "../chunk-CXY5WV45.js";
-import "../chunk-GT6MP4SU.js";
+import "../chunk-NGLZ6UJR.js";
+import "../chunk-R4YRH5PG.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-7HF3T7P2.js";
-import "../chunk-WEFPBAAG.js";
+import "../chunk-ATIVTEML.js";
+import "../chunk-OU4FV2RP.js";
 
 // src/mcp/thorbit-restricted-tool-surface.ts
 var THORBIT_RESTRICTED_TOOL_NAMES = [

package/dist/bin/mcp-scraper-install.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   renderInstallTerminal
-} from "../chunk-GT6MP4SU.js";
+} from "../chunk-R4YRH5PG.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-7HF3T7P2.js";
+} from "../chunk-ATIVTEML.js";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-stdio-server.js

@@ -1,20 +1,20 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-IP7UDXMU.js";
-import "../chunk-DTJ56S5B.js";
+} from "../chunk-Q54QVYJ2.js";
+import "../chunk-2JF4ECSJ.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
-import "../chunk-7RQULQF3.js";
-import "../chunk-6W4ADSWE.js";
+import "../chunk-B6L73AO4.js";
+import "../chunk-VFCUGTCF.js";
 import "../chunk-OM7HVEJ3.js";
-import "../chunk-CXY5WV45.js";
-import "../chunk-GT6MP4SU.js";
+import "../chunk-NGLZ6UJR.js";
+import "../chunk-R4YRH5PG.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-7HF3T7P2.js";
-import "../chunk-WEFPBAAG.js";
+import "../chunk-ATIVTEML.js";
+import "../chunk-OU4FV2RP.js";
 
 // bin/mcp-stdio-server.ts
 runMcpScraperStdio();

package/dist/bin/paa-harvest.js

@@ -1,13 +1,13 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-F6MUGMRN.js";
+} from "../chunk-UFJJGXVO.js";
 import {
   browserServiceApiKey
 } from "../chunk-OM7HVEJ3.js";
-import "../chunk-CXY5WV45.js";
-import "../chunk-77LQV73B.js";
-import "../chunk-WEFPBAAG.js";
+import "../chunk-NGLZ6UJR.js";
+import "../chunk-OWF2JJKN.js";
+import "../chunk-OU4FV2RP.js";
 
 // src/cli.ts
 import { Command } from "commander";