mcp-scraper 0.86.5 → 0.88.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -1
- package/README.md +17 -4
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-core.js +5 -5
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-QJJQ4OAZ.js → chunk-2JVZQGSN.js} +50 -8
- package/dist/{chunk-NKRO2SUO.js → chunk-4QMUF6XM.js} +2 -2
- package/dist/{chunk-3JRZZVWL.js → chunk-ABF2CGOZ.js} +2 -2
- package/dist/{chunk-OWF2JJKN.js → chunk-GXBZXWXB.js} +8 -8
- package/dist/{chunk-KQWVJOVR.js → chunk-IHXAXYIS.js} +1 -1
- package/dist/{chunk-WFGAR4BZ.js → chunk-MZDNZQWT.js} +1 -1
- package/dist/{chunk-TV32LC76.js → chunk-P5FPC54B.js} +3 -3
- package/dist/{chunk-NH4QAH3X.js → chunk-S24LFPL7.js} +178 -53
- package/dist/{chunk-6IOGTFNS.js → chunk-UZPTGUDV.js} +1 -1
- package/dist/{chunk-BA6BPDSP.js → chunk-XIJI34RM.js} +1 -1
- package/dist/{extract-bundle-FALNLKYY.js → extract-bundle-565SBZCR.js} +2 -2
- package/dist/{gmail-service-BUSU4BVX.js → gmail-service-E6ALS7JG.js} +2 -2
- package/dist/index.cjs +185 -60
- package/dist/index.d.cts +11 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +2 -2
- package/dist/{server-ARMF2IRI.js → server-PD3TDL6Q.js} +526 -428
- package/dist/{site-extract-repository-S5FYNFDY.js → site-extract-repository-VYFZASPU.js} +2 -2
- package/dist/{worker-OQSFSLNG.js → worker-LDCAULWL.js} +3 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,25 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.88.0] - 2026-09-02
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- Made one `MCP_SCRAPER_API_KEY` authenticate scraping, Memory, Research, and native CRM through the canonical root MCP endpoint; root-to-Memory calls now use short-lived audience-bound owner assertions.
|
|
12
|
+
- Reduced the customer-callable unified catalog from 377 to 375 tools by retiring Memory key issuance and scope mutation, while preserving explicit compatibility errors in SDK clients.
|
|
13
|
+
- Added strict-schema validation for all 148 direct Memory tools and an authenticated 30-screenshot CRM workspace proof runner covering ten surfaces at wide, compact, and mobile viewports.
|
|
14
|
+
|
|
15
|
+
### Security
|
|
16
|
+
|
|
17
|
+
- Stopped normal creation, storage, display, and use of long-lived per-user Memory keys. Existing direct `mk_` credentials remain accepted for one documented compatibility release and can still be listed or revoked.
|
|
18
|
+
|
|
19
|
+
## [0.87.0] - 2026-09-02
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Added optional `pages: 2` support to synchronous and durable PAA harvests, capturing a second organic result page before expanding questions on the untouched first page.
|
|
24
|
+
- Added requested/captured page counts and bounded second-page outcomes to live results, durable recovery, the dashboard, and generated client contracts. A failed optional page-two capture preserves first-page evidence.
|
|
25
|
+
|
|
7
26
|
## [0.86.5] - 2026-09-02
|
|
8
27
|
|
|
9
28
|
### Fixed
|
|
@@ -1705,7 +1724,9 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1705
1724
|
- Write actions remain unavailable until the account owner explicitly enables them.
|
|
1706
1725
|
- Provider-specific connection data is normalized into one agent-facing contract.
|
|
1707
1726
|
|
|
1708
|
-
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.
|
|
1727
|
+
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.88.0...HEAD
|
|
1728
|
+
[0.88.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.87.0...v0.88.0
|
|
1729
|
+
[0.87.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.86.5...v0.87.0
|
|
1709
1730
|
[0.86.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.85.0...v0.86.0
|
|
1710
1731
|
[0.84.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.5...v0.84.0
|
|
1711
1732
|
[0.83.5]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.83.4...v0.83.5
|
package/README.md
CHANGED
|
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
175
175
|
npm run build:mcpb
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.88.0`, SHA-256 `ff593dd4184122b9d9ef10bb0a9c21a907f8e2e180520dfbe56001bff0ee090d`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
179
179
|
|
|
180
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
181
181
|
|
|
@@ -235,9 +235,22 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
|
|
|
235
235
|
|
|
236
236
|
## Tools
|
|
237
237
|
|
|
238
|
+
### Two-page PAA harvests
|
|
239
|
+
|
|
240
|
+
Pass this input to `harvest_paa` (or add an `idempotencyKey` for `harvest_paa_start`):
|
|
241
|
+
|
|
242
|
+
```json
|
|
243
|
+
{"query":"commercial truck insurance","pages":2,"maxQuestions":10}
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
The harvester attempts page 2 organic results before expanding PAA on the untouched first page. It does not harvest a second PAA graph. `pages` accepts 1 or 2 and defaults to 1; pricing is unchanged. If page 2 is missing, empty, or cannot load, available first-page evidence is retained.
|
|
247
|
+
|
|
248
|
+
Check `pagination.requestedPages`, `pagination.capturedPages`, and `pagination.page2Status` in the MCP result before assuming both pages were captured. Raw REST results place the same fields under `diagnostics.pagination`. Legacy saved results may have no pagination metadata, which MCP formatters expose as `null`.
|
|
249
|
+
|
|
238
250
|
### Web-intelligence tools
|
|
239
251
|
|
|
240
|
-
- `harvest_paa`
|
|
252
|
+
- `harvest_paa` — expand People Also Ask on the original first result page. Optional `pages: 2` captures a second organic result page first; the default is one page.
|
|
253
|
+
- `harvest_paa_start` — start the same harvest as a durable job, with the same optional `pages: 2`.
|
|
241
254
|
- `search_serp`
|
|
242
255
|
- `extract_url` — extract normal or Wayback-replayed page copy; Wayback results omit playback chrome and can include a timestamp-matched featured image. Set `preserveMedia:true` to union static and rendered/lazy media, collapse responsive variants, attach up to `maxInlineImages` AI-readable images, and receive an owner-scoped ZIP manifest readable with `archive_read`. Branding output ranks the site logo separately from evidence-bounded proof images such as certifications, awards, memberships, partner/customer marks, and press mentions.
|
|
243
256
|
- `map_site_urls`
|
|
@@ -337,9 +350,9 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
|
|
|
337
350
|
|
|
338
351
|
All MCP tools return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step, plus readable text content for compatibility. Runtime `tools/list` omits output schemas so strict clients can register the complete catalog; the generated developer manifest retains every canonical output schema for validation and typed SDK generation. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
|
|
339
352
|
|
|
340
|
-
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes
|
|
353
|
+
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. The unified server exposes 375 tools: 249 scraper, browser, workflow, billing, connected-service, and personal-assistant tools plus 126 durable-memory tools. The two retired customer credential mutations are no longer advertised. The scraper-side inventory includes complete Gmail selection, message, attachment, export, bulk-action, and Memory-import workflows; governed personal-assistant commands, messaging, approvals, grants, number setup, and execution readback; durable PAA starts and status; rendered site-content similarity; governed Local Sourcebook and Transparent Commons workflows; direct site-export reads; Editorial Reading Room and News Publisher templates; and production X-Ray setup, analytics, seven-model attribution, structured post-purchase surveys, reported impact, truthful view-evidence status, CRM policy and receipt, campaign, export, and scheduled-report tools. Provider setup remains absent until its authorization, ingestion, reconciliation, privacy, canary, cleanup, and deployment receipts are complete. Successful evidence-compiled Local Sourcebook revisions publish automatically to their canonical `localsourcebook.com` category profile and review URLs; administrator controls handle exceptional rejection or unpublishing. Release verification compares the exact local and hosted tool-name sets, not only the count.
|
|
341
354
|
|
|
342
|
-
For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone
|
|
355
|
+
For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone Memory service, not a second customer setup path. Existing direct Memory credentials remain compatible for one release, but all new customer setup uses the root endpoint and `MCP_SCRAPER_API_KEY`.
|
|
343
356
|
|
|
344
357
|
## Resources
|
|
345
358
|
|
package/dist/bin/api-server.js
CHANGED
|
@@ -17,8 +17,8 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
20
|
+
import("../server-PD3TDL6Q.js"),
|
|
21
|
+
import("../worker-LDCAULWL.js"),
|
|
22
22
|
import("../db-Z34LPZNR.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
@@ -6,9 +6,9 @@ import {
|
|
|
6
6
|
openWorkflowReport,
|
|
7
7
|
runWorkflow,
|
|
8
8
|
workflowOutputBaseDir
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-UZPTGUDV.js";
|
|
10
10
|
import "../chunk-RJ7JVYKU.js";
|
|
11
|
-
import "../chunk-
|
|
11
|
+
import "../chunk-GXBZXWXB.js";
|
|
12
12
|
import {
|
|
13
13
|
XRAY_INSTALL_CONTRACT,
|
|
14
14
|
XRAY_INSTALL_PROTOCOL_DIGEST,
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
import "../chunk-GGZEC22A.js";
|
|
20
20
|
import {
|
|
21
21
|
PACKAGE_VERSION
|
|
22
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-XIJI34RM.js";
|
|
23
23
|
|
|
24
24
|
// src/cli/human-cli.ts
|
|
25
25
|
import { Command } from "commander";
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-P5FPC54B.js";
|
|
5
|
+
import "../chunk-2JVZQGSN.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
|
-
import "../chunk-
|
|
10
|
+
import "../chunk-4QMUF6XM.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
12
|
import "../chunk-OPQIGAFB.js";
|
|
13
|
-
import "../chunk-
|
|
13
|
+
import "../chunk-ABF2CGOZ.js";
|
|
14
14
|
import "../chunk-P7FWOMU7.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-XIJI34RM.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// src/mcp/thorbit-restricted-tool-surface.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-ABF2CGOZ.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-XIJI34RM.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-P5FPC54B.js";
|
|
5
|
+
import "../chunk-2JVZQGSN.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
|
-
import "../chunk-
|
|
10
|
+
import "../chunk-4QMUF6XM.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
12
|
import "../chunk-OPQIGAFB.js";
|
|
13
|
-
import "../chunk-
|
|
13
|
+
import "../chunk-ABF2CGOZ.js";
|
|
14
14
|
import "../chunk-P7FWOMU7.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-XIJI34RM.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// bin/mcp-stdio-server.ts
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
harvest
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-S24LFPL7.js";
|
|
5
5
|
import {
|
|
6
6
|
browserServiceApiKey
|
|
7
7
|
} from "../chunk-OM7HVEJ3.js";
|
|
8
8
|
import "../chunk-OPQIGAFB.js";
|
|
9
|
-
import "../chunk-
|
|
9
|
+
import "../chunk-GXBZXWXB.js";
|
|
10
10
|
import "../chunk-YXNDOQXN.js";
|
|
11
11
|
|
|
12
12
|
// src/cli.ts
|
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
PAA_QUESTION_CREDITS,
|
|
27
27
|
PAGE_SCRAPE_CREDITS,
|
|
28
28
|
SERP_SEARCH_CREDITS
|
|
29
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-4QMUF6XM.js";
|
|
30
30
|
import {
|
|
31
31
|
browserServiceProfileName,
|
|
32
32
|
browserServiceProfileSaveChanges
|
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
} from "./chunk-P7FWOMU7.js";
|
|
51
51
|
import {
|
|
52
52
|
PACKAGE_VERSION
|
|
53
|
-
} from "./chunk-
|
|
53
|
+
} from "./chunk-XIJI34RM.js";
|
|
54
54
|
import {
|
|
55
55
|
PUBLIC_ERROR_CODES,
|
|
56
56
|
buildPublicErrorEnvelope,
|
|
@@ -1126,6 +1126,28 @@ function errorAttemptsSection(body) {
|
|
|
1126
1126
|
Attempts:
|
|
1127
1127
|
${lines.join("\n")}`;
|
|
1128
1128
|
}
|
|
1129
|
+
function publicHarvestPagination(value) {
|
|
1130
|
+
const row = structuredRecord(value);
|
|
1131
|
+
const statuses = ["not_requested", "not_attempted", "captured", "unavailable", "failed"];
|
|
1132
|
+
const codes = ["missing_next", "invalid_next", "empty_page", "captcha", "timeout", "navigation_error", "unsupported_driver"];
|
|
1133
|
+
if (row.requestedPages !== 1 && row.requestedPages !== 2 || row.capturedPages !== 1 && row.capturedPages !== 2 || !statuses.includes(row.page2Status) || !Number.isInteger(row.page1OrganicCount) || Number(row.page1OrganicCount) < 0 || !Number.isInteger(row.page2OrganicCount) || Number(row.page2OrganicCount) < 0) return null;
|
|
1134
|
+
const captured = row.page2Status === "captured";
|
|
1135
|
+
if (captured !== (row.capturedPages === 2) || captured && (row.requestedPages !== 2 || Number(row.page2OrganicCount) === 0) || !captured && row.page2OrganicCount !== 0 || row.requestedPages === 1 && row.page2Status !== "not_requested" || row.requestedPages === 2 && row.page2Status === "not_requested" || row.page2Status === "not_attempted" && row.failureCode !== void 0) return null;
|
|
1136
|
+
return {
|
|
1137
|
+
requestedPages: row.requestedPages,
|
|
1138
|
+
capturedPages: row.capturedPages,
|
|
1139
|
+
page2Status: row.page2Status,
|
|
1140
|
+
page1OrganicCount: Number(row.page1OrganicCount),
|
|
1141
|
+
page2OrganicCount: Number(row.page2OrganicCount),
|
|
1142
|
+
...codes.includes(row.failureCode) ? { failureCode: row.failureCode } : {}
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
function harvestPaginationText(pagination) {
|
|
1146
|
+
if (!pagination) return "";
|
|
1147
|
+
const outcome = pagination.page2Status === "captured" ? "Page 2 organic results captured; PAA harvested on the original first page." : pagination.requestedPages === 2 ? `Page 2 ${pagination.page2Status.replaceAll("_", " ")}${pagination.failureCode ? ` (${pagination.failureCode})` : ""}; first-page evidence retained.` : "PAA harvested on the first page.";
|
|
1148
|
+
return `
|
|
1149
|
+
**Pagination:** ${pagination.capturedPages} of ${pagination.requestedPages} requested organic pages captured. ${outcome}`;
|
|
1150
|
+
}
|
|
1129
1151
|
function publicPaaLifecycle(value) {
|
|
1130
1152
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
1131
1153
|
const row = value;
|
|
@@ -1285,6 +1307,7 @@ function formatHarvestPaa(raw, input) {
|
|
|
1285
1307
|
const aiOvw = d.aiOverview;
|
|
1286
1308
|
const diagnostics = d.diagnostics;
|
|
1287
1309
|
const durationMs = d.stats?.durationMs;
|
|
1310
|
+
const pagination = publicHarvestPagination(diagnostics?.pagination);
|
|
1288
1311
|
const paaRows = flat.map(
|
|
1289
1312
|
(r, i) => `| ${i + 1} | ${cell(r.question)} | ${cell(truncate(r.answer, 120))} | ${cell(r.source_title || r.source_site || "")} |`
|
|
1290
1313
|
).join("\n");
|
|
@@ -1317,7 +1340,7 @@ ${serpRows}` : "";
|
|
|
1317
1340
|
- Dig into a result: use \`extract_url\` on any organic URL`;
|
|
1318
1341
|
const full = `# PAA Report: "${input.query}"${input.location ? ` \xB7 ${input.location}` : ""}
|
|
1319
1342
|
|
|
1320
|
-
${paaTable}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${debugSection(diagnostics?.debug)}${tips}`;
|
|
1343
|
+
${paaTable}${harvestPaginationText(pagination)}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${debugSection(diagnostics?.debug)}${tips}`;
|
|
1321
1344
|
return {
|
|
1322
1345
|
...oneBlock(full),
|
|
1323
1346
|
structuredContent: {
|
|
@@ -1330,6 +1353,7 @@ ${paaTable}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${d
|
|
|
1330
1353
|
degradationReasons: diagnostics?.degradationReasons ?? [],
|
|
1331
1354
|
retryRecommended: diagnostics?.retryRecommended ?? null,
|
|
1332
1355
|
paaLifecycle: publicPaaLifecycle(diagnostics?.paaLifecycle),
|
|
1356
|
+
pagination,
|
|
1333
1357
|
questions: flat.map((r) => publicPaaQuestion(r)),
|
|
1334
1358
|
organicResults: organic.map((r) => publicOrganicResult(r)),
|
|
1335
1359
|
aiOverview: publicAiOverview(aiOvw),
|
|
@@ -1390,6 +1414,7 @@ function durablePaaProgress(result, options) {
|
|
|
1390
1414
|
);
|
|
1391
1415
|
}
|
|
1392
1416
|
return {
|
|
1417
|
+
pagination: publicHarvestPagination(diagnostics.pagination ?? structuredRecord(rawProgress.material).pagination),
|
|
1393
1418
|
requestedQuestions: finiteNonNegative(completeness.requestedQuestions ?? options.maxQuestions),
|
|
1394
1419
|
capturedQuestions: captured,
|
|
1395
1420
|
answeredQuestions: answered,
|
|
@@ -1410,6 +1435,7 @@ function durablePaaResult(result) {
|
|
|
1410
1435
|
resultQuality: nullableBoundedString(diagnostics.resultQuality ?? result.resultQuality),
|
|
1411
1436
|
retryRecommended: typeof diagnostics.retryRecommended === "boolean" ? diagnostics.retryRecommended : typeof result.retryRecommended === "boolean" ? result.retryRecommended : null,
|
|
1412
1437
|
paaLifecycle: publicPaaLifecycle(diagnostics.paaLifecycle ?? structuredRecord(result.progress).lifecycle),
|
|
1438
|
+
pagination: publicHarvestPagination(diagnostics.pagination ?? structuredRecord(structuredRecord(result.progress).material).pagination),
|
|
1413
1439
|
questionCount: rows.length,
|
|
1414
1440
|
questions: rows.map(publicPaaQuestion),
|
|
1415
1441
|
organicResults: Array.isArray(result.organicResults) ? result.organicResults.filter((row) => row && typeof row === "object").map((row) => publicOrganicResult(row)) : [],
|
|
@@ -1521,7 +1547,7 @@ function formatHarvestPaaStatus(raw) {
|
|
|
1521
1547
|
return {
|
|
1522
1548
|
content: [{
|
|
1523
1549
|
type: "text",
|
|
1524
|
-
text: recovering ? `PAA job ${jobId}: recovering automatically after an interrupted worker. ${progress.capturedQuestions ?? 0} questions are preserved; keep polling this same job.` : `PAA job ${jobId}: ${state}. ${progress.capturedQuestions ?? 0} questions are preserved.`
|
|
1550
|
+
text: (recovering ? `PAA job ${jobId}: recovering automatically after an interrupted worker. ${progress.capturedQuestions ?? 0} questions are preserved; keep polling this same job.` : `PAA job ${jobId}: ${state}. ${progress.capturedQuestions ?? 0} questions are preserved.`) + harvestPaginationText(publicHarvestPagination(structuredRecord(result.diagnostics).pagination ?? structuredRecord(structuredRecord(result.progress).material).pagination))
|
|
1525
1551
|
}],
|
|
1526
1552
|
structuredContent
|
|
1527
1553
|
};
|
|
@@ -8725,6 +8751,7 @@ var WebsiteUrlOrDomainSchema = z7.string().trim().min(1).transform((raw, ctx) =>
|
|
|
8725
8751
|
}
|
|
8726
8752
|
});
|
|
8727
8753
|
var HarvestPaaInputSchema = {
|
|
8754
|
+
pages: z7.number().int().min(1).max(2).default(1).describe("Organic result pages to capture. Default 1, maximum 2. Page 2 is captured when available before harvesting PAA on the original first page; it does not add a second PAA graph. Pagination output reports the pages actually captured."),
|
|
8728
8755
|
query: z7.string().min(1).describe('The search topic, exactly as it should be searched, e.g. "best hvac company in Denver". Include the place here when you want it in the search terms \u2014 the server sends your query to Google unchanged and never adds or removes a location.'),
|
|
8729
8756
|
location: z7.string().optional().describe('Where Google should think the searcher is, e.g. "Denver, CO". Sets the Google UULE parameter only \u2014 it never changes your query text and never selects a proxy. To put the place in the search terms too, write it into query.'),
|
|
8730
8757
|
maxQuestions: z7.number().int().min(1).max(200).default(30).describe("PAA questions to extract. Default 30, maximum 200. Use 10 for quick probes, 100-200 for deep research. Billed per extracted question; unused hold refunded."),
|
|
@@ -10019,6 +10046,14 @@ var PaaInteractionOutput = z7.object({
|
|
|
10019
10046
|
sourceCount: z7.number().int().min(0),
|
|
10020
10047
|
errorCode: z7.enum(["click_failed", "click_ack_timeout", "control_missing", "confirmation_timeout"]).nullable()
|
|
10021
10048
|
});
|
|
10049
|
+
var HarvestPaginationOutput = z7.object({
|
|
10050
|
+
requestedPages: z7.union([z7.literal(1), z7.literal(2)]),
|
|
10051
|
+
capturedPages: z7.union([z7.literal(1), z7.literal(2)]),
|
|
10052
|
+
page2Status: z7.enum(["not_requested", "not_attempted", "captured", "unavailable", "failed"]),
|
|
10053
|
+
page1OrganicCount: z7.number().int().min(0),
|
|
10054
|
+
page2OrganicCount: z7.number().int().min(0),
|
|
10055
|
+
failureCode: z7.enum(["missing_next", "invalid_next", "empty_page", "captcha", "timeout", "navigation_error", "unsupported_driver"]).optional()
|
|
10056
|
+
}).nullable();
|
|
10022
10057
|
var HarvestPaaOutputSchema = {
|
|
10023
10058
|
query: z7.string(),
|
|
10024
10059
|
location: NullableString,
|
|
@@ -10029,6 +10064,7 @@ var HarvestPaaOutputSchema = {
|
|
|
10029
10064
|
degradationReasons: z7.array(z7.string()),
|
|
10030
10065
|
retryRecommended: z7.boolean().nullable(),
|
|
10031
10066
|
paaLifecycle: PaaLifecycleOutput,
|
|
10067
|
+
pagination: HarvestPaginationOutput,
|
|
10032
10068
|
questions: z7.array(PaaQuestionOutput),
|
|
10033
10069
|
organicResults: z7.array(OrganicResultOutput),
|
|
10034
10070
|
aiOverview: AiOverviewOutput,
|
|
@@ -10036,6 +10072,7 @@ var HarvestPaaOutputSchema = {
|
|
|
10036
10072
|
durationMs: z7.number().min(0).nullable()
|
|
10037
10073
|
};
|
|
10038
10074
|
var HarvestPaaDurableProgressOutput = z7.object({
|
|
10075
|
+
pagination: HarvestPaginationOutput,
|
|
10039
10076
|
requestedQuestions: z7.number().int().min(0).nullable(),
|
|
10040
10077
|
capturedQuestions: z7.number().int().min(0),
|
|
10041
10078
|
answeredQuestions: z7.number().int().min(0),
|
|
@@ -10077,6 +10114,7 @@ var HarvestPaaDurableBillingOutput = z7.object({
|
|
|
10077
10114
|
refundMc: z7.number().int().min(0).nullable()
|
|
10078
10115
|
});
|
|
10079
10116
|
var HarvestPaaDurableResultOutput = z7.object({
|
|
10117
|
+
pagination: HarvestPaginationOutput,
|
|
10080
10118
|
completionStatus: NullableString,
|
|
10081
10119
|
resultQuality: NullableString,
|
|
10082
10120
|
retryRecommended: z7.boolean().nullable(),
|
|
@@ -17286,7 +17324,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17286
17324
|
registerPersonalAssistantMcpSurface(server, executor);
|
|
17287
17325
|
server.registerTool("harvest_paa", {
|
|
17288
17326
|
title: "Google PAA + SERP Harvest",
|
|
17289
|
-
description: `Expand one Google People Also Ask SERP into questions, answers, every preserved source, AI Overview evidence, ranking URLs, and entity IDs. maxQuestions is a target count, not traversal depth. Results distinguish target_reached, proven frontier_exhausted, interruption, and recovery_exhausted; a failed click or browser timeout is never reported as exhaustion. This compatibility tool waits; use harvest_paa_start plus harvest_paa_status for long runs. Optional SERP modules require their include flags. Use gl and location for regional context. Costs ${PAA_BASE_CREDITS} Credits per harvest plus ${PAA_QUESTION_CREDITS} Credits per question actually returned; unused hold is refunded. After a timeout or unknown response, reuse the same idempotencyKey. Call credits_info for current pricing and balance.`,
|
|
17327
|
+
description: `Expand one Google People Also Ask SERP into questions, answers, every preserved source, AI Overview evidence, ranking URLs, and entity IDs. Set pages to 2 to add the second organic-results page when available; PAA is still expanded once on the preserved first page, and pagination reports what was captured. maxQuestions is a target count, not traversal depth. Results distinguish target_reached, proven frontier_exhausted, interruption, and recovery_exhausted; a failed click or browser timeout is never reported as exhaustion. This compatibility tool waits; use harvest_paa_start plus harvest_paa_status for long runs. Optional SERP modules require their include flags. Use gl and location for regional context. Costs ${PAA_BASE_CREDITS} Credits per harvest plus ${PAA_QUESTION_CREDITS} Credits per question actually returned; unused hold is refunded. After a timeout or unknown response, reuse the same idempotencyKey. Call credits_info for current pricing and balance.`,
|
|
17290
17328
|
inputSchema: harvestPaaInputSchema,
|
|
17291
17329
|
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
17292
17330
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
@@ -17299,7 +17337,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17299
17337
|
});
|
|
17300
17338
|
server.registerTool("harvest_paa_start", {
|
|
17301
17339
|
title: "Start Durable Google PAA Harvest",
|
|
17302
|
-
description: `Start a durable Google People Also Ask harvest and return its job receipt. maxQuestions is the requested target count. The job automatically resumes an interrupted serverless worker under the same jobId, idempotency key, and billing hold while preserving checkpoints. Keep one idempotencyKey after a timeout, unknown response, or in-progress reply; replaying it recovers the existing job without a duplicate charge. Poll jobId with harvest_paa_status. Costs ${PAA_BASE_CREDITS} Credits plus ${PAA_QUESTION_CREDITS} per retained question; unused hold is refunded.`,
|
|
17340
|
+
description: `Start a durable Google People Also Ask harvest and return its job receipt. Set pages to 2 to add the second organic-results page when available; PAA is still expanded once on the preserved first page, and pagination survives checkpoint recovery. maxQuestions is the requested target count. The job automatically resumes an interrupted serverless worker under the same jobId, idempotency key, and billing hold while preserving checkpoints. Keep one idempotencyKey after a timeout, unknown response, or in-progress reply; replaying it recovers the existing job without a duplicate charge. Poll jobId with harvest_paa_status. Costs ${PAA_BASE_CREDITS} Credits plus ${PAA_QUESTION_CREDITS} per retained question; unused hold is refunded.`,
|
|
17303
17341
|
inputSchema: harvestPaaStartInputSchema,
|
|
17304
17342
|
outputSchema: recordOutputSchema("harvest_paa_start", HarvestPaaStartOutputSchema),
|
|
17305
17343
|
annotations: { ...liveWebToolAnnotations("Start Durable Google PAA Harvest"), readOnlyHint: false, idempotentHint: true }
|
|
@@ -17309,7 +17347,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17309
17347
|
});
|
|
17310
17348
|
server.registerTool("harvest_paa_status", {
|
|
17311
17349
|
title: "Check Durable Google PAA Harvest",
|
|
17312
|
-
description: "Poll an owner-scoped harvest_paa_start job. Returns the job/operation/task correlation ID, saved progress, automatic-recovery count, target and discovery status, material completeness, recent per-control interaction outcomes with 0.7/1.0/1.4-second confirmation telemetry, provider-session-correlated attempts, terminal rows, and billing. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
|
|
17350
|
+
description: "Poll an owner-scoped harvest_paa_start job. Returns the job/operation/task correlation ID, saved progress, automatic-recovery count, target and discovery status, material completeness, bounded organic-page pagination, recent per-control interaction outcomes with 0.7/1.0/1.4-second confirmation telemetry, provider-session-correlated attempts, terminal rows, and billing. Page-two outcomes describe organic results only; the PAA graph always comes from the preserved first page. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
|
|
17313
17351
|
inputSchema: HarvestPaaStatusInputSchema,
|
|
17314
17352
|
outputSchema: recordOutputSchema("harvest_paa_status", HarvestPaaStatusOutputSchema),
|
|
17315
17353
|
annotations: { ...liveWebToolAnnotations("Check Durable Google PAA Harvest"), idempotentHint: true, openWorldHint: false }
|
|
@@ -32266,6 +32304,10 @@ var MEMORY_TOOL_SCHEMAS = [
|
|
|
32266
32304
|
var MEMORY_TOOL_DESCRIPTION_OVERRIDES = {
|
|
32267
32305
|
"access-swap-vault": "Set the account's persisted active vault so later memory calls may omit vault and use it by default across stateless requests. The vault must be entitled to the key and the key must hold 'swap' scope."
|
|
32268
32306
|
};
|
|
32307
|
+
var LEGACY_DIRECT_CREDENTIAL_TOOL_IDS = /* @__PURE__ */ new Set([
|
|
32308
|
+
"access-issue-key",
|
|
32309
|
+
"access-set-scope"
|
|
32310
|
+
]);
|
|
32269
32311
|
var MEMORY_TOOL_REGISTRY = [
|
|
32270
32312
|
...MEMORY_TOOL_SCHEMAS.map((schema) => ({
|
|
32271
32313
|
...schema,
|
|
@@ -32273,7 +32315,7 @@ var MEMORY_TOOL_REGISTRY = [
|
|
|
32273
32315
|
})),
|
|
32274
32316
|
...MEMORY_GOVERNED_TOOL_SCHEMAS,
|
|
32275
32317
|
...MEMORY_IMAGE_TOOL_SCHEMAS
|
|
32276
|
-
];
|
|
32318
|
+
].filter((schema) => !LEGACY_DIRECT_CREDENTIAL_TOOL_IDS.has(schema.id));
|
|
32277
32319
|
|
|
32278
32320
|
// src/mcp/memory-mcp-server.ts
|
|
32279
32321
|
function registerMemoryMcpTools(server, executor) {
|
|
@@ -213,9 +213,9 @@ var PUBLIC_PRODUCT_CONTRACT = {
|
|
|
213
213
|
"freeSignupCredits": 0
|
|
214
214
|
},
|
|
215
215
|
"inventory": {
|
|
216
|
-
"totalTools":
|
|
216
|
+
"totalTools": 375,
|
|
217
217
|
"scraperTools": 249,
|
|
218
|
-
"memoryTools":
|
|
218
|
+
"memoryTools": 126
|
|
219
219
|
},
|
|
220
220
|
"concurrencyPack": {
|
|
221
221
|
"ratePolicyVersion": "2026-08-14.concurrency-pack-v1",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// release-message.json
|
|
2
2
|
var release_message_default = {
|
|
3
|
-
message: "
|
|
3
|
+
message: "PAA harvests can now request two organic result pages before expanding questions on the original first page. Use pages: 2 in MCP, SDKs, or the dashboard and check the reported capture status; an unavailable second page preserves first-page evidence."
|
|
4
4
|
};
|
|
5
5
|
|
|
6
6
|
// src/install-terminal.ts
|
|
@@ -55,7 +55,7 @@ function renderInstallTerminal(options) {
|
|
|
55
55
|
"1/1 install surfaces ready",
|
|
56
56
|
colorize(`Newest in v${options.version}: ${release_message_default.message}`, "lime", color),
|
|
57
57
|
"",
|
|
58
|
-
`${colorize("Tools", "cyan", color)} ${colorize("(
|
|
58
|
+
`${colorize("Tools", "cyan", color)} ${colorize("(375 MCP tools)", "muted", color)}`,
|
|
59
59
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
60
60
|
toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
|
|
61
61
|
toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
|
|
@@ -150,18 +150,18 @@ var RawMapsAboutAttributeSchema = z.object({
|
|
|
150
150
|
|
|
151
151
|
// src/output/memory-library-sink.ts
|
|
152
152
|
async function postToMemoryLibrary(args) {
|
|
153
|
-
const
|
|
154
|
-
const
|
|
155
|
-
if (!
|
|
153
|
+
const key = process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
154
|
+
const baseUrl = (process.env.MCP_SCRAPER_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
155
|
+
if (!key) return;
|
|
156
156
|
try {
|
|
157
|
-
const res = await fetch(
|
|
157
|
+
const res = await fetch(`${baseUrl}/memory/mcp-call`, {
|
|
158
158
|
method: "POST",
|
|
159
|
-
headers: { "content-type": "application/json" },
|
|
160
|
-
body: JSON.stringify({
|
|
159
|
+
headers: { "content-type": "application/json", "x-api-key": key },
|
|
160
|
+
body: JSON.stringify({ toolName: "libraryIngestTool", args })
|
|
161
161
|
});
|
|
162
162
|
const body = await res.json().catch(() => null);
|
|
163
|
-
if (!res.ok || body?.
|
|
164
|
-
console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.
|
|
163
|
+
if (!res.ok || body?.ok === false) {
|
|
164
|
+
console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.error ?? "");
|
|
165
165
|
}
|
|
166
166
|
} catch (err) {
|
|
167
167
|
console.warn("[memory-library-sink] ingest failed:", err?.message);
|
|
@@ -12,13 +12,13 @@ import {
|
|
|
12
12
|
registerScheduledResultsMcpTools,
|
|
13
13
|
registerSerpIntelligenceCaptureTools,
|
|
14
14
|
resolveDeploymentProfile
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-2JVZQGSN.js";
|
|
16
16
|
import {
|
|
17
17
|
renderInstallTerminal
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-ABF2CGOZ.js";
|
|
19
19
|
import {
|
|
20
20
|
PACKAGE_VERSION
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-XIJI34RM.js";
|
|
22
22
|
|
|
23
23
|
// src/mcp/stdio-runtime.ts
|
|
24
24
|
import { readFileSync } from "fs";
|