mcp-scraper 0.21.2 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1061 -308
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +101 -34
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +22 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-ZJDANZTQ.js → chunk-E5J4HJBO.js} +3 -2
- package/dist/chunk-E5J4HJBO.js.map +1 -0
- package/dist/{chunk-DZBGULWW.js → chunk-ICT7DDHL.js} +399 -3
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-2FF4GBQ4.js → chunk-JHI373VB.js} +49 -7
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-GMZFQRH2.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-ZNNYPE2U.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-X3NIXHHG.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-JBME5WJR.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-EONGTNJM.js → db-WRILVOFY.js} +6 -2
- package/dist/{extract-bundle-HMCYUVBJ.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +22 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-CHTX6YQA.js → server-OLOOFN5W.js} +499 -271
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-5V5EEQ72.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-GE5JMHE6.js → worker-RV2TYXE3.js} +15 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-2FF4GBQ4.js.map +0 -1
- package/dist/chunk-DZBGULWW.js.map +0 -1
- package/dist/chunk-GMZFQRH2.js.map +0 -1
- package/dist/chunk-NBF4F4UT.js +0 -7
- package/dist/chunk-NBF4F4UT.js.map +0 -1
- package/dist/chunk-ZJDANZTQ.js.map +0 -1
- package/dist/chunk-ZNNYPE2U.js.map +0 -1
- package/dist/server-CHTX6YQA.js.map +0 -1
- package/dist/worker-GE5JMHE6.js.map +0 -1
- /package/dist/{chunk-X3NIXHHG.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-JBME5WJR.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-EONGTNJM.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-HMCYUVBJ.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-5V5EEQ72.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
|
@@ -7,19 +7,19 @@ import {
|
|
|
7
7
|
registerMemoryMcpTools,
|
|
8
8
|
registerPaaExtractorMcpTools,
|
|
9
9
|
registerSerpIntelligenceCaptureTools
|
|
10
|
-
} from "../chunk-
|
|
10
|
+
} from "../chunk-N4JHXPW6.js";
|
|
11
11
|
import "../chunk-R7EETU7Z.js";
|
|
12
|
-
import "../chunk-
|
|
12
|
+
import "../chunk-MTSBI7ZH.js";
|
|
13
13
|
import {
|
|
14
14
|
renderInstallTerminal
|
|
15
15
|
} from "../chunk-YNHPZU6B.js";
|
|
16
16
|
import "../chunk-XGIPATLV.js";
|
|
17
17
|
import {
|
|
18
18
|
PACKAGE_VERSION
|
|
19
|
-
} from "../chunk-
|
|
20
|
-
import "../chunk-
|
|
19
|
+
} from "../chunk-WQI7JIIN.js";
|
|
20
|
+
import "../chunk-JHI373VB.js";
|
|
21
21
|
import "../chunk-M2S27J6Z.js";
|
|
22
|
-
import "../chunk-
|
|
22
|
+
import "../chunk-ICT7DDHL.js";
|
|
23
23
|
|
|
24
24
|
// bin/mcp-stdio-server.ts
|
|
25
25
|
import { readFileSync } from "fs";
|
package/dist/bin/paa-harvest.cjs
CHANGED
|
@@ -272,6 +272,7 @@ function getDb() {
|
|
|
272
272
|
if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
|
|
273
273
|
return _db;
|
|
274
274
|
}
|
|
275
|
+
var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
275
276
|
var SiteAuditJobRowSchema = import_zod2.z.object({
|
|
276
277
|
id: import_zod2.z.string(),
|
|
277
278
|
user_id: import_zod2.z.number(),
|
|
@@ -313,9 +314,29 @@ function kernelCostUsd(ms, headful) {
|
|
|
313
314
|
|
|
314
315
|
// src/api/cost-telemetry.ts
|
|
315
316
|
var _ready = false;
|
|
317
|
+
var _migrationPromise = null;
|
|
316
318
|
async function migrateCostTelemetry() {
|
|
317
319
|
if (_ready) return;
|
|
320
|
+
if (_migrationPromise) return _migrationPromise;
|
|
321
|
+
_migrationPromise = runCostTelemetryMigration().finally(() => {
|
|
322
|
+
_migrationPromise = null;
|
|
323
|
+
});
|
|
324
|
+
return _migrationPromise;
|
|
325
|
+
}
|
|
326
|
+
async function runCostTelemetryMigration() {
|
|
318
327
|
const db = getDb();
|
|
328
|
+
const existing = await db.execute(`
|
|
329
|
+
SELECT
|
|
330
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
331
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
332
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
|
|
333
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
334
|
+
AS ready
|
|
335
|
+
`);
|
|
336
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
337
|
+
_ready = true;
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
319
340
|
await db.execute(`
|
|
320
341
|
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
321
342
|
id TEXT PRIMARY KEY,
|
|
@@ -583,6 +604,7 @@ var BrowserDriver = class {
|
|
|
583
604
|
else if (config.headlessMode === "headless") useHeadless = true;
|
|
584
605
|
else useHeadless = costCtx?.forceHeadless === true || process.env.KERNEL_HEADLESS_DISABLED !== "true" && !!costCtx?.op && HEADLESS_OPS.has(costCtx.op);
|
|
585
606
|
this.kernelHeadlessSent = useHeadless ? true : null;
|
|
607
|
+
if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
|
|
586
608
|
const kernelBrowser = await this.kernelClient.browsers.create({
|
|
587
609
|
stealth: true,
|
|
588
610
|
timeout_seconds: timeoutSeconds,
|