mcp-scraper 0.21.3 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +728 -181
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +97 -32
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +21 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js} +2 -2
- package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js} +366 -50
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-OBNKOYOE.js → chunk-JHI373VB.js} +7 -5
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-5IU5ZFVO.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-XDRHBGJR.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-2P5VFIJM.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-Z35S224E.js → db-WRILVOFY.js} +4 -2
- package/dist/{extract-bundle-4RO3YU4Q.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +21 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-PBBRSLBP.js → server-OLOOFN5W.js} +246 -100
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js} +9 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-5IU5ZFVO.js.map +0 -1
- package/dist/chunk-7LO6KLKD.js.map +0 -1
- package/dist/chunk-H7APH5UG.js.map +0 -1
- package/dist/chunk-OBNKOYOE.js.map +0 -1
- package/dist/chunk-VYUWWIVR.js +0 -7
- package/dist/chunk-VYUWWIVR.js.map +0 -1
- package/dist/server-PBBRSLBP.js.map +0 -1
- package/dist/worker-BXJQ7J4N.js.map +0 -1
- /package/dist/{chunk-JJ5RMEGX.js.map → chunk-E5J4HJBO.js.map} +0 -0
- /package/dist/{chunk-XDRHBGJR.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-2P5VFIJM.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-Z35S224E.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-4RO3YU4Q.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-FDH6ZXKT.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
LedgerOperation
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-JHI373VB.js";
|
|
4
4
|
import {
|
|
5
5
|
sanitizeVendorName
|
|
6
6
|
} from "./chunk-M2S27J6Z.js";
|
|
7
7
|
import {
|
|
8
8
|
getDb
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-ICT7DDHL.js";
|
|
10
10
|
|
|
11
11
|
// src/api/site-extract-repository.ts
|
|
12
12
|
function rowToJob(r) {
|
|
@@ -127,4 +127,4 @@ export {
|
|
|
127
127
|
failExtractJob,
|
|
128
128
|
settleExtractJob
|
|
129
129
|
};
|
|
130
|
-
//# sourceMappingURL=chunk-
|
|
130
|
+
//# sourceMappingURL=chunk-PNBYYAEK.js.map
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
import {
|
|
7
7
|
finishHarvestAttempt,
|
|
8
8
|
startHarvestAttempt
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-ICT7DDHL.js";
|
|
10
10
|
|
|
11
11
|
// src/api/harvest-problems.ts
|
|
12
12
|
function errorMessage(err) {
|
|
@@ -155,4 +155,4 @@ export {
|
|
|
155
155
|
harvestProblemResponse,
|
|
156
156
|
createHarvestAttemptRecorder
|
|
157
157
|
};
|
|
158
|
-
//# sourceMappingURL=chunk-
|
|
158
|
+
//# sourceMappingURL=chunk-SDHPQTFM.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.21.4'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -95,12 +95,13 @@ import {
|
|
|
95
95
|
setStripeCustomerId,
|
|
96
96
|
setSubscriptionTier,
|
|
97
97
|
setUserMemoryCreds,
|
|
98
|
+
settleDebitMcIdempotent,
|
|
98
99
|
startHarvestAttempt,
|
|
99
100
|
stripeEventAlreadyProcessed,
|
|
100
101
|
updateKpoJobState,
|
|
101
102
|
upsertPageSnapshot,
|
|
102
103
|
verifyPassword
|
|
103
|
-
} from "./chunk-
|
|
104
|
+
} from "./chunk-ICT7DDHL.js";
|
|
104
105
|
export {
|
|
105
106
|
CREDIT_LOT_TTL,
|
|
106
107
|
SiteAuditJobRowSchema,
|
|
@@ -198,10 +199,11 @@ export {
|
|
|
198
199
|
setStripeCustomerId,
|
|
199
200
|
setSubscriptionTier,
|
|
200
201
|
setUserMemoryCreds,
|
|
202
|
+
settleDebitMcIdempotent,
|
|
201
203
|
startHarvestAttempt,
|
|
202
204
|
stripeEventAlreadyProcessed,
|
|
203
205
|
updateKpoJobState,
|
|
204
206
|
upsertPageSnapshot,
|
|
205
207
|
verifyPassword
|
|
206
208
|
};
|
|
207
|
-
//# sourceMappingURL=db-
|
|
209
|
+
//# sourceMappingURL=db-WRILVOFY.js.map
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
} from "./chunk-R7EETU7Z.js";
|
|
11
11
|
import {
|
|
12
12
|
getDb
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-ICT7DDHL.js";
|
|
14
14
|
|
|
15
15
|
// src/api/extract-bundle.ts
|
|
16
16
|
import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
|
|
@@ -328,4 +328,4 @@ ${renderImageSection(extras.imageAudit)}`;
|
|
|
328
328
|
export {
|
|
329
329
|
assembleExtractArtifacts
|
|
330
330
|
};
|
|
331
|
-
//# sourceMappingURL=extract-bundle-
|
|
331
|
+
//# sourceMappingURL=extract-bundle-NFFVMUMT.js.map
|
package/dist/index.cjs
CHANGED
|
@@ -282,6 +282,7 @@ function getDb() {
|
|
|
282
282
|
if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
|
|
283
283
|
return _db;
|
|
284
284
|
}
|
|
285
|
+
var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
285
286
|
var SiteAuditJobRowSchema = import_zod2.z.object({
|
|
286
287
|
id: import_zod2.z.string(),
|
|
287
288
|
user_id: import_zod2.z.number(),
|
|
@@ -337,9 +338,29 @@ function vendorCostUsd(vendor, units) {
|
|
|
337
338
|
|
|
338
339
|
// src/api/cost-telemetry.ts
|
|
339
340
|
var _ready = false;
|
|
341
|
+
var _migrationPromise = null;
|
|
340
342
|
async function migrateCostTelemetry() {
|
|
341
343
|
if (_ready) return;
|
|
344
|
+
if (_migrationPromise) return _migrationPromise;
|
|
345
|
+
_migrationPromise = runCostTelemetryMigration().finally(() => {
|
|
346
|
+
_migrationPromise = null;
|
|
347
|
+
});
|
|
348
|
+
return _migrationPromise;
|
|
349
|
+
}
|
|
350
|
+
async function runCostTelemetryMigration() {
|
|
342
351
|
const db = getDb();
|
|
352
|
+
const existing = await db.execute(`
|
|
353
|
+
SELECT
|
|
354
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
355
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
356
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name = 'method') = 1
|
|
357
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
358
|
+
AS ready
|
|
359
|
+
`);
|
|
360
|
+
if (Number(existing.rows[0]?.ready ?? 0) === 1) {
|
|
361
|
+
_ready = true;
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
343
364
|
await db.execute(`
|
|
344
365
|
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
345
366
|
id TEXT PRIMARY KEY,
|