mcp-scraper 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -5
- package/dist/bin/api-server.cjs +1072 -349
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +4 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +659 -210
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +108 -9
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -2
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/chunk-DUEW4EOO.js +214 -0
- package/dist/chunk-DUEW4EOO.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/chunk-HS6OKELW.js +1786 -0
- package/dist/chunk-HS6OKELW.js.map +1 -0
- package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
- package/dist/chunk-IPW4LFOT.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/chunk-WN7PBKMV.js +28 -0
- package/dist/chunk-WN7PBKMV.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
- package/dist/server-24XIQ6W6.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
- package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1534 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js +0 -793
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
finishHarvestAttempt,
|
|
3
3
|
startHarvestAttempt
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-DBQDG7EH.js";
|
|
5
5
|
import {
|
|
6
6
|
CaptchaError,
|
|
7
7
|
LocationMismatchError,
|
|
@@ -375,4 +375,4 @@ export {
|
|
|
375
375
|
harvestProblemResponse,
|
|
376
376
|
createHarvestAttemptRecorder
|
|
377
377
|
};
|
|
378
|
-
//# sourceMappingURL=chunk-
|
|
378
|
+
//# sourceMappingURL=chunk-SXTXMFEQ.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// src/lib/browser-service-env.ts
|
|
2
|
+
function browserServiceApiKey() {
|
|
3
|
+
const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
|
|
4
|
+
return value || void 0;
|
|
5
|
+
}
|
|
6
|
+
function browserServiceProxyId() {
|
|
7
|
+
const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
|
|
8
|
+
return value || void 0;
|
|
9
|
+
}
|
|
10
|
+
function browserServiceProfileName() {
|
|
11
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
|
|
12
|
+
return value || void 0;
|
|
13
|
+
}
|
|
14
|
+
function browserServiceProfileSaveChanges() {
|
|
15
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
|
|
16
|
+
if (!value) return void 0;
|
|
17
|
+
if (["1", "true", "yes", "on"].includes(value)) return true;
|
|
18
|
+
if (["0", "false", "no", "off"].includes(value)) return false;
|
|
19
|
+
return void 0;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export {
|
|
23
|
+
browserServiceApiKey,
|
|
24
|
+
browserServiceProxyId,
|
|
25
|
+
browserServiceProfileName,
|
|
26
|
+
browserServiceProfileSaveChanges
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=chunk-WN7PBKMV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/browser-service-env.ts"],"sourcesContent":["export function browserServiceApiKey(): string | undefined {\n const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim()\n return value || undefined\n}\n\nexport function browserServiceProxyId(): string | undefined {\n const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim()\n return value || undefined\n}\n\nexport function browserServiceProfileName(): string | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_NAME ??\n process.env.BROWSER_SERVICE_PROFILE_NAME ??\n process.env.KERNEL_BROWSER_PROFILE_NAME ??\n process.env.KERNEL_PROFILE_NAME\n )?.trim()\n return value || undefined\n}\n\nexport function browserServiceProfileSaveChanges(): boolean | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ??\n process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_PROFILE_SAVE_CHANGES\n )?.trim().toLowerCase()\n if (!value) return undefined\n if (['1', 'true', 'yes', 'on'].includes(value)) return true\n if (['0', 'false', 'no', 'off'].includes(value)) return false\n return undefined\n}\n"],"mappings":";AAAO,SAAS,uBAA2C;AACzD,QAAM,SAAS,QAAQ,IAAI,2BAA2B,QAAQ,IAAI,iBAAiB,KAAK;AACxF,SAAO,SAAS;AAClB;AAEO,SAAS,wBAA4C;AAC1D,QAAM,SAAS,QAAQ,IAAI,4BAA4B,QAAQ,IAAI,kBAAkB,KAAK;AAC1F,SAAO,SAAS;AAClB;AAEO,SAAS,4BAAgD;AAC9D,QAAM,SACJ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,gCACZ,QAAQ,IAAI,+BACZ,QAAQ,IAAI,sBACX,KAAK;AACR,SAAO,SAAS;AAClB;AAEO,SAAS,mCAAwD;AACtE,QAAM,SACJ,QAAQ,IAAI,sCACZ,QAAQ,IAAI,wCACZ,QAAQ,IAAI,uCACZ,QAAQ,IAAI,8BACX,KAAK,EAAE,YAAY;AACtB,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,CAAC,KAAK,QAAQ,OAAO,IAAI,EAAE,SAAS,KAAK,EAAG,QAAO;AACvD,MAAI,CAAC,KAAK,SAAS,MAAM,KAAK,EAAE,SAAS,KAAK,EAAG,QAAO;AACxD,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.2.20'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
SiteAuditJobRowSchema,
|
|
3
3
|
SiteAuditPhaseLogRowSchema,
|
|
4
|
+
advanceWorkflowRunStep,
|
|
5
|
+
appendWorkflowArtifacts,
|
|
4
6
|
cancelJob,
|
|
5
7
|
checkRateLimit,
|
|
6
8
|
claimMonthlyFreeRefresh,
|
|
@@ -41,6 +43,7 @@ import {
|
|
|
41
43
|
getWorkflowArtifact,
|
|
42
44
|
getWorkflowRun,
|
|
43
45
|
getWorkflowRunByIdempotencyKey,
|
|
46
|
+
getWorkflowRunStepState,
|
|
44
47
|
getWorkflowSchedule,
|
|
45
48
|
hashPassword,
|
|
46
49
|
ledgerExistsForOperation,
|
|
@@ -74,10 +77,12 @@ import {
|
|
|
74
77
|
stripeEventAlreadyProcessed,
|
|
75
78
|
updateKpoJobState,
|
|
76
79
|
verifyPassword
|
|
77
|
-
} from "./chunk-
|
|
80
|
+
} from "./chunk-DBQDG7EH.js";
|
|
78
81
|
export {
|
|
79
82
|
SiteAuditJobRowSchema,
|
|
80
83
|
SiteAuditPhaseLogRowSchema,
|
|
84
|
+
advanceWorkflowRunStep,
|
|
85
|
+
appendWorkflowArtifacts,
|
|
81
86
|
cancelJob,
|
|
82
87
|
checkRateLimit,
|
|
83
88
|
claimMonthlyFreeRefresh,
|
|
@@ -118,6 +123,7 @@ export {
|
|
|
118
123
|
getWorkflowArtifact,
|
|
119
124
|
getWorkflowRun,
|
|
120
125
|
getWorkflowRunByIdempotencyKey,
|
|
126
|
+
getWorkflowRunStepState,
|
|
121
127
|
getWorkflowSchedule,
|
|
122
128
|
hashPassword,
|
|
123
129
|
ledgerExistsForOperation,
|
|
@@ -152,4 +158,4 @@ export {
|
|
|
152
158
|
updateKpoJobState,
|
|
153
159
|
verifyPassword
|
|
154
160
|
};
|
|
155
|
-
//# sourceMappingURL=db-
|
|
161
|
+
//# sourceMappingURL=db-BE4JVB3V.js.map
|
package/dist/index.cjs
CHANGED
|
@@ -38,13 +38,15 @@ module.exports = __toCommonJS(src_exports);
|
|
|
38
38
|
|
|
39
39
|
// src/schemas.ts
|
|
40
40
|
var import_zod = require("zod");
|
|
41
|
+
var DEFAULT_PROXY_MODE = "configured";
|
|
42
|
+
var DEFAULT_MAPS_PROXY_MODE = "location";
|
|
41
43
|
var HarvestOptionsSchema = import_zod.z.object({
|
|
42
44
|
query: import_zod.z.string().min(1),
|
|
43
45
|
location: import_zod.z.string().optional(),
|
|
44
46
|
gl: import_zod.z.string().length(2).default("us"),
|
|
45
47
|
hl: import_zod.z.string().length(2).default("en"),
|
|
46
48
|
device: import_zod.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
47
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(
|
|
49
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
|
|
48
50
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
|
|
49
51
|
debug: import_zod.z.boolean().default(false),
|
|
50
52
|
depth: import_zod.z.number().int().min(1).max(30).default(3),
|
|
@@ -77,7 +79,7 @@ var MapsSearchOptionsSchema = import_zod.z.object({
|
|
|
77
79
|
gl: import_zod.z.string().length(2).default("us"),
|
|
78
80
|
hl: import_zod.z.string().length(2).default("en"),
|
|
79
81
|
maxResults: import_zod.z.number().int().min(1).max(50).default(10),
|
|
80
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(
|
|
82
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
81
83
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
|
|
82
84
|
debug: import_zod.z.boolean().default(false),
|
|
83
85
|
kernelApiKey: import_zod.z.string().optional(),
|
|
@@ -2790,7 +2792,7 @@ async function harvest(rawOptions) {
|
|
|
2790
2792
|
const signal = getAbortSignal(rawOptions);
|
|
2791
2793
|
const onAttemptEvent = getAttemptLogSink(rawOptions);
|
|
2792
2794
|
const requestedProxyMode = raw.proxyMode;
|
|
2793
|
-
const proxyMode = requestedProxyMode === "
|
|
2795
|
+
const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
|
|
2794
2796
|
const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
|
|
2795
2797
|
const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
|
|
2796
2798
|
const proxyOpts = {
|