mcp-scraper 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/bin/api-server.cjs +1020 -348
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +314 -55
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +321 -167
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +1128 -530
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +5 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -1
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/{chunk-76P4DKWR.js → chunk-L4OWOUGR.js} +315 -56
- package/dist/chunk-L4OWOUGR.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/{chunk-CQTAKXBN.js → chunk-XGUDTDZ2.js} +7 -104
- package/dist/chunk-XGUDTDZ2.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-7NE45K2A.js} +211 -37
- package/dist/server-7NE45K2A.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-MIYG2B2I.js} +5 -4
- package/dist/{worker-OZSWIS3F.js.map → worker-MIYG2B2I.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1406 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
finishHarvestAttempt,
|
|
3
3
|
startHarvestAttempt
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-DBQDG7EH.js";
|
|
5
5
|
import {
|
|
6
6
|
CaptchaError,
|
|
7
7
|
LocationMismatchError,
|
|
@@ -375,4 +375,4 @@ export {
|
|
|
375
375
|
harvestProblemResponse,
|
|
376
376
|
createHarvestAttemptRecorder
|
|
377
377
|
};
|
|
378
|
-
//# sourceMappingURL=chunk-
|
|
378
|
+
//# sourceMappingURL=chunk-SXTXMFEQ.js.map
|
|
@@ -6,6 +6,11 @@ import {
|
|
|
6
6
|
RequestAbortedError,
|
|
7
7
|
sanitizeVendorName
|
|
8
8
|
} from "./chunk-M2S27J6Z.js";
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_PROXY_MODE,
|
|
11
|
+
HarvestOptionsSchema,
|
|
12
|
+
RawPAAItemSchema
|
|
13
|
+
} from "./chunk-LFATOGDF.js";
|
|
9
14
|
|
|
10
15
|
// src/lib/browser-service-env.ts
|
|
11
16
|
function browserServiceApiKey() {
|
|
@@ -17,102 +22,6 @@ function browserServiceProxyId() {
|
|
|
17
22
|
return value || void 0;
|
|
18
23
|
}
|
|
19
24
|
|
|
20
|
-
// src/schemas.ts
|
|
21
|
-
import { z } from "zod";
|
|
22
|
-
var HarvestOptionsSchema = z.object({
|
|
23
|
-
query: z.string().min(1),
|
|
24
|
-
location: z.string().optional(),
|
|
25
|
-
gl: z.string().length(2).default("us"),
|
|
26
|
-
hl: z.string().length(2).default("en"),
|
|
27
|
-
device: z.enum(["desktop", "mobile"]).default("desktop"),
|
|
28
|
-
proxyMode: z.enum(["location", "configured", "none"]).default("location"),
|
|
29
|
-
proxyZip: z.string().regex(/^\d{5}$/).optional(),
|
|
30
|
-
debug: z.boolean().default(false),
|
|
31
|
-
depth: z.number().int().min(1).max(30).default(3),
|
|
32
|
-
maxQuestions: z.number().int().min(1).max(1e3).default(100),
|
|
33
|
-
headless: z.boolean().default(false),
|
|
34
|
-
profileDir: z.string().optional(),
|
|
35
|
-
proxy: z.string().url().optional(),
|
|
36
|
-
kernelApiKey: z.string().optional(),
|
|
37
|
-
kernelProxyId: z.string().optional(),
|
|
38
|
-
kernelProxyResolution: z.unknown().optional(),
|
|
39
|
-
outputDir: z.string().default("./paa-output"),
|
|
40
|
-
format: z.enum(["json", "csv", "both"]).default("both"),
|
|
41
|
-
serpOnly: z.boolean().default(false),
|
|
42
|
-
pages: z.number().int().min(1).max(2).default(1)
|
|
43
|
-
});
|
|
44
|
-
var MapsPlaceOptionsSchema = z.object({
|
|
45
|
-
businessName: z.string().min(1),
|
|
46
|
-
location: z.string().min(1),
|
|
47
|
-
gl: z.string().length(2).default("us"),
|
|
48
|
-
hl: z.string().length(2).default("en"),
|
|
49
|
-
includeReviews: z.boolean().default(false),
|
|
50
|
-
maxReviews: z.number().int().min(1).max(500).default(50),
|
|
51
|
-
kernelApiKey: z.string().optional(),
|
|
52
|
-
kernelProxyId: z.string().optional(),
|
|
53
|
-
headless: z.boolean().default(true)
|
|
54
|
-
});
|
|
55
|
-
var MapsSearchOptionsSchema = z.object({
|
|
56
|
-
query: z.string().min(1),
|
|
57
|
-
location: z.string().optional(),
|
|
58
|
-
gl: z.string().length(2).default("us"),
|
|
59
|
-
hl: z.string().length(2).default("en"),
|
|
60
|
-
maxResults: z.number().int().min(1).max(50).default(10),
|
|
61
|
-
proxyMode: z.enum(["location", "configured", "none"]).default("location"),
|
|
62
|
-
proxyZip: z.string().regex(/^\d{5}$/).optional(),
|
|
63
|
-
debug: z.boolean().default(false),
|
|
64
|
-
kernelApiKey: z.string().optional(),
|
|
65
|
-
kernelProxyId: z.string().optional(),
|
|
66
|
-
kernelProxyResolution: z.unknown().optional(),
|
|
67
|
-
headless: z.boolean().default(true)
|
|
68
|
-
});
|
|
69
|
-
var RawPAAItemSchema = z.object({
|
|
70
|
-
question: z.string().min(1),
|
|
71
|
-
answer: z.string().optional(),
|
|
72
|
-
sourceTitle: z.string().optional(),
|
|
73
|
-
sourceSite: z.string().optional(),
|
|
74
|
-
sourceCite: z.string().optional()
|
|
75
|
-
});
|
|
76
|
-
var RawMapsOverviewSchema = z.object({
|
|
77
|
-
name: z.string().nullable(),
|
|
78
|
-
rating: z.string().nullable(),
|
|
79
|
-
reviewCount: z.string().nullable(),
|
|
80
|
-
category: z.string().nullable(),
|
|
81
|
-
address: z.string().nullable(),
|
|
82
|
-
hoursSummary: z.string().nullable(),
|
|
83
|
-
phone: z.string().nullable(),
|
|
84
|
-
phoneDisplay: z.string().nullable(),
|
|
85
|
-
website: z.string().nullable(),
|
|
86
|
-
plusCode: z.string().nullable(),
|
|
87
|
-
bookingUrl: z.string().nullable()
|
|
88
|
-
});
|
|
89
|
-
var RawMapsHoursRowSchema = z.object({
|
|
90
|
-
day: z.string(),
|
|
91
|
-
hours: z.string()
|
|
92
|
-
});
|
|
93
|
-
var RawMapsReviewStatsSchema = z.object({
|
|
94
|
-
reviewHistogram: z.array(z.object({
|
|
95
|
-
stars: z.number(),
|
|
96
|
-
count: z.string()
|
|
97
|
-
})),
|
|
98
|
-
reviewTopics: z.array(z.object({
|
|
99
|
-
label: z.string(),
|
|
100
|
-
count: z.string()
|
|
101
|
-
}))
|
|
102
|
-
});
|
|
103
|
-
var RawMapsReviewCardSchema = z.object({
|
|
104
|
-
reviewId: z.string(),
|
|
105
|
-
author: z.string().nullable(),
|
|
106
|
-
stars: z.string().nullable(),
|
|
107
|
-
date: z.string().nullable(),
|
|
108
|
-
text: z.string().nullable(),
|
|
109
|
-
ownerResponse: z.string().nullable()
|
|
110
|
-
});
|
|
111
|
-
var RawMapsAboutAttributeSchema = z.object({
|
|
112
|
-
section: z.string(),
|
|
113
|
-
attribute: z.string()
|
|
114
|
-
});
|
|
115
|
-
|
|
116
25
|
// src/driver/BrowserDriver.ts
|
|
117
26
|
import { chromium } from "playwright-extra";
|
|
118
27
|
import StealthPlugin from "puppeteer-extra-plugin-stealth";
|
|
@@ -2743,7 +2652,7 @@ async function harvest(rawOptions) {
|
|
|
2743
2652
|
const signal = getAbortSignal(rawOptions);
|
|
2744
2653
|
const onAttemptEvent = getAttemptLogSink(rawOptions);
|
|
2745
2654
|
const requestedProxyMode = raw.proxyMode;
|
|
2746
|
-
const proxyMode = requestedProxyMode === "
|
|
2655
|
+
const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
|
|
2747
2656
|
const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
|
|
2748
2657
|
const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
|
|
2749
2658
|
const proxyOpts = {
|
|
@@ -2957,12 +2866,6 @@ async function harvest(rawOptions) {
|
|
|
2957
2866
|
}
|
|
2958
2867
|
|
|
2959
2868
|
export {
|
|
2960
|
-
MapsPlaceOptionsSchema,
|
|
2961
|
-
MapsSearchOptionsSchema,
|
|
2962
|
-
RawMapsOverviewSchema,
|
|
2963
|
-
RawMapsHoursRowSchema,
|
|
2964
|
-
RawMapsReviewStatsSchema,
|
|
2965
|
-
RawMapsAboutAttributeSchema,
|
|
2966
2869
|
browserServiceApiKey,
|
|
2967
2870
|
browserServiceProxyId,
|
|
2968
2871
|
MapsSelectors,
|
|
@@ -2972,4 +2875,4 @@ export {
|
|
|
2972
2875
|
resolveKernelProxyId,
|
|
2973
2876
|
harvest
|
|
2974
2877
|
};
|
|
2975
|
-
//# sourceMappingURL=chunk-
|
|
2878
|
+
//# sourceMappingURL=chunk-XGUDTDZ2.js.map
|