mcp-scraper 0.32.3 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +13 -15
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +5 -5
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +2 -2
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +6 -6
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-MKZIL4EA.js → chunk-AT2PFWHW.js} +5 -5
- package/dist/chunk-AT2PFWHW.js.map +1 -0
- package/dist/{chunk-AEMABLEL.js → chunk-ERORC7AP.js} +1 -3
- package/dist/chunk-IASKABFL.js +7 -0
- package/dist/chunk-IASKABFL.js.map +1 -0
- package/dist/{chunk-GNRHBIYZ.js → chunk-JURXY3DP.js} +7 -7
- package/dist/chunk-JURXY3DP.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/{server-NHLEIKDZ.js → server-RKAKXRAR.js} +9 -12
- package/dist/server-RKAKXRAR.js.map +1 -0
- package/dist/{worker-ZZCZNY3R.js → worker-JU5FX3NE.js} +2 -2
- package/docs/adr/0003-waive-unrecoverable-scheduled-model-cost.md +22 -0
- package/docs/adr/README.md +1 -0
- package/docs/mcp-tool-manifest.generated.json +7 -12
- package/package.json +1 -1
- package/dist/chunk-GNRHBIYZ.js.map +0 -1
- package/dist/chunk-MKZIL4EA.js.map +0 -1
- package/dist/chunk-XF46AB5C.js +0 -7
- package/dist/chunk-XF46AB5C.js.map +0 -1
- package/dist/server-NHLEIKDZ.js.map +0 -1
- /package/dist/{chunk-AEMABLEL.js.map → chunk-ERORC7AP.js.map} +0 -0
- /package/dist/{worker-ZZCZNY3R.js.map → worker-JU5FX3NE.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -12,12 +12,10 @@ import {
|
|
|
12
12
|
MapsSelectors,
|
|
13
13
|
buildYouTubeChannelVideosUrl,
|
|
14
14
|
deleteKernelProxyId,
|
|
15
|
-
encodeUule,
|
|
16
15
|
harvest,
|
|
17
|
-
normalizeLocation,
|
|
18
16
|
resolveKernelProxyId,
|
|
19
17
|
serpArrivalUrl
|
|
20
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-ERORC7AP.js";
|
|
21
19
|
import {
|
|
22
20
|
HttpMcpToolExecutor,
|
|
23
21
|
assessTranscriptSignal,
|
|
@@ -37,7 +35,7 @@ import {
|
|
|
37
35
|
sanitizeAttempts,
|
|
38
36
|
sanitizeHarvestResult,
|
|
39
37
|
transcribeMediaUrl
|
|
40
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-JURXY3DP.js";
|
|
41
39
|
import {
|
|
42
40
|
auditImages,
|
|
43
41
|
buildLinkReport,
|
|
@@ -66,7 +64,7 @@ import {
|
|
|
66
64
|
workflowOutputBaseDir,
|
|
67
65
|
workflowStepCount,
|
|
68
66
|
workflowSupportsSteps
|
|
69
|
-
} from "./chunk-
|
|
67
|
+
} from "./chunk-AT2PFWHW.js";
|
|
70
68
|
import "./chunk-K443GQY5.js";
|
|
71
69
|
import {
|
|
72
70
|
DEFAULT_MAPS_PROXY_MODE,
|
|
@@ -78,7 +76,7 @@ import {
|
|
|
78
76
|
RawMapsOverviewSchema,
|
|
79
77
|
RawMapsReviewStatsSchema
|
|
80
78
|
} from "./chunk-V73MPRU6.js";
|
|
81
|
-
import "./chunk-
|
|
79
|
+
import "./chunk-IASKABFL.js";
|
|
82
80
|
import {
|
|
83
81
|
completeExtractJob,
|
|
84
82
|
countSuccessfulPages,
|
|
@@ -10029,7 +10027,7 @@ var HarvestBodySchema = z13.object({
|
|
|
10029
10027
|
gl: z13.string().optional(),
|
|
10030
10028
|
hl: z13.string().optional(),
|
|
10031
10029
|
device: z13.enum(["desktop", "mobile"]).optional(),
|
|
10032
|
-
proxyMode: z13.enum(["
|
|
10030
|
+
proxyMode: z13.enum(["configured", "none"]).optional(),
|
|
10033
10031
|
proxyZip: z13.string().regex(/^\d{5}$/).optional(),
|
|
10034
10032
|
debug: z13.boolean().optional(),
|
|
10035
10033
|
serpOnly: z13.boolean().optional(),
|
|
@@ -13088,13 +13086,11 @@ var MAPS_REDIRECT_BASE = "https://serp-redirector.vercel.app";
|
|
|
13088
13086
|
function buildGoogleLocalResultsUrl(input) {
|
|
13089
13087
|
const query = [input.query, input.location].filter(Boolean).join(" ");
|
|
13090
13088
|
const params = new URLSearchParams({ q: query, udm: "1", gl: input.gl, hl: input.hl, pws: "0" });
|
|
13091
|
-
if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
|
|
13092
13089
|
return `https://www.google.com/search?${params.toString()}`;
|
|
13093
13090
|
}
|
|
13094
13091
|
function buildGoogleOrganicSearchUrl(input) {
|
|
13095
13092
|
const query = [input.query, input.location].filter(Boolean).join(" ");
|
|
13096
13093
|
const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
|
|
13097
|
-
if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
|
|
13098
13094
|
return `https://www.google.com/search?${params.toString()}`;
|
|
13099
13095
|
}
|
|
13100
13096
|
function mapsIdentifiersFromUrl(url) {
|
|
@@ -13975,6 +13971,7 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
|
13975
13971
|
}
|
|
13976
13972
|
}
|
|
13977
13973
|
async function runMapsSearchWithRotation(options) {
|
|
13974
|
+
if (options.proxyMode === "location") options = { ...options, proxyMode: "none" };
|
|
13978
13975
|
const attempts = [];
|
|
13979
13976
|
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode) + 1;
|
|
13980
13977
|
const started = Date.now();
|
|
@@ -14766,7 +14763,7 @@ var DirectoryWorkflowOptionsSchema = z23.object({
|
|
|
14766
14763
|
saveCsv: z23.boolean().default(true),
|
|
14767
14764
|
gl: z23.string().length(2).default("us"),
|
|
14768
14765
|
hl: z23.string().length(2).default("en"),
|
|
14769
|
-
proxyMode: z23.enum(["
|
|
14766
|
+
proxyMode: z23.enum(["configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
14770
14767
|
proxyZip: z23.string().regex(/^\d{5}$/).optional(),
|
|
14771
14768
|
debug: z23.boolean().default(false),
|
|
14772
14769
|
headless: z23.boolean().default(true),
|
|
@@ -27564,7 +27561,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
27564
27561
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
27565
27562
|
return c.json({ error: "Unauthorized" }, 401);
|
|
27566
27563
|
}
|
|
27567
|
-
const { drainQueue } = await import("./worker-
|
|
27564
|
+
const { drainQueue } = await import("./worker-JU5FX3NE.js");
|
|
27568
27565
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
27569
27566
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
27570
27567
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -27767,4 +27764,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
27767
27764
|
export {
|
|
27768
27765
|
app
|
|
27769
27766
|
};
|
|
27770
|
-
//# sourceMappingURL=server-
|
|
27767
|
+
//# sourceMappingURL=server-RKAKXRAR.js.map
|