mcp-scraper 0.75.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -1
- package/README.md +2 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +61 -0
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +8 -161
- package/dist/{chunk-DT7UD2ER.js → chunk-BOGRXBH2.js} +4440 -4446
- package/dist/{chunk-RDPWFAR3.js → chunk-JNFOBQVH.js} +1 -1
- package/dist/chunk-SDX6TRJ6.js +223 -0
- package/dist/{chunk-45AZ5CR2.js → chunk-XSYEDC25.js} +1 -1
- package/dist/{gmail-service-E52EE4GX.js → gmail-service-TIU4V5JA.js} +2 -2
- package/dist/{server-KNCC6FXQ.js → server-IAGYCPN6.js} +93 -93
- package/package.json +2 -1
- package/dist/{chunk-GLVWJCNU.js → chunk-E6Y3VCOJ.js} +4 -4
- package/dist/{extract-bundle-L2GGQCR6.js → extract-bundle-OO74L67T.js} +7 -7
- package/dist/{worker-7ASMH3MT.js → worker-DR6R7J46.js} +4 -4
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import {
|
|
2
|
+
HttpMcpToolExecutor,
|
|
3
|
+
MEMORY_TOOL_REGISTRY,
|
|
4
|
+
SERVER_INSTRUCTIONS,
|
|
5
|
+
ScheduledResultsMcpExecutor,
|
|
6
|
+
hashOwnerId,
|
|
7
|
+
installInputFieldDescriptions,
|
|
8
|
+
permitsLocalNetworkAccess,
|
|
9
|
+
registerBrowserAgentMcpTools,
|
|
10
|
+
registerMemoryMcpTools,
|
|
11
|
+
registerPaaExtractorMcpTools,
|
|
12
|
+
registerScheduledResultsMcpTools,
|
|
13
|
+
registerSerpIntelligenceCaptureTools,
|
|
14
|
+
resolveDeploymentProfile
|
|
15
|
+
} from "./chunk-BOGRXBH2.js";
|
|
16
|
+
import {
|
|
17
|
+
renderInstallTerminal
|
|
18
|
+
} from "./chunk-XSYEDC25.js";
|
|
19
|
+
import {
|
|
20
|
+
PACKAGE_VERSION
|
|
21
|
+
} from "./chunk-JNFOBQVH.js";
|
|
22
|
+
|
|
23
|
+
// src/mcp/stdio-runtime.ts
|
|
24
|
+
import { readFileSync } from "fs";
|
|
25
|
+
import { homedir } from "os";
|
|
26
|
+
import { join } from "path";
|
|
27
|
+
import { serveStdio } from "@modelcontextprotocol/server/stdio";
|
|
28
|
+
import { McpServer } from "@modelcontextprotocol/server";
|
|
29
|
+
|
|
30
|
+
// src/mcp/hosted-memory-mcp-tool-executor.ts
|
|
31
|
+
var publicToolNameByUpstreamName = new Map(
|
|
32
|
+
MEMORY_TOOL_REGISTRY.map((schema) => [schema.upstreamName, schema.id])
|
|
33
|
+
);
|
|
34
|
+
function parseJsonRpcEnvelope(text) {
|
|
35
|
+
const payloads = text.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).filter(Boolean);
|
|
36
|
+
if (!payloads.length) return JSON.parse(text);
|
|
37
|
+
for (const payload of payloads) {
|
|
38
|
+
const parsed = JSON.parse(payload);
|
|
39
|
+
if (parsed.result || parsed.error) return parsed;
|
|
40
|
+
}
|
|
41
|
+
throw new Error("hosted MCP returned no JSON-RPC result");
|
|
42
|
+
}
|
|
43
|
+
function errorResult(message) {
|
|
44
|
+
return {
|
|
45
|
+
content: [{ type: "text", text: JSON.stringify({ ok: false, error: message }) }],
|
|
46
|
+
isError: true
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
var HostedMemoryMcpToolExecutor = class {
|
|
50
|
+
baseUrl;
|
|
51
|
+
apiKey;
|
|
52
|
+
constructor(baseUrl, apiKey) {
|
|
53
|
+
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
54
|
+
this.apiKey = apiKey;
|
|
55
|
+
}
|
|
56
|
+
async callMemoryTool(upstreamName, args) {
|
|
57
|
+
const publicToolName = publicToolNameByUpstreamName.get(upstreamName);
|
|
58
|
+
if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`);
|
|
59
|
+
try {
|
|
60
|
+
const res = await fetch(`${this.baseUrl}/mcp`, {
|
|
61
|
+
method: "POST",
|
|
62
|
+
headers: {
|
|
63
|
+
"content-type": "application/json",
|
|
64
|
+
accept: "application/json, text/event-stream",
|
|
65
|
+
"x-api-key": this.apiKey
|
|
66
|
+
},
|
|
67
|
+
body: JSON.stringify({
|
|
68
|
+
jsonrpc: "2.0",
|
|
69
|
+
id: `memory:${publicToolName}`,
|
|
70
|
+
method: "tools/call",
|
|
71
|
+
params: { name: publicToolName, arguments: args }
|
|
72
|
+
})
|
|
73
|
+
});
|
|
74
|
+
const text = await res.text();
|
|
75
|
+
if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`);
|
|
76
|
+
const envelope = parseJsonRpcEnvelope(text);
|
|
77
|
+
if (envelope.error) {
|
|
78
|
+
return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`);
|
|
79
|
+
}
|
|
80
|
+
return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`);
|
|
81
|
+
} catch (err) {
|
|
82
|
+
return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// src/mcp/exact-tool-registration.ts
|
|
88
|
+
function installExactToolRegistrationGuard(server, requiredToolNames) {
|
|
89
|
+
if (requiredToolNames.length === 0) {
|
|
90
|
+
throw new Error("Restricted MCP tool allowlist must contain at least one tool name");
|
|
91
|
+
}
|
|
92
|
+
const required = /* @__PURE__ */ new Set();
|
|
93
|
+
for (const rawName of requiredToolNames) {
|
|
94
|
+
if (typeof rawName !== "string" || rawName.trim() !== rawName || rawName.length === 0) {
|
|
95
|
+
throw new Error("Restricted MCP tool allowlist contains a malformed tool name");
|
|
96
|
+
}
|
|
97
|
+
if (required.has(rawName)) {
|
|
98
|
+
throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${rawName}`);
|
|
99
|
+
}
|
|
100
|
+
required.add(rawName);
|
|
101
|
+
}
|
|
102
|
+
const registered = /* @__PURE__ */ new Set();
|
|
103
|
+
const mutableServer = server;
|
|
104
|
+
const registerTool = mutableServer.registerTool.bind(server);
|
|
105
|
+
mutableServer.registerTool = (name, config, callback) => {
|
|
106
|
+
if (!required.has(name)) return void 0;
|
|
107
|
+
registered.add(name);
|
|
108
|
+
return registerTool(name, config, callback);
|
|
109
|
+
};
|
|
110
|
+
return {
|
|
111
|
+
assertComplete() {
|
|
112
|
+
const missing = [...required].filter((name) => !registered.has(name));
|
|
113
|
+
if (missing.length > 0) {
|
|
114
|
+
throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${missing.join(", ")}`);
|
|
115
|
+
}
|
|
116
|
+
},
|
|
117
|
+
registeredToolNames() {
|
|
118
|
+
return [...registered];
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// src/mcp/stdio-runtime.ts
|
|
124
|
+
var ALL_STDIO_TOOLSETS = /* @__PURE__ */ new Set([
|
|
125
|
+
"paa",
|
|
126
|
+
"serp",
|
|
127
|
+
"browser-agent",
|
|
128
|
+
"scheduled-results",
|
|
129
|
+
"memory"
|
|
130
|
+
]);
|
|
131
|
+
function readApiKeyFile() {
|
|
132
|
+
const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
|
|
133
|
+
const fileNames = [explicitPath, join(homedir(), ".mcp-scraper-key")].filter(Boolean);
|
|
134
|
+
for (const fileName of fileNames) {
|
|
135
|
+
try {
|
|
136
|
+
const value = readFileSync(fileName, "utf8").trim();
|
|
137
|
+
if (value) return value;
|
|
138
|
+
} catch {
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return void 0;
|
|
142
|
+
}
|
|
143
|
+
function buildStdioServer(requiredApiKey, options = {}) {
|
|
144
|
+
const toolsets = options.toolsets ?? ALL_STDIO_TOOLSETS;
|
|
145
|
+
const baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
|
|
146
|
+
const deploymentProfile = resolveDeploymentProfile();
|
|
147
|
+
const localNetworkAccess = permitsLocalNetworkAccess({
|
|
148
|
+
deploymentProfile,
|
|
149
|
+
transportProfile: "stdio",
|
|
150
|
+
baseUrl,
|
|
151
|
+
explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1"
|
|
152
|
+
});
|
|
153
|
+
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
154
|
+
const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, {
|
|
155
|
+
instructions: SERVER_INSTRUCTIONS,
|
|
156
|
+
cacheHints: {
|
|
157
|
+
"server/discover": { ttlMs: 3e5, cacheScope: "private" },
|
|
158
|
+
"tools/list": { ttlMs: 3e5, cacheScope: "private" },
|
|
159
|
+
"resources/list": { ttlMs: 3e5, cacheScope: "private" },
|
|
160
|
+
"resources/templates/list": { ttlMs: 3e5, cacheScope: "private" },
|
|
161
|
+
"resources/read": { ttlMs: 6e4, cacheScope: "private" }
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
installInputFieldDescriptions(server);
|
|
165
|
+
const registrationGuard = options.allowedToolNames ? installExactToolRegistrationGuard(server, options.allowedToolNames) : void 0;
|
|
166
|
+
const httpExecutor = options.httpExecutor ?? new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess });
|
|
167
|
+
if (toolsets.has("paa")) {
|
|
168
|
+
registerPaaExtractorMcpTools(server, httpExecutor, {
|
|
169
|
+
ownerId: hashOwnerId(requiredApiKey),
|
|
170
|
+
deploymentProfile,
|
|
171
|
+
transportProfile: "stdio",
|
|
172
|
+
baseUrl,
|
|
173
|
+
localNetworkAccess,
|
|
174
|
+
taskHandleSecret: requiredApiKey
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
if (toolsets.has("serp")) {
|
|
178
|
+
registerSerpIntelligenceCaptureTools(server, httpExecutor, {
|
|
179
|
+
exposeDevelopmentDiagnostics: deploymentProfile === "development" || deploymentProfile === "test"
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
if (toolsets.has("browser-agent")) {
|
|
183
|
+
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl });
|
|
184
|
+
}
|
|
185
|
+
if (toolsets.has("scheduled-results")) {
|
|
186
|
+
registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey));
|
|
187
|
+
}
|
|
188
|
+
if (toolsets.has("memory")) {
|
|
189
|
+
registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey));
|
|
190
|
+
}
|
|
191
|
+
registrationGuard?.assertComplete();
|
|
192
|
+
return server;
|
|
193
|
+
}
|
|
194
|
+
function runMcpScraperStdio(options = {}) {
|
|
195
|
+
const forceStdio = process.argv.includes("--stdio") || process.env.MCP_SCRAPER_FORCE_STDIO === "1";
|
|
196
|
+
const interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
197
|
+
const wantsHelp = process.argv.includes("--help") || process.argv.includes("-h");
|
|
198
|
+
if (!forceStdio && (interactiveTerminal || wantsHelp)) {
|
|
199
|
+
const noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
200
|
+
process.stdout.write(renderInstallTerminal({
|
|
201
|
+
version: PACKAGE_VERSION,
|
|
202
|
+
color: !noColor,
|
|
203
|
+
apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim())
|
|
204
|
+
}));
|
|
205
|
+
process.exit(0);
|
|
206
|
+
}
|
|
207
|
+
const apiKey = (process.env.MCP_SCRAPER_API_KEY ?? process.env.MCP_SCRAPER_KEY ?? process.env.MCP_API_KEY ?? readApiKeyFile())?.trim();
|
|
208
|
+
if (!apiKey) {
|
|
209
|
+
process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
|
|
210
|
+
process.exit(1);
|
|
211
|
+
}
|
|
212
|
+
serveStdio(() => buildStdioServer(apiKey, options), {
|
|
213
|
+
legacy: "serve",
|
|
214
|
+
onerror(error) {
|
|
215
|
+
process.stderr.write(`${error.message}
|
|
216
|
+
`);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export {
|
|
222
|
+
runMcpScraperStdio
|
|
223
|
+
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// release-message.json
|
|
2
2
|
var release_message_default = {
|
|
3
|
-
message: "
|
|
3
|
+
message: "MCP Scraper now ships a fail-closed 34-tool restricted entrypoint for Thorbit while the normal package and Desktop extension remain full-surface."
|
|
4
4
|
};
|
|
5
5
|
|
|
6
6
|
// src/install-terminal.ts
|
|
@@ -7,11 +7,11 @@ import {
|
|
|
7
7
|
GmailServiceError,
|
|
8
8
|
normalizeGmailMessage,
|
|
9
9
|
parseGmailAddresses
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-E6Y3VCOJ.js";
|
|
11
11
|
import "./chunk-PJEEKOUM.js";
|
|
12
12
|
import "./chunk-T3MZISOF.js";
|
|
13
|
-
import "./chunk-CXY5WV45.js";
|
|
14
13
|
import "./chunk-6W4ADSWE.js";
|
|
14
|
+
import "./chunk-CXY5WV45.js";
|
|
15
15
|
import "./chunk-WEFPBAAG.js";
|
|
16
16
|
export {
|
|
17
17
|
GMAIL_ATTACHMENT_REF_TTL_MS,
|
|
@@ -11,6 +11,37 @@ import {
|
|
|
11
11
|
semanticCommonsEntityScores,
|
|
12
12
|
typeFromMime
|
|
13
13
|
} from "./chunk-WFQ2E4WE.js";
|
|
14
|
+
import {
|
|
15
|
+
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
16
|
+
abandonExtractSettlement,
|
|
17
|
+
cleanupExpiredSiteExtractArtifacts,
|
|
18
|
+
countSuccessfulPages,
|
|
19
|
+
createOrGetExtractJob,
|
|
20
|
+
createSiteExtractContentReader,
|
|
21
|
+
extractJobLimitInfo,
|
|
22
|
+
failExtractJob,
|
|
23
|
+
failStaleRunningExtractJob,
|
|
24
|
+
failUnfundedExtractJob,
|
|
25
|
+
finishExtractJob,
|
|
26
|
+
getExtractJob,
|
|
27
|
+
getExtractJobByIdempotencyKey,
|
|
28
|
+
getExtractedImageLinks,
|
|
29
|
+
listExtractPages,
|
|
30
|
+
listFundedPendingExtractJobs,
|
|
31
|
+
listStaleRunningExtractJobs,
|
|
32
|
+
listUnsettledExtractJobs,
|
|
33
|
+
markExtractJobDispatchAttempt,
|
|
34
|
+
readOwnedSiteExtractArtifactBuffer,
|
|
35
|
+
readOwnedSiteExtractImageArtifact,
|
|
36
|
+
recordExtractJobDispatchFailure,
|
|
37
|
+
recordExtractSettlementFailure,
|
|
38
|
+
renewSiteExtractArtifactDownload,
|
|
39
|
+
saveExtractPages,
|
|
40
|
+
setExtractJobPublicError,
|
|
41
|
+
setExtractJobTotal,
|
|
42
|
+
settleExtractJob,
|
|
43
|
+
terminalExtractJobStatus
|
|
44
|
+
} from "./chunk-D2O7V2CT.js";
|
|
14
45
|
import {
|
|
15
46
|
REQUIRED_LOCATION_STATE_CODES,
|
|
16
47
|
importHostedCensusPlacesCsv,
|
|
@@ -70,7 +101,7 @@ import {
|
|
|
70
101
|
probeNangoConnectionDirect,
|
|
71
102
|
recordNangoConnectionCost,
|
|
72
103
|
settleConnectedUsage
|
|
73
|
-
} from "./chunk-
|
|
104
|
+
} from "./chunk-E6Y3VCOJ.js";
|
|
74
105
|
import {
|
|
75
106
|
ANALYTICS_CONTENT_SORTS,
|
|
76
107
|
AnalyticsIdentityProfileError,
|
|
@@ -283,13 +314,20 @@ import {
|
|
|
283
314
|
resolveDeploymentProfile,
|
|
284
315
|
resolveLocalSourcebookSchemaType,
|
|
285
316
|
transcribeMediaUrl
|
|
286
|
-
} from "./chunk-
|
|
317
|
+
} from "./chunk-BOGRXBH2.js";
|
|
287
318
|
import {
|
|
288
319
|
auditImageUrls,
|
|
289
320
|
auditImages,
|
|
290
321
|
buildLinkReport,
|
|
291
322
|
computeIssues
|
|
292
323
|
} from "./chunk-PGJQDMC2.js";
|
|
324
|
+
import {
|
|
325
|
+
createPrivateArtifact,
|
|
326
|
+
privateArtifactOwnerId,
|
|
327
|
+
readPrivateArtifactBuffer,
|
|
328
|
+
readPrivateArtifactWindow,
|
|
329
|
+
renewPrivateArtifactDownload
|
|
330
|
+
} from "./chunk-DNM65UCK.js";
|
|
293
331
|
import {
|
|
294
332
|
cleanupExpiredConnectedDataArtifacts,
|
|
295
333
|
createConnectedDataArtifact,
|
|
@@ -301,92 +339,6 @@ import {
|
|
|
301
339
|
sanitizeHarvestResult,
|
|
302
340
|
sanitizeOutboundDiagnostics
|
|
303
341
|
} from "./chunk-7RQULQF3.js";
|
|
304
|
-
import {
|
|
305
|
-
browserServiceApiKey,
|
|
306
|
-
browserServiceProfileName,
|
|
307
|
-
browserServiceProfileSaveChanges,
|
|
308
|
-
browserServiceProxyId
|
|
309
|
-
} from "./chunk-OM7HVEJ3.js";
|
|
310
|
-
import {
|
|
311
|
-
currentCostContext,
|
|
312
|
-
recordKernelSession,
|
|
313
|
-
recordVendorUsage,
|
|
314
|
-
runWithCostContext
|
|
315
|
-
} from "./chunk-CXY5WV45.js";
|
|
316
|
-
import {
|
|
317
|
-
listWorkflowDefinitions,
|
|
318
|
-
runWorkflow,
|
|
319
|
-
runWorkflowStep,
|
|
320
|
-
slugify,
|
|
321
|
-
workflowDefinition,
|
|
322
|
-
workflowOutputBaseDir,
|
|
323
|
-
workflowStepCount,
|
|
324
|
-
workflowSupportsSteps
|
|
325
|
-
} from "./chunk-KBTUPRTF.js";
|
|
326
|
-
import {
|
|
327
|
-
csvRecords,
|
|
328
|
-
parseCsv,
|
|
329
|
-
rowsToCsv
|
|
330
|
-
} from "./chunk-RJ7JVYKU.js";
|
|
331
|
-
import {
|
|
332
|
-
DEFAULT_MAPS_PROXY_MODE,
|
|
333
|
-
DEFAULT_PROXY_MODE,
|
|
334
|
-
MapsPlaceOptionsSchema,
|
|
335
|
-
MapsSearchOptionsSchema,
|
|
336
|
-
RawMapsAboutAttributeSchema,
|
|
337
|
-
RawMapsHoursRowSchema,
|
|
338
|
-
RawMapsOverviewSchema,
|
|
339
|
-
RawMapsReviewStatsSchema
|
|
340
|
-
} from "./chunk-77LQV73B.js";
|
|
341
|
-
import {
|
|
342
|
-
extractContactEvidence
|
|
343
|
-
} from "./chunk-VXLU74YZ.js";
|
|
344
|
-
import {
|
|
345
|
-
isPrivateIpAddress,
|
|
346
|
-
loadHtmlDocument,
|
|
347
|
-
validatePublicHttpUrl
|
|
348
|
-
} from "./chunk-GGZEC22A.js";
|
|
349
|
-
import {
|
|
350
|
-
PACKAGE_VERSION
|
|
351
|
-
} from "./chunk-RDPWFAR3.js";
|
|
352
|
-
import {
|
|
353
|
-
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
354
|
-
abandonExtractSettlement,
|
|
355
|
-
cleanupExpiredSiteExtractArtifacts,
|
|
356
|
-
countSuccessfulPages,
|
|
357
|
-
createOrGetExtractJob,
|
|
358
|
-
createSiteExtractContentReader,
|
|
359
|
-
extractJobLimitInfo,
|
|
360
|
-
failExtractJob,
|
|
361
|
-
failStaleRunningExtractJob,
|
|
362
|
-
failUnfundedExtractJob,
|
|
363
|
-
finishExtractJob,
|
|
364
|
-
getExtractJob,
|
|
365
|
-
getExtractJobByIdempotencyKey,
|
|
366
|
-
getExtractedImageLinks,
|
|
367
|
-
listExtractPages,
|
|
368
|
-
listFundedPendingExtractJobs,
|
|
369
|
-
listStaleRunningExtractJobs,
|
|
370
|
-
listUnsettledExtractJobs,
|
|
371
|
-
markExtractJobDispatchAttempt,
|
|
372
|
-
readOwnedSiteExtractArtifactBuffer,
|
|
373
|
-
readOwnedSiteExtractImageArtifact,
|
|
374
|
-
recordExtractJobDispatchFailure,
|
|
375
|
-
recordExtractSettlementFailure,
|
|
376
|
-
renewSiteExtractArtifactDownload,
|
|
377
|
-
saveExtractPages,
|
|
378
|
-
setExtractJobPublicError,
|
|
379
|
-
setExtractJobTotal,
|
|
380
|
-
settleExtractJob,
|
|
381
|
-
terminalExtractJobStatus
|
|
382
|
-
} from "./chunk-D2O7V2CT.js";
|
|
383
|
-
import {
|
|
384
|
-
createPrivateArtifact,
|
|
385
|
-
privateArtifactOwnerId,
|
|
386
|
-
readPrivateArtifactBuffer,
|
|
387
|
-
readPrivateArtifactWindow,
|
|
388
|
-
renewPrivateArtifactDownload
|
|
389
|
-
} from "./chunk-DNM65UCK.js";
|
|
390
342
|
import {
|
|
391
343
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
392
344
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -434,6 +386,54 @@ import {
|
|
|
434
386
|
tierCreditsPerInterval,
|
|
435
387
|
vendorCostUsdToMc
|
|
436
388
|
} from "./chunk-6W4ADSWE.js";
|
|
389
|
+
import {
|
|
390
|
+
browserServiceApiKey,
|
|
391
|
+
browserServiceProfileName,
|
|
392
|
+
browserServiceProfileSaveChanges,
|
|
393
|
+
browserServiceProxyId
|
|
394
|
+
} from "./chunk-OM7HVEJ3.js";
|
|
395
|
+
import {
|
|
396
|
+
currentCostContext,
|
|
397
|
+
recordKernelSession,
|
|
398
|
+
recordVendorUsage,
|
|
399
|
+
runWithCostContext
|
|
400
|
+
} from "./chunk-CXY5WV45.js";
|
|
401
|
+
import {
|
|
402
|
+
listWorkflowDefinitions,
|
|
403
|
+
runWorkflow,
|
|
404
|
+
runWorkflowStep,
|
|
405
|
+
slugify,
|
|
406
|
+
workflowDefinition,
|
|
407
|
+
workflowOutputBaseDir,
|
|
408
|
+
workflowStepCount,
|
|
409
|
+
workflowSupportsSteps
|
|
410
|
+
} from "./chunk-KBTUPRTF.js";
|
|
411
|
+
import {
|
|
412
|
+
csvRecords,
|
|
413
|
+
parseCsv,
|
|
414
|
+
rowsToCsv
|
|
415
|
+
} from "./chunk-RJ7JVYKU.js";
|
|
416
|
+
import {
|
|
417
|
+
DEFAULT_MAPS_PROXY_MODE,
|
|
418
|
+
DEFAULT_PROXY_MODE,
|
|
419
|
+
MapsPlaceOptionsSchema,
|
|
420
|
+
MapsSearchOptionsSchema,
|
|
421
|
+
RawMapsAboutAttributeSchema,
|
|
422
|
+
RawMapsHoursRowSchema,
|
|
423
|
+
RawMapsOverviewSchema,
|
|
424
|
+
RawMapsReviewStatsSchema
|
|
425
|
+
} from "./chunk-77LQV73B.js";
|
|
426
|
+
import {
|
|
427
|
+
extractContactEvidence
|
|
428
|
+
} from "./chunk-VXLU74YZ.js";
|
|
429
|
+
import {
|
|
430
|
+
isPrivateIpAddress,
|
|
431
|
+
loadHtmlDocument,
|
|
432
|
+
validatePublicHttpUrl
|
|
433
|
+
} from "./chunk-GGZEC22A.js";
|
|
434
|
+
import {
|
|
435
|
+
PACKAGE_VERSION
|
|
436
|
+
} from "./chunk-JNFOBQVH.js";
|
|
437
437
|
import {
|
|
438
438
|
CaptchaError,
|
|
439
439
|
MapsSearchIncompletePageError,
|
|
@@ -9641,7 +9641,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9641
9641
|
});
|
|
9642
9642
|
}) : null;
|
|
9643
9643
|
const finalized = await step.run("finalize", async () => {
|
|
9644
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9644
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-OO74L67T.js");
|
|
9645
9645
|
const current = await getExtractJob(jobId2);
|
|
9646
9646
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9647
9647
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -42600,7 +42600,7 @@ async function getCommonsEntityLedger(idOrSlug) {
|
|
|
42600
42600
|
SELECT *
|
|
42601
42601
|
FROM commons_ledger
|
|
42602
42602
|
WHERE entity_id = ?
|
|
42603
|
-
ORDER BY created_at DESC
|
|
42603
|
+
ORDER BY created_at DESC, rowid DESC
|
|
42604
42604
|
LIMIT 100
|
|
42605
42605
|
`,
|
|
42606
42606
|
args: [entity.id]
|
|
@@ -47156,7 +47156,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
47156
47156
|
});
|
|
47157
47157
|
}
|
|
47158
47158
|
if (connection.providerConfigKey === "google-mail" && input.dataset === "emails") {
|
|
47159
|
-
const { GmailService: GmailService2 } = await import("./gmail-service-
|
|
47159
|
+
const { GmailService: GmailService2 } = await import("./gmail-service-TIU4V5JA.js");
|
|
47160
47160
|
const service = new GmailService2();
|
|
47161
47161
|
const ownerId3 = createHash29("sha256").update(identity.toLowerCase()).digest("hex").slice(0, 24);
|
|
47162
47162
|
const after = Math.floor(Date.parse(input.from) / 1e3);
|
|
@@ -62509,7 +62509,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
62509
62509
|
return c.json({ error: "Unauthorized" }, 401);
|
|
62510
62510
|
}
|
|
62511
62511
|
const startedAt = Date.now();
|
|
62512
|
-
const { drainQueue } = await import("./worker-
|
|
62512
|
+
const { drainQueue } = await import("./worker-DR6R7J46.js");
|
|
62513
62513
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
62514
62514
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
62515
62515
|
const unfinished = [];
|
|
@@ -62632,7 +62632,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
62632
62632
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
62633
62633
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
62634
62634
|
} = await import("./site-extract-repository-B47JB62U.js");
|
|
62635
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
62635
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-OO74L67T.js");
|
|
62636
62636
|
const existing = await getExtractJob2(jobId2);
|
|
62637
62637
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
62638
62638
|
if (existing.status !== "failed") {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-scraper",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.76.0",
|
|
4
4
|
"description": "MCP server for MCP Scraper web intelligence tools",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
"mcp-scraper": "dist/bin/mcp-stdio-server.js",
|
|
20
20
|
"mcp-scraper-combined": "dist/bin/mcp-stdio-server.js",
|
|
21
21
|
"browser-agent": "dist/bin/mcp-stdio-server.js",
|
|
22
|
+
"mcp-scraper-core": "dist/bin/mcp-scraper-core.js",
|
|
22
23
|
"mcp-scraper-install": "dist/bin/mcp-scraper-install.js",
|
|
23
24
|
"mcp-scraper-cli": "dist/bin/mcp-scraper-cli.js"
|
|
24
25
|
},
|
|
@@ -10,10 +10,6 @@ import {
|
|
|
10
10
|
createConnectedDataArtifact,
|
|
11
11
|
createConnectedDataBinaryArtifact
|
|
12
12
|
} from "./chunk-T3MZISOF.js";
|
|
13
|
-
import {
|
|
14
|
-
recordVendorUsage,
|
|
15
|
-
runWithCostContext
|
|
16
|
-
} from "./chunk-CXY5WV45.js";
|
|
17
13
|
import {
|
|
18
14
|
CONNECTED_COMPUTE_CREDITS_PER_SECOND,
|
|
19
15
|
CONNECTED_FUNCTION_RUN_CREDITS,
|
|
@@ -24,6 +20,10 @@ import {
|
|
|
24
20
|
MC_PER_CREDIT,
|
|
25
21
|
calculateConnectedUsageCharge
|
|
26
22
|
} from "./chunk-6W4ADSWE.js";
|
|
23
|
+
import {
|
|
24
|
+
recordVendorUsage,
|
|
25
|
+
runWithCostContext
|
|
26
|
+
} from "./chunk-CXY5WV45.js";
|
|
27
27
|
import {
|
|
28
28
|
debitMcIdempotent,
|
|
29
29
|
getDb,
|
|
@@ -5,6 +5,12 @@ import {
|
|
|
5
5
|
embedCommonsTexts,
|
|
6
6
|
publicizeExtractionFailure
|
|
7
7
|
} from "./chunk-WFQ2E4WE.js";
|
|
8
|
+
import {
|
|
9
|
+
createSiteExtractBundleArtifactStream,
|
|
10
|
+
createSiteExtractContentReader,
|
|
11
|
+
createSiteExtractImageArtifact,
|
|
12
|
+
extractJobLimitInfo
|
|
13
|
+
} from "./chunk-D2O7V2CT.js";
|
|
8
14
|
import "./chunk-OZJMVCDK.js";
|
|
9
15
|
import {
|
|
10
16
|
computeIssues,
|
|
@@ -12,15 +18,9 @@ import {
|
|
|
12
18
|
renderIssueReport,
|
|
13
19
|
renderLinkReport
|
|
14
20
|
} from "./chunk-PGJQDMC2.js";
|
|
15
|
-
import "./chunk-GGZEC22A.js";
|
|
16
|
-
import {
|
|
17
|
-
createSiteExtractBundleArtifactStream,
|
|
18
|
-
createSiteExtractContentReader,
|
|
19
|
-
createSiteExtractImageArtifact,
|
|
20
|
-
extractJobLimitInfo
|
|
21
|
-
} from "./chunk-D2O7V2CT.js";
|
|
22
21
|
import "./chunk-DNM65UCK.js";
|
|
23
22
|
import "./chunk-6W4ADSWE.js";
|
|
23
|
+
import "./chunk-GGZEC22A.js";
|
|
24
24
|
import {
|
|
25
25
|
getDb
|
|
26
26
|
} from "./chunk-WEFPBAAG.js";
|
|
@@ -14,6 +14,10 @@ import {
|
|
|
14
14
|
sanitizeAttempts,
|
|
15
15
|
sanitizeHarvestResult
|
|
16
16
|
} from "./chunk-7RQULQF3.js";
|
|
17
|
+
import {
|
|
18
|
+
MC_COSTS,
|
|
19
|
+
serpActualCostMc
|
|
20
|
+
} from "./chunk-6W4ADSWE.js";
|
|
17
21
|
import {
|
|
18
22
|
browserServiceApiKey
|
|
19
23
|
} from "./chunk-OM7HVEJ3.js";
|
|
@@ -21,10 +25,6 @@ import {
|
|
|
21
25
|
runWithCostContext
|
|
22
26
|
} from "./chunk-CXY5WV45.js";
|
|
23
27
|
import "./chunk-77LQV73B.js";
|
|
24
|
-
import {
|
|
25
|
-
MC_COSTS,
|
|
26
|
-
serpActualCostMc
|
|
27
|
-
} from "./chunk-6W4ADSWE.js";
|
|
28
28
|
import {
|
|
29
29
|
claimPendingJob,
|
|
30
30
|
completeJob,
|