mcp-scraper 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{analytics-repository-WWXVU3VW.js → analytics-repository-IKE53S3E.js} +10 -2
- package/dist/bin/api-server.cjs +573 -139
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +30 -6
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-MFNGUM4L.js → chunk-2XTYLJQK.js} +15 -1
- package/dist/chunk-2XTYLJQK.js.map +1 -0
- package/dist/{chunk-64WBUDPC.js → chunk-44LNKZTE.js} +32 -8
- package/dist/chunk-44LNKZTE.js.map +1 -0
- package/dist/{chunk-YW2LXLDU.js → chunk-5BEPOFKG.js} +160 -15
- package/dist/chunk-5BEPOFKG.js.map +1 -0
- package/dist/chunk-C7NN57WH.js +7 -0
- package/dist/chunk-C7NN57WH.js.map +1 -0
- package/dist/{chunk-MFE6RMIA.js → chunk-GOZIG6HD.js} +2 -2
- package/dist/{chunk-R66PJOZW.js → chunk-NNZXZTJ2.js} +2 -2
- package/dist/{chunk-6DPIE262.js → chunk-QHVFKFME.js} +2 -2
- package/dist/{chunk-PODASLGT.js → chunk-QXAY44SA.js} +2 -2
- package/dist/{chunk-ZEMHL36J.js → chunk-YMSZO62N.js} +2 -2
- package/dist/{db-A4YUPV4Z.js → db-UPSS5BGM.js} +2 -2
- package/dist/{extract-bundle-2HONVC5T.js → extract-bundle-RCTNANCH.js} +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{location-data-repository-YWFJGNIP.js → location-data-repository-UUSHH5M2.js} +3 -3
- package/dist/{server-JE7SLZLJ.js → server-3ND2EXYO.js} +364 -128
- package/dist/server-3ND2EXYO.js.map +1 -0
- package/dist/{site-extract-repository-STNOI5OH.js → site-extract-repository-3VQAALNR.js} +3 -3
- package/dist/{worker-4MHZS3OB.js → worker-KQN673JF.js} +5 -5
- package/package.json +1 -1
- package/dist/chunk-64WBUDPC.js.map +0 -1
- package/dist/chunk-MFNGUM4L.js.map +0 -1
- package/dist/chunk-XLEDHV3J.js +0 -7
- package/dist/chunk-XLEDHV3J.js.map +0 -1
- package/dist/chunk-YW2LXLDU.js.map +0 -1
- package/dist/server-JE7SLZLJ.js.map +0 -1
- /package/dist/{analytics-repository-WWXVU3VW.js.map → analytics-repository-IKE53S3E.js.map} +0 -0
- /package/dist/{chunk-MFE6RMIA.js.map → chunk-GOZIG6HD.js.map} +0 -0
- /package/dist/{chunk-R66PJOZW.js.map → chunk-NNZXZTJ2.js.map} +0 -0
- /package/dist/{chunk-6DPIE262.js.map → chunk-QHVFKFME.js.map} +0 -0
- /package/dist/{chunk-PODASLGT.js.map → chunk-QXAY44SA.js.map} +0 -0
- /package/dist/{chunk-ZEMHL36J.js.map → chunk-YMSZO62N.js.map} +0 -0
- /package/dist/{db-A4YUPV4Z.js.map → db-UPSS5BGM.js.map} +0 -0
- /package/dist/{extract-bundle-2HONVC5T.js.map → extract-bundle-RCTNANCH.js.map} +0 -0
- /package/dist/{location-data-repository-YWFJGNIP.js.map → location-data-repository-UUSHH5M2.js.map} +0 -0
- /package/dist/{site-extract-repository-STNOI5OH.js.map → site-extract-repository-3VQAALNR.js.map} +0 -0
- /package/dist/{worker-4MHZS3OB.js.map → worker-KQN673JF.js.map} +0 -0
|
@@ -18,10 +18,10 @@ import {
|
|
|
18
18
|
browserServiceProfileSaveChanges,
|
|
19
19
|
recordVendorUsage,
|
|
20
20
|
vendorCostUsd
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-QXAY44SA.js";
|
|
22
22
|
import {
|
|
23
23
|
PACKAGE_VERSION
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-C7NN57WH.js";
|
|
25
25
|
import {
|
|
26
26
|
MC_PER_CREDIT
|
|
27
27
|
} from "./chunk-SOEYDWJU.js";
|
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
|
|
34
34
|
// src/harvest-timeout.ts
|
|
35
35
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
36
|
+
var DEFAULT_TOOL_CLIENT_TIMEOUT_MS = 3e5;
|
|
36
37
|
var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
|
|
37
38
|
function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
|
|
38
39
|
const requested = Number.isFinite(maxQuestions) && maxQuestions > 0 ? Math.trunc(maxQuestions) : 30;
|
|
@@ -9138,6 +9139,29 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
9138
9139
|
|
|
9139
9140
|
// src/mcp/http-mcp-tool-executor.ts
|
|
9140
9141
|
import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
|
|
9142
|
+
function unclassifiedTransportFailure(path, err) {
|
|
9143
|
+
const name = err instanceof Error ? err.name : typeof err;
|
|
9144
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
9145
|
+
console.error(JSON.stringify({
|
|
9146
|
+
event: "mcp_executor_transport_failed",
|
|
9147
|
+
path,
|
|
9148
|
+
error_name: name,
|
|
9149
|
+
error_message: sanitizeVendorName(detail).slice(0, 400)
|
|
9150
|
+
}));
|
|
9151
|
+
return {
|
|
9152
|
+
content: [{
|
|
9153
|
+
type: "text",
|
|
9154
|
+
text: JSON.stringify({
|
|
9155
|
+
error: "service_unavailable",
|
|
9156
|
+
error_type: name,
|
|
9157
|
+
retryable: true,
|
|
9158
|
+
path,
|
|
9159
|
+
message: publicErrorMessage("service_unavailable")
|
|
9160
|
+
})
|
|
9161
|
+
}],
|
|
9162
|
+
isError: true
|
|
9163
|
+
};
|
|
9164
|
+
}
|
|
9141
9165
|
function youtubeVideoIdFromUrl(url) {
|
|
9142
9166
|
if (!url) return null;
|
|
9143
9167
|
try {
|
|
@@ -9238,7 +9262,7 @@ var HttpMcpToolExecutor = class {
|
|
|
9238
9262
|
const rawOverride = process.env.MCP_SCRAPER_HTTP_TIMEOUT_MS;
|
|
9239
9263
|
const parsedOverride = rawOverride === void 0 ? NaN : Number(rawOverride);
|
|
9240
9264
|
this.httpTimeoutOverrideMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : null;
|
|
9241
|
-
this.timeoutMs = this.httpTimeoutOverrideMs ??
|
|
9265
|
+
this.timeoutMs = this.httpTimeoutOverrideMs ?? DEFAULT_TOOL_CLIENT_TIMEOUT_MS;
|
|
9242
9266
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
9243
9267
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
9244
9268
|
}
|
|
@@ -9277,7 +9301,7 @@ var HttpMcpToolExecutor = class {
|
|
|
9277
9301
|
isError: true
|
|
9278
9302
|
};
|
|
9279
9303
|
}
|
|
9280
|
-
return
|
|
9304
|
+
return unclassifiedTransportFailure(path, err);
|
|
9281
9305
|
}
|
|
9282
9306
|
}
|
|
9283
9307
|
async callConnectedMutation(path, body, idempotencyKey, timeoutMs = this.timeoutMs) {
|
|
@@ -9311,8 +9335,8 @@ var HttpMcpToolExecutor = class {
|
|
|
9311
9335
|
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
|
|
9312
9336
|
}
|
|
9313
9337
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
9314
|
-
} catch {
|
|
9315
|
-
return
|
|
9338
|
+
} catch (err) {
|
|
9339
|
+
return unclassifiedTransportFailure(path, err);
|
|
9316
9340
|
}
|
|
9317
9341
|
}
|
|
9318
9342
|
async getTextArtifact(path, maxBytes, timeoutMs = this.timeoutMs) {
|
|
@@ -9349,7 +9373,7 @@ var HttpMcpToolExecutor = class {
|
|
|
9349
9373
|
}]
|
|
9350
9374
|
};
|
|
9351
9375
|
} catch (err) {
|
|
9352
|
-
return
|
|
9376
|
+
return unclassifiedTransportFailure(path, err);
|
|
9353
9377
|
}
|
|
9354
9378
|
}
|
|
9355
9379
|
harvestPaa(input) {
|
|
@@ -14467,4 +14491,4 @@ export {
|
|
|
14467
14491
|
ScheduledResultsMcpExecutor,
|
|
14468
14492
|
registerScheduledResultsMcpTools
|
|
14469
14493
|
};
|
|
14470
|
-
//# sourceMappingURL=chunk-
|
|
14494
|
+
//# sourceMappingURL=chunk-44LNKZTE.js.map
|