mcp-scraper 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +86 -22
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +198 -27
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +3 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +254 -56
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +38 -11
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +21 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-3YGKXXUG.js +133 -0
- package/dist/chunk-3YGKXXUG.js.map +1 -0
- package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} +22 -2
- package/dist/chunk-4OPKIDON.js.map +1 -0
- package/dist/{chunk-VMH7SRWY.js → chunk-7R7VBQRV.js} +39 -12
- package/dist/chunk-7R7VBQRV.js.map +1 -0
- package/dist/{chunk-L27GJQV7.js → chunk-BW3DGFNQ.js} +28 -5
- package/dist/chunk-BW3DGFNQ.js.map +1 -0
- package/dist/chunk-FB5ZJGFN.js +7 -0
- package/dist/chunk-FB5ZJGFN.js.map +1 -0
- package/dist/index.cjs +21 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-3SA5Q4OF.js → server-CUPDJLWM.js} +42 -151
- package/dist/server-CUPDJLWM.js.map +1 -0
- package/dist/{worker-56IXWOQU.js → worker-FG7ZWEGA.js} +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +1 -1
- package/docs/mcp-tool-manifest.generated.json +1 -1
- package/package.json +1 -1
- package/dist/chunk-L27GJQV7.js.map +0 -1
- package/dist/chunk-RRE7WVHQ.js +0 -7
- package/dist/chunk-RRE7WVHQ.js.map +0 -1
- package/dist/chunk-UWSG3C5J.js.map +0 -1
- package/dist/chunk-VMH7SRWY.js.map +0 -1
- package/dist/server-3SA5Q4OF.js.map +0 -1
- /package/dist/{worker-56IXWOQU.js.map → worker-FG7ZWEGA.js.map} +0 -0
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "./chunk-LFATOGDF.js";
|
|
8
8
|
import {
|
|
9
9
|
PACKAGE_VERSION
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-FB5ZJGFN.js";
|
|
11
11
|
|
|
12
12
|
// src/harvest-timeout.ts
|
|
13
13
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
@@ -1530,7 +1530,7 @@ function formatInstagramProfileContent(raw, input) {
|
|
|
1530
1530
|
const itemRows = items.slice(0, 100).map(
|
|
1531
1531
|
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
1532
1532
|
).join("\n");
|
|
1533
|
-
const browserLabel = "hosted browser";
|
|
1533
|
+
const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
|
|
1534
1534
|
const full = [
|
|
1535
1535
|
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
1536
1536
|
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
@@ -1605,7 +1605,7 @@ function formatInstagramMediaDownload(raw, input) {
|
|
|
1605
1605
|
const transcriptText = transcript?.text ?? "";
|
|
1606
1606
|
const chunks = transcript?.chunks ?? [];
|
|
1607
1607
|
const browser = structuredInstagramBrowser(d.browser);
|
|
1608
|
-
const browserLabel = "hosted browser";
|
|
1608
|
+
const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
|
|
1609
1609
|
const downloadRows = downloads.map((download, i) => {
|
|
1610
1610
|
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
1611
1611
|
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
@@ -1840,13 +1840,17 @@ var FacebookVideoTranscribeInputSchema = {
|
|
|
1840
1840
|
var InstagramProfileContentInputSchema = {
|
|
1841
1841
|
handle: z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
|
|
1842
1842
|
url: z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
1843
|
-
|
|
1844
|
-
|
|
1843
|
+
profile: z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
|
|
1844
|
+
saveProfileChanges: z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
|
|
1845
|
+
maxItems: z.number().int().min(1).max(2e3).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 2000. Use higher values only when the user asks for a fuller archive."),
|
|
1846
|
+
maxScrolls: z.number().int().min(0).max(250).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 250. Increase for long profiles when Instagram continues loading more grid links."),
|
|
1845
1847
|
scrollDelayMs: z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
1846
1848
|
stableScrollLimit: z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
1847
1849
|
};
|
|
1848
1850
|
var InstagramMediaDownloadInputSchema = {
|
|
1849
1851
|
url: z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
1852
|
+
profile: z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
|
|
1853
|
+
saveProfileChanges: z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
|
|
1850
1854
|
mediaTypes: z.array(z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
|
|
1851
1855
|
downloadMedia: z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
1852
1856
|
downloadAllTracks: z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
@@ -2258,8 +2262,8 @@ var InstagramBrowserOutput = z.object({
|
|
|
2258
2262
|
executablePathConfigured: z.boolean()
|
|
2259
2263
|
});
|
|
2260
2264
|
var InstagramPaginationOutput = z.object({
|
|
2261
|
-
maxItems: z.number().int().min(1).max(
|
|
2262
|
-
maxScrolls: z.number().int().min(0).max(
|
|
2265
|
+
maxItems: z.number().int().min(1).max(2e3),
|
|
2266
|
+
maxScrolls: z.number().int().min(0).max(250),
|
|
2263
2267
|
attemptedScrolls: z.number().int().min(0),
|
|
2264
2268
|
stableScrolls: z.number().int().min(0),
|
|
2265
2269
|
stableScrollLimit: z.number().int().min(1).max(10),
|
|
@@ -3114,6 +3118,29 @@ function youtubeVideoIdFromUrl(url) {
|
|
|
3114
3118
|
}
|
|
3115
3119
|
return null;
|
|
3116
3120
|
}
|
|
3121
|
+
async function readResponseData(res) {
|
|
3122
|
+
const text = await res.text();
|
|
3123
|
+
if (!text.trim()) return null;
|
|
3124
|
+
try {
|
|
3125
|
+
return JSON.parse(text);
|
|
3126
|
+
} catch {
|
|
3127
|
+
return text;
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
function httpErrorPayload(path, res, data) {
|
|
3131
|
+
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
|
|
3132
|
+
const bodyMessage = objectData ? objectData.message ?? objectData.error ?? objectData.error_code : data;
|
|
3133
|
+
return {
|
|
3134
|
+
...objectData ?? { body: data },
|
|
3135
|
+
error: objectData?.error ?? objectData?.error_code ?? "mcp_http_error",
|
|
3136
|
+
error_type: "http",
|
|
3137
|
+
retryable: res.status === 429 || res.status >= 500,
|
|
3138
|
+
status: res.status,
|
|
3139
|
+
statusText: res.statusText,
|
|
3140
|
+
path,
|
|
3141
|
+
message: typeof bodyMessage === "string" && bodyMessage.trim() ? bodyMessage : `MCP Scraper HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ""} for ${path}`
|
|
3142
|
+
};
|
|
3143
|
+
}
|
|
3117
3144
|
var HttpMcpToolExecutor = class {
|
|
3118
3145
|
baseUrl;
|
|
3119
3146
|
apiKey;
|
|
@@ -3141,9 +3168,9 @@ var HttpMcpToolExecutor = class {
|
|
|
3141
3168
|
body: JSON.stringify(body),
|
|
3142
3169
|
signal: AbortSignal.timeout(timeoutMs)
|
|
3143
3170
|
});
|
|
3144
|
-
const data = await res
|
|
3171
|
+
const data = await readResponseData(res);
|
|
3145
3172
|
if (!res.ok) {
|
|
3146
|
-
return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
|
|
3173
|
+
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
|
|
3147
3174
|
}
|
|
3148
3175
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
3149
3176
|
} catch (err) {
|
|
@@ -3176,9 +3203,9 @@ var HttpMcpToolExecutor = class {
|
|
|
3176
3203
|
},
|
|
3177
3204
|
signal: AbortSignal.timeout(timeoutMs)
|
|
3178
3205
|
});
|
|
3179
|
-
const data = await res
|
|
3206
|
+
const data = await readResponseData(res);
|
|
3180
3207
|
if (!res.ok) {
|
|
3181
|
-
return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
|
|
3208
|
+
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
|
|
3182
3209
|
}
|
|
3183
3210
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
3184
3211
|
} catch (err) {
|
|
@@ -3335,4 +3362,4 @@ export {
|
|
|
3335
3362
|
registerPaaExtractorMcpTools,
|
|
3336
3363
|
HttpMcpToolExecutor
|
|
3337
3364
|
};
|
|
3338
|
-
//# sourceMappingURL=chunk-
|
|
3365
|
+
//# sourceMappingURL=chunk-7R7VBQRV.js.map
|