mcp-scraper 0.3.6 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -26
- package/dist/bin/api-server.cjs +158 -321
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -4
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +47 -328
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +7 -91
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +18 -25
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2NEVRPK3.js +7 -0
- package/dist/chunk-2NEVRPK3.js.map +1 -0
- package/dist/{chunk-ROZEDHAA.js → chunk-2O7RAK6E.js} +19 -26
- package/dist/chunk-2O7RAK6E.js.map +1 -0
- package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
- package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
- package/dist/{chunk-TK4D2BZ5.js → chunk-IWNLHCKA.js} +13 -694
- package/dist/chunk-IWNLHCKA.js.map +1 -0
- package/dist/{server-YSKBD2VX.js → server-HRVD52CX.js} +836 -114
- package/dist/server-HRVD52CX.js.map +1 -0
- package/docs/mcp-tool-craft-lint.generated.md +2 -5
- package/docs/mcp-tool-manifest.generated.json +9 -101
- package/package.json +1 -1
- package/dist/chunk-DUEW4EOO.js +0 -214
- package/dist/chunk-DUEW4EOO.js.map +0 -1
- package/dist/chunk-IHXHQL53.js +0 -7
- package/dist/chunk-IHXHQL53.js.map +0 -1
- package/dist/chunk-NXRWFOEZ.js +0 -816
- package/dist/chunk-NXRWFOEZ.js.map +0 -1
- package/dist/chunk-ROZEDHAA.js.map +0 -1
- package/dist/chunk-TK4D2BZ5.js.map +0 -1
- package/dist/server-YSKBD2VX.js.map +0 -1
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
import {
|
|
3
3
|
HttpMcpToolExecutor,
|
|
4
4
|
buildPaaExtractorMcpServer
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-2O7RAK6E.js";
|
|
6
6
|
import "../chunk-M2S27J6Z.js";
|
|
7
7
|
import "../chunk-LFATOGDF.js";
|
|
8
|
-
import "../chunk-
|
|
8
|
+
import "../chunk-2NEVRPK3.js";
|
|
9
9
|
|
|
10
10
|
// bin/mcp-stdio-server.ts
|
|
11
11
|
import { readFileSync } from "fs";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.7'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "./chunk-LFATOGDF.js";
|
|
8
8
|
import {
|
|
9
9
|
PACKAGE_VERSION
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-2NEVRPK3.js";
|
|
11
11
|
|
|
12
12
|
// src/harvest-timeout.ts
|
|
13
13
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
@@ -1482,16 +1482,13 @@ ${chunkRows}` : "",
|
|
|
1482
1482
|
}
|
|
1483
1483
|
function structuredInstagramBrowser(raw) {
|
|
1484
1484
|
const browser = raw && typeof raw === "object" ? raw : {};
|
|
1485
|
-
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
1486
|
-
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
1487
|
-
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
1488
1485
|
return {
|
|
1489
|
-
mode,
|
|
1490
|
-
requestedMode,
|
|
1486
|
+
mode: "hosted",
|
|
1487
|
+
requestedMode: "hosted",
|
|
1491
1488
|
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
1492
|
-
profileSource,
|
|
1493
|
-
profileDirConfigured:
|
|
1494
|
-
executablePathConfigured:
|
|
1489
|
+
profileSource: "hosted",
|
|
1490
|
+
profileDirConfigured: false,
|
|
1491
|
+
executablePathConfigured: false
|
|
1495
1492
|
};
|
|
1496
1493
|
}
|
|
1497
1494
|
function structuredInstagramPagination(raw, input) {
|
|
@@ -1533,7 +1530,7 @@ function formatInstagramProfileContent(raw, input) {
|
|
|
1533
1530
|
const itemRows = items.slice(0, 100).map(
|
|
1534
1531
|
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
1535
1532
|
).join("\n");
|
|
1536
|
-
const browserLabel =
|
|
1533
|
+
const browserLabel = "hosted browser";
|
|
1537
1534
|
const full = [
|
|
1538
1535
|
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
1539
1536
|
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
@@ -1608,7 +1605,7 @@ function formatInstagramMediaDownload(raw, input) {
|
|
|
1608
1605
|
const transcriptText = transcript?.text ?? "";
|
|
1609
1606
|
const chunks = transcript?.chunks ?? [];
|
|
1610
1607
|
const browser = structuredInstagramBrowser(d.browser);
|
|
1611
|
-
const browserLabel =
|
|
1608
|
+
const browserLabel = "hosted browser";
|
|
1612
1609
|
const downloadRows = downloads.map((download, i) => {
|
|
1613
1610
|
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
1614
1611
|
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
@@ -1841,14 +1838,12 @@ var FacebookVideoTranscribeInputSchema = {
|
|
|
1841
1838
|
quality: z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
1842
1839
|
};
|
|
1843
1840
|
var InstagramProfileContentInputSchema = {
|
|
1844
|
-
handle: z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url.
|
|
1841
|
+
handle: z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
|
|
1845
1842
|
url: z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
1846
|
-
maxItems: z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values
|
|
1847
|
-
maxScrolls: z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase
|
|
1843
|
+
maxItems: z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
|
|
1844
|
+
maxScrolls: z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
|
|
1848
1845
|
scrollDelayMs: z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
1849
|
-
stableScrollLimit: z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
1850
|
-
browserMode: z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
1851
|
-
browserProfile: z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
1846
|
+
stableScrollLimit: z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
1852
1847
|
};
|
|
1853
1848
|
var InstagramMediaDownloadInputSchema = {
|
|
1854
1849
|
url: z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
@@ -1856,9 +1851,7 @@ var InstagramMediaDownloadInputSchema = {
|
|
|
1856
1851
|
downloadMedia: z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
1857
1852
|
downloadAllTracks: z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
1858
1853
|
includeTranscript: z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
1859
|
-
mux: z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
1860
|
-
browserMode: z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
1861
|
-
browserProfile: z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
1854
|
+
mux: z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
1862
1855
|
};
|
|
1863
1856
|
var MapsPlaceIntelInputSchema = {
|
|
1864
1857
|
businessName: z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
@@ -2257,10 +2250,10 @@ var TranscriptChunkOutput = z.object({
|
|
|
2257
2250
|
text: z.string()
|
|
2258
2251
|
});
|
|
2259
2252
|
var InstagramBrowserOutput = z.object({
|
|
2260
|
-
mode: z.
|
|
2261
|
-
requestedMode: z.
|
|
2253
|
+
mode: z.literal("hosted"),
|
|
2254
|
+
requestedMode: z.literal("hosted"),
|
|
2262
2255
|
profileName: NullableString,
|
|
2263
|
-
profileSource: z.
|
|
2256
|
+
profileSource: z.literal("hosted"),
|
|
2264
2257
|
profileDirConfigured: z.boolean(),
|
|
2265
2258
|
executablePathConfigured: z.boolean()
|
|
2266
2259
|
});
|
|
@@ -3002,14 +2995,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
3002
2995
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
3003
2996
|
server.registerTool("instagram_profile_content", {
|
|
3004
2997
|
title: "Instagram Profile Content Discovery",
|
|
3005
|
-
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser
|
|
2998
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser details, pagination attempts, stop reason, and limitations."),
|
|
3006
2999
|
inputSchema: InstagramProfileContentInputSchema,
|
|
3007
3000
|
outputSchema: InstagramProfileContentOutputSchema,
|
|
3008
3001
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
3009
3002
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
3010
3003
|
server.registerTool("instagram_media_download", {
|
|
3011
3004
|
title: "Instagram Post/Reel Media Download",
|
|
3012
|
-
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available.
|
|
3005
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available."),
|
|
3013
3006
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
3014
3007
|
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
3015
3008
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
@@ -3342,4 +3335,4 @@ export {
|
|
|
3342
3335
|
registerPaaExtractorMcpTools,
|
|
3343
3336
|
HttpMcpToolExecutor
|
|
3344
3337
|
};
|
|
3345
|
-
//# sourceMappingURL=chunk-
|
|
3338
|
+
//# sourceMappingURL=chunk-2O7RAK6E.js.map
|