mcp-scraper 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -5
- package/dist/bin/api-server.cjs +1072 -349
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +4 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +659 -210
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +108 -9
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -2
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/chunk-DUEW4EOO.js +214 -0
- package/dist/chunk-DUEW4EOO.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/chunk-HS6OKELW.js +1786 -0
- package/dist/chunk-HS6OKELW.js.map +1 -0
- package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
- package/dist/chunk-IPW4LFOT.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/chunk-WN7PBKMV.js +28 -0
- package/dist/chunk-WN7PBKMV.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
- package/dist/server-24XIQ6W6.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
- package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1534 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js +0 -793
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// src/lib/chrome-profiles.ts
|
|
2
|
+
import { readFile } from "fs/promises";
|
|
3
|
+
import { homedir } from "os";
|
|
4
|
+
import { join } from "path";
|
|
5
|
+
function defaultChromeUserDataDir() {
|
|
6
|
+
if (process.platform === "darwin") return join(homedir(), "Library", "Application Support", "Google", "Chrome");
|
|
7
|
+
if (process.platform === "win32") {
|
|
8
|
+
return join(process.env.LOCALAPPDATA ?? join(homedir(), "AppData", "Local"), "Google", "Chrome", "User Data");
|
|
9
|
+
}
|
|
10
|
+
return join(process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config"), "google-chrome");
|
|
11
|
+
}
|
|
12
|
+
function recommendedKernelProfileName(value) {
|
|
13
|
+
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
14
|
+
}
|
|
15
|
+
async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
|
|
16
|
+
const raw = await readFile(join(userDataDir, "Local State"), "utf8");
|
|
17
|
+
const localState = JSON.parse(raw);
|
|
18
|
+
const wanted = emailFilter?.trim().toLowerCase();
|
|
19
|
+
return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
|
|
20
|
+
const email = (info.user_name ?? "").trim();
|
|
21
|
+
const displayName = (info.name ?? info.shortcut_name ?? "").trim();
|
|
22
|
+
const labelSource = email || displayName || directory;
|
|
23
|
+
return {
|
|
24
|
+
email,
|
|
25
|
+
displayName,
|
|
26
|
+
chromeProfileDirectory: directory,
|
|
27
|
+
chromeProfilePath: join(userDataDir, directory),
|
|
28
|
+
recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
|
|
29
|
+
};
|
|
30
|
+
}).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// src/lib/local-browser-profiles.ts
|
|
34
|
+
import { existsSync } from "fs";
|
|
35
|
+
import { cp, mkdir, readFile as readFile2, rm, stat, writeFile } from "fs/promises";
|
|
36
|
+
import { homedir as homedir2 } from "os";
|
|
37
|
+
import { join as join2, relative, sep } from "path";
|
|
38
|
+
var EXCLUDED_PROFILE_NAMES = [
|
|
39
|
+
"Cache",
|
|
40
|
+
"Code Cache",
|
|
41
|
+
"Crashpad",
|
|
42
|
+
"DawnCache",
|
|
43
|
+
"DawnGraphiteCache",
|
|
44
|
+
"DawnWebGPUCache",
|
|
45
|
+
"GPUCache",
|
|
46
|
+
"GrShaderCache",
|
|
47
|
+
"ShaderCache",
|
|
48
|
+
"Shared Dictionary"
|
|
49
|
+
];
|
|
50
|
+
function defaultImportedBrowserProfilesDir() {
|
|
51
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || join2(homedir2(), ".mcp-scraper", "browser-profiles");
|
|
52
|
+
}
|
|
53
|
+
function defaultChromeExecutablePath() {
|
|
54
|
+
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
55
|
+
if (envPath) return envPath;
|
|
56
|
+
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
57
|
+
join2(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
58
|
+
join2(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
59
|
+
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
60
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
61
|
+
}
|
|
62
|
+
function localBrowserModeEnabled() {
|
|
63
|
+
return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
|
|
64
|
+
}
|
|
65
|
+
function importedProfileName(value) {
|
|
66
|
+
return recommendedKernelProfileName(value);
|
|
67
|
+
}
|
|
68
|
+
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
69
|
+
return join2(outputDir, importedProfileName(name));
|
|
70
|
+
}
|
|
71
|
+
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
72
|
+
return join2(importedProfileRoot(name, outputDir), "manifest.json");
|
|
73
|
+
}
|
|
74
|
+
function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
75
|
+
return join2(importedProfileRoot(name, outputDir), "user-data");
|
|
76
|
+
}
|
|
77
|
+
async function readExistingManifest(path) {
|
|
78
|
+
try {
|
|
79
|
+
return JSON.parse(await readFile2(path, "utf8"));
|
|
80
|
+
} catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function isExcludedProfilePath(sourceRoot, sourcePath) {
|
|
85
|
+
const rel = relative(sourceRoot, sourcePath);
|
|
86
|
+
if (!rel) return false;
|
|
87
|
+
const parts = rel.split(sep).filter(Boolean);
|
|
88
|
+
const last = parts[parts.length - 1] ?? "";
|
|
89
|
+
if (last.startsWith("Singleton")) return true;
|
|
90
|
+
if (last === "LOCK" || last.endsWith(".lock")) return true;
|
|
91
|
+
if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
|
|
92
|
+
if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
|
|
96
|
+
await cp(sourceProfilePath, destinationProfilePath, {
|
|
97
|
+
recursive: true,
|
|
98
|
+
force: true,
|
|
99
|
+
errorOnExist: false,
|
|
100
|
+
dereference: false,
|
|
101
|
+
filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function selectProfile(profiles, options) {
|
|
105
|
+
const requestedDirectory = options.chromeProfileDirectory?.trim();
|
|
106
|
+
const requestedEmail = options.email?.trim().toLowerCase();
|
|
107
|
+
const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
|
|
108
|
+
if (exactDirectory) return exactDirectory;
|
|
109
|
+
const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
|
|
110
|
+
if (exactEmail) return exactEmail;
|
|
111
|
+
const first = profiles[0];
|
|
112
|
+
if (first) return first;
|
|
113
|
+
const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
|
|
114
|
+
throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
|
|
115
|
+
}
|
|
116
|
+
async function ensureProfilePathExists(profile) {
|
|
117
|
+
const details = await stat(profile.chromeProfilePath).catch(() => null);
|
|
118
|
+
if (!details?.isDirectory()) {
|
|
119
|
+
throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
|
|
123
|
+
const raw = await readFile2(sourceLocalStatePath, "utf8");
|
|
124
|
+
const localState = JSON.parse(raw);
|
|
125
|
+
const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
|
|
126
|
+
localState.profile = {
|
|
127
|
+
...localState.profile ?? {},
|
|
128
|
+
info_cache: { Default: selectedInfo },
|
|
129
|
+
last_used: "Default",
|
|
130
|
+
last_active_profiles: ["Default"]
|
|
131
|
+
};
|
|
132
|
+
await writeFile(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
|
|
133
|
+
`);
|
|
134
|
+
}
|
|
135
|
+
async function importChromeProfile(options = {}) {
|
|
136
|
+
const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
|
|
137
|
+
const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
|
|
138
|
+
const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
|
|
139
|
+
await ensureProfilePathExists(sourceProfile);
|
|
140
|
+
const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
|
|
141
|
+
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
142
|
+
const profileRoot = importedProfileRoot(name, outputDir);
|
|
143
|
+
const manifestPath = importedProfileManifestPath(name, outputDir);
|
|
144
|
+
const userDataDir = importedProfileUserDataDir(name, outputDir);
|
|
145
|
+
const existingManifest = await readExistingManifest(manifestPath);
|
|
146
|
+
const existing = existsSync(profileRoot);
|
|
147
|
+
if (existing && !options.overwrite) {
|
|
148
|
+
throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
|
|
149
|
+
}
|
|
150
|
+
if (existing) await rm(profileRoot, { recursive: true, force: true });
|
|
151
|
+
await mkdir(join2(userDataDir, "Default"), { recursive: true });
|
|
152
|
+
await writeNormalizedLocalState(
|
|
153
|
+
join2(sourceUserDataDir, "Local State"),
|
|
154
|
+
join2(userDataDir, "Local State"),
|
|
155
|
+
sourceProfile.chromeProfileDirectory
|
|
156
|
+
);
|
|
157
|
+
await copyProfileTree(sourceProfile.chromeProfilePath, join2(userDataDir, "Default"));
|
|
158
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
159
|
+
const manifest = {
|
|
160
|
+
layoutVersion: 1,
|
|
161
|
+
name,
|
|
162
|
+
profileRoot,
|
|
163
|
+
userDataDir,
|
|
164
|
+
browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
|
|
165
|
+
sourceUserDataDir,
|
|
166
|
+
sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
|
|
167
|
+
sourceProfilePath: sourceProfile.chromeProfilePath,
|
|
168
|
+
email: sourceProfile.email,
|
|
169
|
+
displayName: sourceProfile.displayName,
|
|
170
|
+
createdAt: existingManifest?.createdAt ?? now,
|
|
171
|
+
updatedAt: now,
|
|
172
|
+
copyMode: "chrome-profile-clone"
|
|
173
|
+
};
|
|
174
|
+
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
175
|
+
`);
|
|
176
|
+
return {
|
|
177
|
+
profile: manifest,
|
|
178
|
+
sourceProfile,
|
|
179
|
+
excludedNames: EXCLUDED_PROFILE_NAMES
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
183
|
+
const profileName = importedProfileName(name);
|
|
184
|
+
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
185
|
+
const manifest = await readExistingManifest(manifestPath);
|
|
186
|
+
if (!manifest) {
|
|
187
|
+
throw new Error(
|
|
188
|
+
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
return manifest;
|
|
192
|
+
}
|
|
193
|
+
async function syncImportedChromeProfile(name, options = {}) {
|
|
194
|
+
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
195
|
+
const manifest = await loadImportedBrowserProfile(name, outputDir);
|
|
196
|
+
return importChromeProfile({
|
|
197
|
+
name: manifest.name,
|
|
198
|
+
sourceUserDataDir: manifest.sourceUserDataDir,
|
|
199
|
+
chromeProfileDirectory: manifest.sourceProfileDirectory,
|
|
200
|
+
outputDir,
|
|
201
|
+
overwrite: true,
|
|
202
|
+
browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export {
|
|
207
|
+
listLocalChromeProfiles,
|
|
208
|
+
defaultChromeExecutablePath,
|
|
209
|
+
localBrowserModeEnabled,
|
|
210
|
+
importChromeProfile,
|
|
211
|
+
loadImportedBrowserProfile,
|
|
212
|
+
syncImportedChromeProfile
|
|
213
|
+
};
|
|
214
|
+
//# sourceMappingURL=chunk-DUEW4EOO.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/chrome-profiles.ts","../src/lib/local-browser-profiles.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\n\nexport interface LocalChromeProfile {\n email: string\n displayName: string\n chromeProfileDirectory: string\n chromeProfilePath: string\n recommendedKernelProfileName: string\n}\n\nexport function defaultChromeUserDataDir(): string {\n if (process.platform === 'darwin') return join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome')\n if (process.platform === 'win32') {\n return join(process.env.LOCALAPPDATA ?? join(homedir(), 'AppData', 'Local'), 'Google', 'Chrome', 'User Data')\n }\n return join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'google-chrome')\n}\n\nexport function recommendedKernelProfileName(value: string): string {\n return value\n .trim()\n .toLowerCase()\n .replace(/@/g, '-')\n .replace(/[^a-z0-9]+/g, '-')\n .replace(/^-+|-+$/g, '')\n .slice(0, 80) || 'browser-profile'\n}\n\nexport async function listLocalChromeProfiles(\n userDataDir = defaultChromeUserDataDir(),\n emailFilter?: string,\n): Promise<LocalChromeProfile[]> {\n const raw = await readFile(join(userDataDir, 'Local State'), 'utf8')\n const localState = JSON.parse(raw) as {\n profile?: {\n info_cache?: Record<string, {\n name?: string\n shortcut_name?: string\n user_name?: string\n gaia_name?: string\n }>\n }\n }\n const wanted = emailFilter?.trim().toLowerCase()\n return Object.entries(localState.profile?.info_cache ?? {})\n .map(([directory, info]) => {\n const email = (info.user_name ?? '').trim()\n const displayName = (info.name ?? info.shortcut_name ?? '').trim()\n const labelSource = email || displayName || directory\n return {\n email,\n displayName,\n chromeProfileDirectory: directory,\n chromeProfilePath: join(userDataDir, directory),\n recommendedKernelProfileName: recommendedKernelProfileName(labelSource),\n }\n })\n .filter(row => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted))\n}\n","import { existsSync } from 'node:fs'\nimport { cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'\nimport { homedir } from 'node:os'\nimport { join, relative, sep } from 'node:path'\nimport {\n defaultChromeUserDataDir,\n listLocalChromeProfiles,\n recommendedKernelProfileName,\n type LocalChromeProfile,\n} from './chrome-profiles.js'\n\nexport interface ImportedBrowserProfileManifest {\n layoutVersion: 1\n name: string\n profileRoot: string\n userDataDir: string\n browserExecutablePath: string | null\n sourceUserDataDir: string\n sourceProfileDirectory: string\n sourceProfilePath: string\n email: string\n displayName: string\n createdAt: string\n updatedAt: string\n copyMode: 'chrome-profile-clone'\n}\n\nexport interface ImportChromeProfileOptions {\n email?: string\n name?: string\n chromeProfileDirectory?: string\n sourceUserDataDir?: string\n outputDir?: string\n overwrite?: boolean\n browserExecutablePath?: string\n}\n\nexport interface ImportChromeProfileResult {\n profile: ImportedBrowserProfileManifest\n sourceProfile: LocalChromeProfile\n excludedNames: string[]\n}\n\nconst EXCLUDED_PROFILE_NAMES = [\n 'Cache',\n 'Code Cache',\n 'Crashpad',\n 'DawnCache',\n 'DawnGraphiteCache',\n 'DawnWebGPUCache',\n 'GPUCache',\n 'GrShaderCache',\n 'ShaderCache',\n 'Shared Dictionary',\n]\n\nexport function defaultImportedBrowserProfilesDir(): string {\n return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim()\n || join(homedir(), '.mcp-scraper', 'browser-profiles')\n}\n\nexport function defaultChromeExecutablePath(): string | undefined {\n const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim()\n if (envPath) return envPath\n\n const candidates = process.platform === 'darwin'\n ? ['/Applications/Google Chrome.app/Contents/MacOS/Google Chrome']\n : process.platform === 'win32'\n ? [\n join(process.env.PROGRAMFILES ?? 'C:\\\\Program Files', 'Google', 'Chrome', 'Application', 'chrome.exe'),\n join(process.env['PROGRAMFILES(X86)'] ?? 'C:\\\\Program Files (x86)', 'Google', 'Chrome', 'Application', 'chrome.exe'),\n ]\n : ['/usr/bin/google-chrome', '/usr/bin/google-chrome-stable', '/usr/bin/chromium-browser', '/usr/bin/chromium']\n\n return candidates.find(candidate => existsSync(candidate))\n}\n\nexport function localBrowserModeEnabled(): boolean {\n return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === 'local'\n}\n\nexport function importedProfileName(value: string): string {\n return recommendedKernelProfileName(value)\n}\n\nexport function importedProfileRoot(name: string, outputDir = defaultImportedBrowserProfilesDir()): string {\n return join(outputDir, importedProfileName(name))\n}\n\nexport function importedProfileManifestPath(name: string, outputDir = defaultImportedBrowserProfilesDir()): string {\n return join(importedProfileRoot(name, outputDir), 'manifest.json')\n}\n\nexport function importedProfileUserDataDir(name: string, outputDir = defaultImportedBrowserProfilesDir()): string {\n return join(importedProfileRoot(name, outputDir), 'user-data')\n}\n\nasync function readExistingManifest(path: string): Promise<ImportedBrowserProfileManifest | null> {\n try {\n return JSON.parse(await readFile(path, 'utf8')) as ImportedBrowserProfileManifest\n } catch {\n return null\n }\n}\n\nfunction isExcludedProfilePath(sourceRoot: string, sourcePath: string): boolean {\n const rel = relative(sourceRoot, sourcePath)\n if (!rel) return false\n const parts = rel.split(sep).filter(Boolean)\n const last = parts[parts.length - 1] ?? ''\n if (last.startsWith('Singleton')) return true\n if (last === 'LOCK' || last.endsWith('.lock')) return true\n if (parts.some(part => EXCLUDED_PROFILE_NAMES.includes(part))) return true\n if (parts.includes('Service Worker') && parts.includes('CacheStorage')) return true\n return false\n}\n\nasync function copyProfileTree(sourceProfilePath: string, destinationProfilePath: string): Promise<void> {\n await cp(sourceProfilePath, destinationProfilePath, {\n recursive: true,\n force: true,\n errorOnExist: false,\n dereference: false,\n filter: source => !isExcludedProfilePath(sourceProfilePath, source),\n })\n}\n\nfunction selectProfile(profiles: LocalChromeProfile[], options: ImportChromeProfileOptions): LocalChromeProfile {\n const requestedDirectory = options.chromeProfileDirectory?.trim()\n const requestedEmail = options.email?.trim().toLowerCase()\n const exactDirectory = requestedDirectory\n ? profiles.find(profile => profile.chromeProfileDirectory === requestedDirectory)\n : undefined\n if (exactDirectory) return exactDirectory\n\n const exactEmail = requestedEmail\n ? profiles.find(profile => profile.email.toLowerCase() === requestedEmail)\n : undefined\n if (exactEmail) return exactEmail\n\n const first = profiles[0]\n if (first) return first\n\n const hint = requestedDirectory\n ? `Chrome profile directory \"${requestedDirectory}\"`\n : requestedEmail\n ? `Chrome account \"${requestedEmail}\"`\n : 'Chrome profile'\n throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`)\n}\n\nasync function ensureProfilePathExists(profile: LocalChromeProfile): Promise<void> {\n const details = await stat(profile.chromeProfilePath).catch(() => null)\n if (!details?.isDirectory()) {\n throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`)\n }\n}\n\nasync function writeNormalizedLocalState(\n sourceLocalStatePath: string,\n destinationLocalStatePath: string,\n sourceProfileDirectory: string,\n): Promise<void> {\n const raw = await readFile(sourceLocalStatePath, 'utf8')\n const localState = JSON.parse(raw) as {\n profile?: {\n info_cache?: Record<string, unknown>\n last_used?: string\n last_active_profiles?: string[]\n }\n }\n const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory]\n ?? localState.profile?.info_cache?.Default\n ?? {}\n localState.profile = {\n ...(localState.profile ?? {}),\n info_cache: { Default: selectedInfo },\n last_used: 'Default',\n last_active_profiles: ['Default'],\n }\n await writeFile(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}\\n`)\n}\n\nexport async function importChromeProfile(options: ImportChromeProfileOptions = {}): Promise<ImportChromeProfileResult> {\n const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir()\n const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email)\n const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir })\n await ensureProfilePathExists(sourceProfile)\n\n const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName)\n const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir()\n const profileRoot = importedProfileRoot(name, outputDir)\n const manifestPath = importedProfileManifestPath(name, outputDir)\n const userDataDir = importedProfileUserDataDir(name, outputDir)\n const existingManifest = await readExistingManifest(manifestPath)\n const existing = existsSync(profileRoot)\n\n if (existing && !options.overwrite) {\n throw new Error(`Imported browser profile \"${name}\" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`)\n }\n\n if (existing) await rm(profileRoot, { recursive: true, force: true })\n await mkdir(join(userDataDir, 'Default'), { recursive: true })\n await writeNormalizedLocalState(\n join(sourceUserDataDir, 'Local State'),\n join(userDataDir, 'Local State'),\n sourceProfile.chromeProfileDirectory,\n )\n await copyProfileTree(sourceProfile.chromeProfilePath, join(userDataDir, 'Default'))\n\n const now = new Date().toISOString()\n const manifest: ImportedBrowserProfileManifest = {\n layoutVersion: 1,\n name,\n profileRoot,\n userDataDir,\n browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,\n sourceUserDataDir,\n sourceProfileDirectory: sourceProfile.chromeProfileDirectory,\n sourceProfilePath: sourceProfile.chromeProfilePath,\n email: sourceProfile.email,\n displayName: sourceProfile.displayName,\n createdAt: existingManifest?.createdAt ?? now,\n updatedAt: now,\n copyMode: 'chrome-profile-clone',\n }\n await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\\n`)\n\n return {\n profile: manifest,\n sourceProfile,\n excludedNames: EXCLUDED_PROFILE_NAMES,\n }\n}\n\nexport async function loadImportedBrowserProfile(\n name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || 'default',\n outputDir = defaultImportedBrowserProfilesDir(),\n): Promise<ImportedBrowserProfileManifest> {\n const profileName = importedProfileName(name)\n const manifestPath = importedProfileManifestPath(profileName, outputDir)\n const manifest = await readExistingManifest(manifestPath)\n if (!manifest) {\n throw new Error(\n `Imported browser profile \"${profileName}\" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`,\n )\n }\n return manifest\n}\n\nexport async function syncImportedChromeProfile(\n name: string,\n options: Pick<ImportChromeProfileOptions, 'outputDir' | 'browserExecutablePath'> = {},\n): Promise<ImportChromeProfileResult> {\n const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir()\n const manifest = await loadImportedBrowserProfile(name, outputDir)\n return importChromeProfile({\n name: manifest.name,\n sourceUserDataDir: manifest.sourceUserDataDir,\n chromeProfileDirectory: manifest.sourceProfileDirectory,\n outputDir,\n overwrite: true,\n browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? undefined,\n })\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,eAAe;AACxB,SAAS,YAAY;AAUd,SAAS,2BAAmC;AACjD,MAAI,QAAQ,aAAa,SAAU,QAAO,KAAK,QAAQ,GAAG,WAAW,uBAAuB,UAAU,QAAQ;AAC9G,MAAI,QAAQ,aAAa,SAAS;AAChC,WAAO,KAAK,QAAQ,IAAI,gBAAgB,KAAK,QAAQ,GAAG,WAAW,OAAO,GAAG,UAAU,UAAU,WAAW;AAAA,EAC9G;AACA,SAAO,KAAK,QAAQ,IAAI,mBAAmB,KAAK,QAAQ,GAAG,SAAS,GAAG,eAAe;AACxF;AAEO,SAAS,6BAA6B,OAAuB;AAClE,SAAO,MACJ,KAAK,EACL,YAAY,EACZ,QAAQ,MAAM,GAAG,EACjB,QAAQ,eAAe,GAAG,EAC1B,QAAQ,YAAY,EAAE,EACtB,MAAM,GAAG,EAAE,KAAK;AACrB;AAEA,eAAsB,wBACpB,cAAc,yBAAyB,GACvC,aAC+B;AAC/B,QAAM,MAAM,MAAM,SAAS,KAAK,aAAa,aAAa,GAAG,MAAM;AACnE,QAAM,aAAa,KAAK,MAAM,GAAG;AAUjC,QAAM,SAAS,aAAa,KAAK,EAAE,YAAY;AAC/C,SAAO,OAAO,QAAQ,WAAW,SAAS,cAAc,CAAC,CAAC,EACvD,IAAI,CAAC,CAAC,WAAW,IAAI,MAAM;AAC1B,UAAM,SAAS,KAAK,aAAa,IAAI,KAAK;AAC1C,UAAM,eAAe,KAAK,QAAQ,KAAK,iBAAiB,IAAI,KAAK;AACjE,UAAM,cAAc,SAAS,eAAe;AAC5C,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,wBAAwB;AAAA,MACxB,mBAAmB,KAAK,aAAa,SAAS;AAAA,MAC9C,8BAA8B,6BAA6B,WAAW;AAAA,IACxE;AAAA,EACF,CAAC,EACA,OAAO,SAAO,CAAC,UAAU,IAAI,MAAM,YAAY,MAAM,UAAU,IAAI,YAAY,YAAY,EAAE,SAAS,MAAM,CAAC;AAClH;;;AC5DA,SAAS,kBAAkB;AAC3B,SAAS,IAAI,OAAO,YAAAA,WAAU,IAAI,MAAM,iBAAiB;AACzD,SAAS,WAAAC,gBAAe;AACxB,SAAS,QAAAC,OAAM,UAAU,WAAW;AAwCpC,IAAM,yBAAyB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,oCAA4C;AAC1D,SAAO,QAAQ,IAAI,sCAAsC,KAAK,KACzDC,MAAKC,SAAQ,GAAG,gBAAgB,kBAAkB;AACzD;AAEO,SAAS,8BAAkD;AAChE,QAAM,UAAU,QAAQ,IAAI,gCAAgC,KAAK,KAAK,QAAQ,IAAI,aAAa,KAAK;AACpG,MAAI,QAAS,QAAO;AAEpB,QAAM,aAAa,QAAQ,aAAa,WACpC,CAAC,8DAA8D,IAC/D,QAAQ,aAAa,UACnB;AAAA,IACED,MAAK,QAAQ,IAAI,gBAAgB,qBAAqB,UAAU,UAAU,eAAe,YAAY;AAAA,IACrGA,MAAK,QAAQ,IAAI,mBAAmB,KAAK,2BAA2B,UAAU,UAAU,eAAe,YAAY;AAAA,EACrH,IACA,CAAC,0BAA0B,iCAAiC,6BAA6B,mBAAmB;AAElH,SAAO,WAAW,KAAK,eAAa,WAAW,SAAS,CAAC;AAC3D;AAEO,SAAS,0BAAmC;AACjD,SAAO,QAAQ,IAAI,0BAA0B,KAAK,EAAE,YAAY,MAAM;AACxE;AAEO,SAAS,oBAAoB,OAAuB;AACzD,SAAO,6BAA6B,KAAK;AAC3C;AAEO,SAAS,oBAAoB,MAAc,YAAY,kCAAkC,GAAW;AACzG,SAAOA,MAAK,WAAW,oBAAoB,IAAI,CAAC;AAClD;AAEO,SAAS,4BAA4B,MAAc,YAAY,kCAAkC,GAAW;AACjH,SAAOA,MAAK,oBAAoB,MAAM,SAAS,GAAG,eAAe;AACnE;AAEO,SAAS,2BAA2B,MAAc,YAAY,kCAAkC,GAAW;AAChH,SAAOA,MAAK,oBAAoB,MAAM,SAAS,GAAG,WAAW;AAC/D;AAEA,eAAe,qBAAqB,MAA8D;AAChG,MAAI;AACF,WAAO,KAAK,MAAM,MAAME,UAAS,MAAM,MAAM,CAAC;AAAA,EAChD,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,sBAAsB,YAAoB,YAA6B;AAC9E,QAAM,MAAM,SAAS,YAAY,UAAU;AAC3C,MAAI,CAAC,IAAK,QAAO;AACjB,QAAM,QAAQ,IAAI,MAAM,GAAG,EAAE,OAAO,OAAO;AAC3C,QAAM,OAAO,MAAM,MAAM,SAAS,CAAC,KAAK;AACxC,MAAI,KAAK,WAAW,WAAW,EAAG,QAAO;AACzC,MAAI,SAAS,UAAU,KAAK,SAAS,OAAO,EAAG,QAAO;AACtD,MAAI,MAAM,KAAK,UAAQ,uBAAuB,SAAS,IAAI,CAAC,EAAG,QAAO;AACtE,MAAI,MAAM,SAAS,gBAAgB,KAAK,MAAM,SAAS,cAAc,EAAG,QAAO;AAC/E,SAAO;AACT;AAEA,eAAe,gBAAgB,mBAA2B,wBAA+C;AACvG,QAAM,GAAG,mBAAmB,wBAAwB;AAAA,IAClD,WAAW;AAAA,IACX,OAAO;AAAA,IACP,cAAc;AAAA,IACd,aAAa;AAAA,IACb,QAAQ,YAAU,CAAC,sBAAsB,mBAAmB,MAAM;AAAA,EACpE,CAAC;AACH;AAEA,SAAS,cAAc,UAAgC,SAAyD;AAC9G,QAAM,qBAAqB,QAAQ,wBAAwB,KAAK;AAChE,QAAM,iBAAiB,QAAQ,OAAO,KAAK,EAAE,YAAY;AACzD,QAAM,iBAAiB,qBACnB,SAAS,KAAK,aAAW,QAAQ,2BAA2B,kBAAkB,IAC9E;AACJ,MAAI,eAAgB,QAAO;AAE3B,QAAM,aAAa,iBACf,SAAS,KAAK,aAAW,QAAQ,MAAM,YAAY,MAAM,cAAc,IACvE;AACJ,MAAI,WAAY,QAAO;AAEvB,QAAM,QAAQ,SAAS,CAAC;AACxB,MAAI,MAAO,QAAO;AAElB,QAAM,OAAO,qBACT,6BAA6B,kBAAkB,MAC/C,iBACE,mBAAmB,cAAc,MACjC;AACN,QAAM,IAAI,MAAM,GAAG,IAAI,qBAAqB,QAAQ,qBAAqB,yBAAyB,CAAC,EAAE;AACvG;AAEA,eAAe,wBAAwB,SAA4C;AACjF,QAAM,UAAU,MAAM,KAAK,QAAQ,iBAAiB,EAAE,MAAM,MAAM,IAAI;AACtE,MAAI,CAAC,SAAS,YAAY,GAAG;AAC3B,UAAM,IAAI,MAAM,6DAA6D,QAAQ,iBAAiB,EAAE;AAAA,EAC1G;AACF;AAEA,eAAe,0BACb,sBACA,2BACA,wBACe;AACf,QAAM,MAAM,MAAMA,UAAS,sBAAsB,MAAM;AACvD,QAAM,aAAa,KAAK,MAAM,GAAG;AAOjC,QAAM,eAAe,WAAW,SAAS,aAAa,sBAAsB,KACvE,WAAW,SAAS,YAAY,WAChC,CAAC;AACN,aAAW,UAAU;AAAA,IACnB,GAAI,WAAW,WAAW,CAAC;AAAA,IAC3B,YAAY,EAAE,SAAS,aAAa;AAAA,IACpC,WAAW;AAAA,IACX,sBAAsB,CAAC,SAAS;AAAA,EAClC;AACA,QAAM,UAAU,2BAA2B,GAAG,KAAK,UAAU,YAAY,MAAM,CAAC,CAAC;AAAA,CAAI;AACvF;AAEA,eAAsB,oBAAoB,UAAsC,CAAC,GAAuC;AACtH,QAAM,oBAAoB,QAAQ,mBAAmB,KAAK,KAAK,yBAAyB;AACxF,QAAM,WAAW,MAAM,wBAAwB,mBAAmB,QAAQ,KAAK;AAC/E,QAAM,gBAAgB,cAAc,UAAU,EAAE,GAAG,SAAS,kBAAkB,CAAC;AAC/E,QAAM,wBAAwB,aAAa;AAE3C,QAAM,OAAO,oBAAoB,QAAQ,MAAM,KAAK,KAAK,cAAc,4BAA4B;AACnG,QAAM,YAAY,QAAQ,WAAW,KAAK,KAAK,kCAAkC;AACjF,QAAM,cAAc,oBAAoB,MAAM,SAAS;AACvD,QAAM,eAAe,4BAA4B,MAAM,SAAS;AAChE,QAAM,cAAc,2BAA2B,MAAM,SAAS;AAC9D,QAAM,mBAAmB,MAAM,qBAAqB,YAAY;AAChE,QAAM,WAAW,WAAW,WAAW;AAEvC,MAAI,YAAY,CAAC,QAAQ,WAAW;AAClC,UAAM,IAAI,MAAM,6BAA6B,IAAI,uBAAuB,WAAW,0CAA0C;AAAA,EAC/H;AAEA,MAAI,SAAU,OAAM,GAAG,aAAa,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AACpE,QAAM,MAAMF,MAAK,aAAa,SAAS,GAAG,EAAE,WAAW,KAAK,CAAC;AAC7D,QAAM;AAAA,IACJA,MAAK,mBAAmB,aAAa;AAAA,IACrCA,MAAK,aAAa,aAAa;AAAA,IAC/B,cAAc;AAAA,EAChB;AACA,QAAM,gBAAgB,cAAc,mBAAmBA,MAAK,aAAa,SAAS,CAAC;AAEnF,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,QAAM,WAA2C;AAAA,IAC/C,eAAe;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,uBAAuB,QAAQ,uBAAuB,KAAK,KAAK,4BAA4B,KAAK;AAAA,IACjG;AAAA,IACA,wBAAwB,cAAc;AAAA,IACtC,mBAAmB,cAAc;AAAA,IACjC,OAAO,cAAc;AAAA,IACrB,aAAa,cAAc;AAAA,IAC3B,WAAW,kBAAkB,aAAa;AAAA,IAC1C,WAAW;AAAA,IACX,UAAU;AAAA,EACZ;AACA,QAAM,UAAU,cAAc,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC;AAAA,CAAI;AAEtE,SAAO;AAAA,IACL,SAAS;AAAA,IACT;AAAA,IACA,eAAe;AAAA,EACjB;AACF;AAEA,eAAsB,2BACpB,OAAO,QAAQ,IAAI,6BAA6B,KAAK,KAAK,WAC1D,YAAY,kCAAkC,GACL;AACzC,QAAM,cAAc,oBAAoB,IAAI;AAC5C,QAAM,eAAe,4BAA4B,aAAa,SAAS;AACvE,QAAM,WAAW,MAAM,qBAAqB,YAAY;AACxD,MAAI,CAAC,UAAU;AACb,UAAM,IAAI;AAAA,MACR,6BAA6B,WAAW,sEAAsE,WAAW;AAAA,IAC3H;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,0BACpB,MACA,UAAmF,CAAC,GAChD;AACpC,QAAM,YAAY,QAAQ,WAAW,KAAK,KAAK,kCAAkC;AACjF,QAAM,WAAW,MAAM,2BAA2B,MAAM,SAAS;AACjE,SAAO,oBAAoB;AAAA,IACzB,MAAM,SAAS;AAAA,IACf,mBAAmB,SAAS;AAAA,IAC5B,wBAAwB,SAAS;AAAA,IACjC;AAAA,IACA,WAAW;AAAA,IACX,uBAAuB,QAAQ,yBAAyB,SAAS,yBAAyB;AAAA,EAC5F,CAAC;AACH;","names":["readFile","homedir","join","join","homedir","readFile"]}
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_MAPS_PROXY_MODE
|
|
3
|
+
} from "./chunk-LFATOGDF.js";
|
|
4
|
+
|
|
1
5
|
// src/workflows/artifact-writer.ts
|
|
2
6
|
import { mkdir, readFile, stat, writeFile } from "fs/promises";
|
|
3
7
|
import { existsSync } from "fs";
|
|
@@ -206,6 +210,7 @@ async function openWorkflowReport(id, outputDir) {
|
|
|
206
210
|
}
|
|
207
211
|
|
|
208
212
|
// src/workflows/registry.ts
|
|
213
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
209
214
|
import { z as z5 } from "zod";
|
|
210
215
|
|
|
211
216
|
// src/workflows/http-client.ts
|
|
@@ -387,102 +392,141 @@ var agentPacketWorkflowDefinition = {
|
|
|
387
392
|
title: "Agent-Ready SEO Packet",
|
|
388
393
|
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
389
394
|
inputSchema: AgentPacketInputSchema,
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
395
|
+
createState: () => ({ serp: null, paa: null, warnings: [] }),
|
|
396
|
+
steps: [
|
|
397
|
+
{
|
|
398
|
+
id: "harvest-serp",
|
|
399
|
+
title: "Harvest organic SERP + AI Overview",
|
|
400
|
+
async run({ input, state, ctx }) {
|
|
401
|
+
if (!input.includeSerp) {
|
|
402
|
+
return { state, output: { skipped: true, reason: "includeSerp is false" } };
|
|
403
|
+
}
|
|
404
|
+
try {
|
|
405
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
406
|
+
query: input.keyword,
|
|
407
|
+
location: input.location,
|
|
408
|
+
serpOnly: true,
|
|
409
|
+
maxQuestions: 1,
|
|
410
|
+
format: "json"
|
|
411
|
+
}, 18e4);
|
|
412
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
413
|
+
return {
|
|
414
|
+
state: { ...state, serp },
|
|
415
|
+
output: {
|
|
416
|
+
organicResults: serp.organicResults?.length ?? 0,
|
|
417
|
+
localPack: serp.localPack?.length ?? 0,
|
|
418
|
+
aiOverviewDetected: Boolean(serp.aiOverview?.detected),
|
|
419
|
+
aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
|
|
420
|
+
}
|
|
421
|
+
};
|
|
422
|
+
} catch (err) {
|
|
423
|
+
const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
424
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
425
|
+
}
|
|
407
426
|
}
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
id: "harvest-paa",
|
|
430
|
+
title: "Harvest People Also Ask",
|
|
431
|
+
async run({ input, state, ctx }) {
|
|
432
|
+
if (!input.includePaa) {
|
|
433
|
+
return { state, output: { skipped: true, reason: "includePaa is false" } };
|
|
434
|
+
}
|
|
435
|
+
try {
|
|
436
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
437
|
+
query: input.keyword,
|
|
438
|
+
location: input.location,
|
|
439
|
+
maxQuestions: input.maxQuestions,
|
|
440
|
+
format: "json"
|
|
441
|
+
}, 28e4);
|
|
442
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
443
|
+
return {
|
|
444
|
+
state: { ...state, paa },
|
|
445
|
+
output: { paaQuestions: paa.flat?.length ?? 0 }
|
|
446
|
+
};
|
|
447
|
+
} catch (err) {
|
|
448
|
+
const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
449
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
},
|
|
453
|
+
{
|
|
454
|
+
id: "assemble",
|
|
455
|
+
title: "Assemble evidence packet",
|
|
456
|
+
async run({ input, state, ctx }) {
|
|
457
|
+
const { serp, paa } = state;
|
|
458
|
+
const warnings = state.warnings;
|
|
459
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
460
|
+
const rows = sourceRows(input, serp, paa);
|
|
461
|
+
const target = normalizeDomain(input.domain);
|
|
462
|
+
const competitors = competitorRows(rows, target);
|
|
463
|
+
const evidence = {
|
|
464
|
+
input,
|
|
465
|
+
serp: serp ?? { status: "skipped" },
|
|
466
|
+
paa: paa ?? { status: "skipped" },
|
|
467
|
+
target: {
|
|
468
|
+
domain: target,
|
|
469
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
470
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
471
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
472
|
+
},
|
|
473
|
+
competitors,
|
|
474
|
+
warnings
|
|
475
|
+
};
|
|
476
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
477
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
478
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
479
|
+
const brief = [
|
|
480
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
481
|
+
"",
|
|
482
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
483
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
484
|
+
"",
|
|
485
|
+
"## Evidence Summary",
|
|
486
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
487
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
488
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
489
|
+
`- Competitor domains: ${competitors.length}`,
|
|
490
|
+
"",
|
|
491
|
+
"## Recommended Use",
|
|
492
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
493
|
+
].join("\n");
|
|
494
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
495
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
496
|
+
"# Agent Tasks",
|
|
497
|
+
"",
|
|
498
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
499
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
500
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
501
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
502
|
+
].join("\n"));
|
|
503
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
504
|
+
"# Agent Instructions",
|
|
505
|
+
"",
|
|
506
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
507
|
+
].join("\n"));
|
|
508
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
509
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
510
|
+
title: "Agent-Ready SEO Packet",
|
|
511
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
512
|
+
summary,
|
|
513
|
+
warnings,
|
|
514
|
+
tables: [
|
|
515
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
516
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
517
|
+
]
|
|
518
|
+
}));
|
|
519
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
520
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
521
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
522
|
+
return {
|
|
523
|
+
state,
|
|
524
|
+
output: { sources: rows.length, competitors: competitors.length, status },
|
|
525
|
+
summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
|
|
526
|
+
};
|
|
419
527
|
}
|
|
420
528
|
}
|
|
421
|
-
|
|
422
|
-
const rows = sourceRows(input, serp, paa);
|
|
423
|
-
const target = normalizeDomain(input.domain);
|
|
424
|
-
const competitors = competitorRows(rows, target);
|
|
425
|
-
const evidence = {
|
|
426
|
-
input,
|
|
427
|
-
serp: serp ?? { status: "skipped" },
|
|
428
|
-
paa: paa ?? { status: "skipped" },
|
|
429
|
-
target: {
|
|
430
|
-
domain: target,
|
|
431
|
-
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
432
|
-
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
433
|
-
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
434
|
-
},
|
|
435
|
-
competitors,
|
|
436
|
-
warnings
|
|
437
|
-
};
|
|
438
|
-
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
439
|
-
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
440
|
-
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
441
|
-
const brief = [
|
|
442
|
-
`# SEO Evidence Brief: ${input.keyword}`,
|
|
443
|
-
"",
|
|
444
|
-
`Location: ${input.location ?? "not specified"}`,
|
|
445
|
-
`Target domain: ${target ?? "not specified"}`,
|
|
446
|
-
"",
|
|
447
|
-
"## Evidence Summary",
|
|
448
|
-
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
449
|
-
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
450
|
-
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
451
|
-
`- Competitor domains: ${competitors.length}`,
|
|
452
|
-
"",
|
|
453
|
-
"## Recommended Use",
|
|
454
|
-
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
455
|
-
].join("\n");
|
|
456
|
-
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
457
|
-
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
458
|
-
"# Agent Tasks",
|
|
459
|
-
"",
|
|
460
|
-
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
461
|
-
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
462
|
-
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
463
|
-
"- [ ] Mark unsupported recommendations as assumptions."
|
|
464
|
-
].join("\n"));
|
|
465
|
-
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
466
|
-
"# Agent Instructions",
|
|
467
|
-
"",
|
|
468
|
-
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
469
|
-
].join("\n"));
|
|
470
|
-
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
471
|
-
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
472
|
-
title: "Agent-Ready SEO Packet",
|
|
473
|
-
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
474
|
-
summary,
|
|
475
|
-
warnings,
|
|
476
|
-
tables: [
|
|
477
|
-
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
478
|
-
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
479
|
-
]
|
|
480
|
-
}));
|
|
481
|
-
const status = warnings.length ? "partial" : "succeeded";
|
|
482
|
-
const counts = { sources: rows.length, competitors: competitors.length };
|
|
483
|
-
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
484
|
-
return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
|
|
485
|
-
}
|
|
529
|
+
]
|
|
486
530
|
};
|
|
487
531
|
|
|
488
532
|
// src/workflows/workflows/directory.ts
|
|
@@ -494,7 +538,7 @@ var DirectoryWorkflowCliInputSchema = z2.object({
|
|
|
494
538
|
maxCities: z2.number().int().min(1).max(100).default(25),
|
|
495
539
|
maxResultsPerCity: z2.number().int().min(1).max(50).default(20),
|
|
496
540
|
concurrency: z2.number().int().min(1).max(5).default(5),
|
|
497
|
-
proxyMode: z2.enum(["location", "configured", "none"]).default(
|
|
541
|
+
proxyMode: z2.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
498
542
|
saveCsv: z2.boolean().default(true)
|
|
499
543
|
});
|
|
500
544
|
function directoryRows(result) {
|
|
@@ -629,7 +673,7 @@ var LocalCompetitiveAuditInputSchema = z3.object({
|
|
|
629
673
|
hydrateTop: z3.number().int().min(0).max(10).default(5),
|
|
630
674
|
maxReviews: z3.number().int().min(0).max(500).default(50),
|
|
631
675
|
concurrency: z3.number().int().min(1).max(5).default(5),
|
|
632
|
-
proxyMode: z3.enum(["location", "configured", "none"]).default(
|
|
676
|
+
proxyMode: z3.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
633
677
|
returnPartial: z3.boolean().default(true)
|
|
634
678
|
});
|
|
635
679
|
async function mapLimit(items, limit, fn) {
|
|
@@ -987,7 +1031,7 @@ var MapComparisonInputSchema = z4.object({
|
|
|
987
1031
|
hydrateTop: z4.number().int().min(0).max(10).default(5),
|
|
988
1032
|
maxReviews: z4.number().int().min(0).max(500).default(25),
|
|
989
1033
|
concurrency: z4.number().int().min(1).max(5).default(5),
|
|
990
|
-
proxyMode: ProxyModeSchema.default(
|
|
1034
|
+
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
991
1035
|
returnPartial: z4.boolean().default(true)
|
|
992
1036
|
}).refine((input) => input.location || input.state, {
|
|
993
1037
|
message: "Either location or state is required for map-comparison"
|
|
@@ -1534,22 +1578,44 @@ function workflowDefinition(id) {
|
|
|
1534
1578
|
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
1535
1579
|
return definition;
|
|
1536
1580
|
}
|
|
1537
|
-
|
|
1581
|
+
function workflowStepCount(id) {
|
|
1538
1582
|
const definition = workflowDefinition(id);
|
|
1539
|
-
|
|
1583
|
+
return definition.steps?.length ?? 1;
|
|
1584
|
+
}
|
|
1585
|
+
function workflowSupportsSteps(id) {
|
|
1586
|
+
const definition = workflowDefinition(id);
|
|
1587
|
+
return Boolean(definition.steps && definition.steps.length > 0);
|
|
1588
|
+
}
|
|
1589
|
+
function resolveApiKey(options) {
|
|
1540
1590
|
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
1541
1591
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
1542
|
-
|
|
1592
|
+
return apiKey;
|
|
1593
|
+
}
|
|
1594
|
+
function resolveApiUrl(options) {
|
|
1595
|
+
return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
1596
|
+
}
|
|
1597
|
+
async function runWorkflow(id, rawInput, options = {}) {
|
|
1598
|
+
const definition = workflowDefinition(id);
|
|
1599
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
1600
|
+
const apiKey = resolveApiKey(options);
|
|
1601
|
+
const apiUrl = resolveApiUrl(options);
|
|
1543
1602
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
1544
1603
|
const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
1604
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
|
|
1545
1605
|
try {
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1606
|
+
if (definition.steps && definition.steps.length > 0) {
|
|
1607
|
+
let state = definition.createState ? definition.createState(input) : {};
|
|
1608
|
+
let summary = null;
|
|
1609
|
+
for (const step of definition.steps) {
|
|
1610
|
+
const outcome = await step.run({ input, state, ctx });
|
|
1611
|
+
state = outcome.state;
|
|
1612
|
+
if (outcome.summary) summary = outcome.summary;
|
|
1613
|
+
}
|
|
1614
|
+
if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
|
|
1615
|
+
return summary;
|
|
1616
|
+
}
|
|
1617
|
+
if (definition.run) return await definition.run(input, ctx);
|
|
1618
|
+
throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
|
|
1553
1619
|
} catch (err) {
|
|
1554
1620
|
const message = err instanceof Error ? err.message : String(err);
|
|
1555
1621
|
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
@@ -1560,6 +1626,45 @@ ${message}
|
|
|
1560
1626
|
throw err;
|
|
1561
1627
|
}
|
|
1562
1628
|
}
|
|
1629
|
+
async function runWorkflowStep(id, rawInput, opts) {
|
|
1630
|
+
const definition = workflowDefinition(id);
|
|
1631
|
+
const steps = definition.steps;
|
|
1632
|
+
if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
|
|
1633
|
+
if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
|
|
1634
|
+
throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
|
|
1635
|
+
}
|
|
1636
|
+
const options = opts.options ?? {};
|
|
1637
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
1638
|
+
const apiKey = resolveApiKey(options);
|
|
1639
|
+
const apiUrl = resolveApiUrl(options);
|
|
1640
|
+
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
1641
|
+
const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
1642
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
|
|
1643
|
+
const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
|
|
1644
|
+
const step = steps[opts.stepIndex];
|
|
1645
|
+
const outcome = await step.run({ input, state, ctx });
|
|
1646
|
+
const writtenArtifacts = await Promise.all(
|
|
1647
|
+
artifacts.artifacts.map(async (artifact) => ({
|
|
1648
|
+
...artifact,
|
|
1649
|
+
content: await readFile2(artifact.path, "utf8").catch(() => "")
|
|
1650
|
+
}))
|
|
1651
|
+
);
|
|
1652
|
+
return {
|
|
1653
|
+
runId: artifacts.runId,
|
|
1654
|
+
workflowId: definition.id,
|
|
1655
|
+
title: definition.title,
|
|
1656
|
+
stepIndex: opts.stepIndex,
|
|
1657
|
+
stepId: step.id,
|
|
1658
|
+
stepTitle: step.title,
|
|
1659
|
+
totalSteps: steps.length,
|
|
1660
|
+
isLast: opts.stepIndex === steps.length - 1,
|
|
1661
|
+
output: outcome.output,
|
|
1662
|
+
state: outcome.state,
|
|
1663
|
+
warnings: outcome.warnings ?? [],
|
|
1664
|
+
summary: outcome.summary ?? null,
|
|
1665
|
+
artifacts: writtenArtifacts
|
|
1666
|
+
};
|
|
1667
|
+
}
|
|
1563
1668
|
|
|
1564
1669
|
export {
|
|
1565
1670
|
csvRecords,
|
|
@@ -1570,6 +1675,9 @@ export {
|
|
|
1570
1675
|
openWorkflowReport,
|
|
1571
1676
|
listWorkflowDefinitions,
|
|
1572
1677
|
workflowDefinition,
|
|
1573
|
-
|
|
1678
|
+
workflowStepCount,
|
|
1679
|
+
workflowSupportsSteps,
|
|
1680
|
+
runWorkflow,
|
|
1681
|
+
runWorkflowStep
|
|
1574
1682
|
};
|
|
1575
|
-
//# sourceMappingURL=chunk-
|
|
1683
|
+
//# sourceMappingURL=chunk-H2R232HK.js.map
|