crawlforge-mcp-server 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/package.json +1 -1
- package/server.js +23 -108
- package/src/cli/commands/login.js +176 -0
- package/src/cli/commands/monitor.js +18 -4
- package/src/cli/index.js +2 -0
- package/src/core/AuthManager.js +19 -6
- package/src/core/ElicitationHelper.js +157 -76
- package/src/server/requestContext.js +50 -0
- package/src/server/transports/streamableHttp.js +17 -4
- package/src/server/withAuth.js +47 -9
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +53 -14
- package/src/tools/advanced/batchScrape/index.js +29 -19
- package/src/tools/agent/agent.js +9 -4
- package/src/tools/crawl/crawlDeep.js +10 -4
- package/src/tools/extract/extractStructured.js +62 -43
- package/src/tools/research/deepResearch.js +10 -4
- package/src/tools/tracking/trackChanges/hosted.js +176 -0
- package/src/tools/tracking/trackChanges/index.js +123 -7
- package/src/tools/tracking/trackChanges/notifier.js +5 -4
- package/src/tools/tracking/trackChanges/schema.js +36 -22
- package/src/core/AlertNotificationSystem.js +0 -602
package/README.md
CHANGED
|
@@ -69,7 +69,13 @@ npm install -g crawlforge-mcp-server
|
|
|
69
69
|
|
|
70
70
|
### 2. Setup Your API Key (required)
|
|
71
71
|
|
|
72
|
-
Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start:
|
|
72
|
+
Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start. The recommended path signs you in through the browser, so the key is never pasted into a terminal (a coding agent can run this for you and relay the URL):
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
crawlforge login
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
It prints an approval URL; open it, approve, and the key is stored in `~/.crawlforge/config.json`. Then run `crawlforge init` to register the MCP server with your client. Or use the interactive wizard, which also configures your clients:
|
|
73
79
|
|
|
74
80
|
```bash
|
|
75
81
|
npx crawlforge-setup
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crawlforge-mcp-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.2.0",
|
|
4
4
|
"mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
|
|
5
5
|
"description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
|
|
6
6
|
"main": "server.js",
|
package/server.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
export { isCreatorModeVerified } from './src/core/creatorMode.js';
|
|
6
6
|
|
|
7
7
|
// Import everything else
|
|
8
|
-
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/server";
|
|
8
|
+
import { McpServer, ResourceTemplate, isInputRequiredResult } from "@modelcontextprotocol/server";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
import { logger } from "./src/utils/Logger.js";
|
|
11
11
|
import { SearchWebTool } from "./src/tools/search/searchWeb.js";
|
|
@@ -23,7 +23,7 @@ import { ListOllamaModelsTool } from "./src/tools/extract/listOllamaModels.js";
|
|
|
23
23
|
import { BatchScrapeTool } from "./src/tools/advanced/BatchScrapeTool.js";
|
|
24
24
|
import { ScrapeWithActionsTool } from "./src/tools/advanced/ScrapeWithActionsTool.js";
|
|
25
25
|
import { DeepResearchTool } from "./src/tools/research/deepResearch.js";
|
|
26
|
-
import { TrackChangesTool } from "./src/tools/tracking/trackChanges/index.js";
|
|
26
|
+
import { TrackChangesTool, TRACK_CHANGES_INPUT_SHAPE } from "./src/tools/tracking/trackChanges/index.js";
|
|
27
27
|
import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
|
|
28
28
|
import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
|
|
29
29
|
import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
|
|
@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
|
|
|
107
107
|
// Create the server
|
|
108
108
|
const server = new McpServer({
|
|
109
109
|
name: "crawlforge",
|
|
110
|
-
version: "6.
|
|
110
|
+
version: "6.2.0",
|
|
111
111
|
description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
|
|
112
112
|
homepage: "https://www.crawlforge.dev",
|
|
113
113
|
icon: "https://www.crawlforge.dev/icon.png",
|
|
@@ -607,12 +607,15 @@ registerToolIfEnabled("crawl_deep", {
|
|
|
607
607
|
...REDACT_PII_PARAM
|
|
608
608
|
},
|
|
609
609
|
outputSchema: OUTPUT_SCHEMAS.crawl_deep
|
|
610
|
-
}, withAuth("crawl_deep", async ({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }) => {
|
|
610
|
+
}, withAuth("crawl_deep", async ({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }, ctx) => {
|
|
611
611
|
try {
|
|
612
612
|
if (!url) {
|
|
613
613
|
return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
|
|
614
614
|
}
|
|
615
|
-
const result = await crawlDeepTool.execute({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session });
|
|
615
|
+
const result = await crawlDeepTool.execute({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }, ctx);
|
|
616
|
+
// A confirmation round trip is the SDK's result shape, not a tool payload:
|
|
617
|
+
// it must reach the transport unwrapped (Phase 4.4).
|
|
618
|
+
if (isInputRequiredResult(result)) return result;
|
|
616
619
|
return dualOutput(result);
|
|
617
620
|
} catch (error) {
|
|
618
621
|
return { content: [{ type: "text", text: `Crawl failed: ${error.message}` }], isError: true };
|
|
@@ -763,12 +766,13 @@ registerToolIfEnabled("extract_structured", {
|
|
|
763
766
|
...VERIFY_NUMBERS_PARAM
|
|
764
767
|
},
|
|
765
768
|
outputSchema: OUTPUT_SCHEMAS.extract_structured
|
|
766
|
-
}, withAuth("extract_structured", async (params) => {
|
|
769
|
+
}, withAuth("extract_structured", async (params, ctx) => {
|
|
767
770
|
try {
|
|
768
771
|
// Forward params whole. This wrapper used to destructure a fixed six, which
|
|
769
772
|
// silently dropped respect_robots and user_agent — both declared here and
|
|
770
773
|
// read by the tool, so the G5 override was accepted and ignored.
|
|
771
|
-
const result = await extractStructuredTool.execute(params);
|
|
774
|
+
const result = await extractStructuredTool.execute(params, ctx);
|
|
775
|
+
if (isInputRequiredResult(result)) return result;
|
|
772
776
|
return dualOutput(result);
|
|
773
777
|
} catch (error) {
|
|
774
778
|
return { content: [{ type: "text", text: `Structured extraction failed: ${error.message}` }], isError: true };
|
|
@@ -855,9 +859,10 @@ registerToolIfEnabled("batch_scrape", {
|
|
|
855
859
|
...MAX_INLINE_CHARS_PARAM,
|
|
856
860
|
...REDACT_PII_PARAM
|
|
857
861
|
}
|
|
858
|
-
}, withAuth("batch_scrape", async (params) => {
|
|
862
|
+
}, withAuth("batch_scrape", async (params, ctx) => {
|
|
859
863
|
try {
|
|
860
|
-
const result = await batchScrapeTool.execute(params);
|
|
864
|
+
const result = await batchScrapeTool.execute(params, ctx);
|
|
865
|
+
if (isInputRequiredResult(result)) return result;
|
|
861
866
|
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
862
867
|
} catch (error) {
|
|
863
868
|
return { content: [{ type: "text", text: `Batch scrape failed: ${error.message}` }], isError: true };
|
|
@@ -1056,9 +1061,10 @@ registerToolIfEnabled("deep_research", {
|
|
|
1056
1061
|
}).optional().describe("Webhook for progress and completion notifications"),
|
|
1057
1062
|
...MAX_INLINE_CHARS_PARAM
|
|
1058
1063
|
}
|
|
1059
|
-
}, withAuth("deep_research", async (params) => {
|
|
1064
|
+
}, withAuth("deep_research", async (params, ctx) => {
|
|
1060
1065
|
try {
|
|
1061
|
-
const result = await deepResearchTool.execute(params);
|
|
1066
|
+
const result = await deepResearchTool.execute(params, ctx);
|
|
1067
|
+
if (isInputRequiredResult(result)) return result;
|
|
1062
1068
|
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
1063
1069
|
} catch (error) {
|
|
1064
1070
|
return { content: [{ type: "text", text: `Deep research failed: ${error.message}` }], isError: true };
|
|
@@ -1120,9 +1126,10 @@ registerToolIfEnabled("agent", {
|
|
|
1120
1126
|
maxSteps: z.number().min(1).max(10).optional().default(5).describe("Max fetch iterations (hard cap: 10)"),
|
|
1121
1127
|
maxUrls: z.number().min(1).max(20).optional().default(10).describe("Max URLs to fetch (hard cap: 20)")
|
|
1122
1128
|
}
|
|
1123
|
-
}, withAuth("agent", async (params) => {
|
|
1129
|
+
}, withAuth("agent", async (params, ctx) => {
|
|
1124
1130
|
try {
|
|
1125
|
-
const result = await agentTool.execute(params);
|
|
1131
|
+
const result = await agentTool.execute(params, ctx);
|
|
1132
|
+
if (isInputRequiredResult(result)) return result;
|
|
1126
1133
|
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
1127
1134
|
} catch (error) {
|
|
1128
1135
|
return { content: [{ type: "text", text: `Agent failed: ${error.message}` }], isError: true };
|
|
@@ -1131,103 +1138,11 @@ registerToolIfEnabled("agent", {
|
|
|
1131
1138
|
|
|
1132
1139
|
// Tool: track_changes
|
|
1133
1140
|
registerToolIfEnabled("track_changes", {
|
|
1134
|
-
description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
|
|
1141
|
+
description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
|
|
1135
1142
|
annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
|
|
1143
|
+
// The tool module owns the schema (G5); this is the same shape it validates with.
|
|
1136
1144
|
inputSchema: {
|
|
1137
|
-
|
|
1138
|
-
operation: z.enum([
|
|
1139
|
-
'create_baseline', 'compare', 'monitor', 'get_history', 'get_stats',
|
|
1140
|
-
'create_scheduled_monitor', 'stop_scheduled_monitor', 'list_scheduled_monitors', 'get_dashboard',
|
|
1141
|
-
'export_history', 'create_alert_rule', 'generate_trend_report', 'get_monitoring_templates'
|
|
1142
|
-
]).default('compare').describe("Tracking operation to perform"),
|
|
1143
|
-
content: z.string().optional().describe("Content to compare against baseline"),
|
|
1144
|
-
html: z.string().optional().describe("HTML content to compare against baseline"),
|
|
1145
|
-
trackingOptions: z.object({
|
|
1146
|
-
granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
|
|
1147
|
-
trackText: z.boolean().default(true),
|
|
1148
|
-
trackStructure: z.boolean().default(true),
|
|
1149
|
-
trackAttributes: z.boolean().default(false),
|
|
1150
|
-
trackImages: z.boolean().default(false),
|
|
1151
|
-
trackLinks: z.boolean().default(true),
|
|
1152
|
-
ignoreWhitespace: z.boolean().default(true),
|
|
1153
|
-
ignoreCase: z.boolean().default(false),
|
|
1154
|
-
customSelectors: z.array(z.string()).optional(),
|
|
1155
|
-
excludeSelectors: z.array(z.string()).optional(),
|
|
1156
|
-
significanceThresholds: z.object({
|
|
1157
|
-
minor: z.number().min(0).max(1).default(0.1),
|
|
1158
|
-
moderate: z.number().min(0).max(1).default(0.3),
|
|
1159
|
-
major: z.number().min(0).max(1).default(0.7)
|
|
1160
|
-
}).optional()
|
|
1161
|
-
}).optional().describe("Options for how changes are tracked and compared"),
|
|
1162
|
-
monitoringOptions: z.object({
|
|
1163
|
-
enabled: z.boolean().default(false),
|
|
1164
|
-
interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
|
|
1165
|
-
maxRetries: z.number().min(0).max(5).default(3),
|
|
1166
|
-
retryDelay: z.number().min(1000).max(60000).default(5000),
|
|
1167
|
-
notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).default('moderate'),
|
|
1168
|
-
enableWebhook: z.boolean().default(false),
|
|
1169
|
-
webhookUrl: z.string().url().optional(),
|
|
1170
|
-
webhookSecret: z.string().optional()
|
|
1171
|
-
}).optional().describe("Monitoring schedule and notification settings"),
|
|
1172
|
-
storageOptions: z.object({
|
|
1173
|
-
enableSnapshots: z.boolean().default(true),
|
|
1174
|
-
retainHistory: z.boolean().default(true),
|
|
1175
|
-
maxHistoryEntries: z.number().min(1).max(1000).default(100),
|
|
1176
|
-
compressionEnabled: z.boolean().default(true),
|
|
1177
|
-
deltaStorageEnabled: z.boolean().default(true)
|
|
1178
|
-
}).optional().describe("Storage and history retention settings"),
|
|
1179
|
-
queryOptions: z.object({
|
|
1180
|
-
limit: z.number().min(1).max(500).default(50),
|
|
1181
|
-
offset: z.number().min(0).default(0),
|
|
1182
|
-
startTime: z.number().optional(),
|
|
1183
|
-
endTime: z.number().optional(),
|
|
1184
|
-
includeContent: z.boolean().default(false),
|
|
1185
|
-
significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
|
|
1186
|
-
}).optional().describe("Query options for history and stats retrieval"),
|
|
1187
|
-
notificationOptions: z.object({
|
|
1188
|
-
webhook: z.object({
|
|
1189
|
-
enabled: z.boolean().default(false),
|
|
1190
|
-
url: z.string().url().optional(),
|
|
1191
|
-
method: z.enum(['POST', 'PUT']).default('POST'),
|
|
1192
|
-
headers: z.record(z.string()).optional(),
|
|
1193
|
-
signingSecret: z.string().optional(),
|
|
1194
|
-
includeContent: z.boolean().default(false)
|
|
1195
|
-
}).optional(),
|
|
1196
|
-
slack: z.object({
|
|
1197
|
-
enabled: z.boolean().default(false),
|
|
1198
|
-
webhookUrl: z.string().url().optional(),
|
|
1199
|
-
channel: z.string().optional(),
|
|
1200
|
-
username: z.string().optional()
|
|
1201
|
-
}).optional()
|
|
1202
|
-
}).optional().describe("Notification configuration for webhooks and Slack"),
|
|
1203
|
-
scheduledMonitorOptions: z.object({
|
|
1204
|
-
schedule: z.string().optional().describe("Optional cron expression (power users)"),
|
|
1205
|
-
templateId: z.string().optional(),
|
|
1206
|
-
enabled: z.boolean().default(true),
|
|
1207
|
-
interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
|
|
1208
|
-
goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
|
|
1209
|
-
monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
|
|
1210
|
-
notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional()
|
|
1211
|
-
}).optional().describe("Scheduled monitoring: recurring compare + notify, optional plain-English goal"),
|
|
1212
|
-
alertRuleOptions: z.object({
|
|
1213
|
-
ruleId: z.string().optional(),
|
|
1214
|
-
condition: z.string().optional(),
|
|
1215
|
-
actions: z.array(z.enum(['webhook', 'email', 'slack'])).optional(),
|
|
1216
|
-
throttle: z.number().min(0).optional(),
|
|
1217
|
-
priority: z.enum(['low', 'medium', 'high']).optional()
|
|
1218
|
-
}).optional().describe("Alert rule configuration for change notifications"),
|
|
1219
|
-
exportOptions: z.object({
|
|
1220
|
-
format: z.enum(['json', 'csv']).default('json'),
|
|
1221
|
-
startTime: z.number().optional(),
|
|
1222
|
-
endTime: z.number().optional(),
|
|
1223
|
-
includeContent: z.boolean().default(false),
|
|
1224
|
-
includeSnapshots: z.boolean().default(false)
|
|
1225
|
-
}).optional().describe("Export options for change history data"),
|
|
1226
|
-
dashboardOptions: z.object({
|
|
1227
|
-
includeRecentAlerts: z.boolean().default(true),
|
|
1228
|
-
includeTrends: z.boolean().default(true),
|
|
1229
|
-
includeMonitorStatus: z.boolean().default(true)
|
|
1230
|
-
}).optional().describe("Dashboard display options"),
|
|
1145
|
+
...TRACK_CHANGES_INPUT_SHAPE,
|
|
1231
1146
|
...COMPLIANCE_PARAMS
|
|
1232
1147
|
}
|
|
1233
1148
|
}, withAuth("track_changes", async (params) => {
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* login command — browser handoff that stores an API key in ~/.crawlforge/config.json.
|
|
3
|
+
*
|
|
4
|
+
* The CLI mints PKCE parameters, prints an approval URL for the human, and polls
|
|
5
|
+
* the website until the signed-in user approves; the key is delivered once, over
|
|
6
|
+
* the poll, and never typed into a terminal. This command stores the credential
|
|
7
|
+
* ONLY — registering the MCP server with a client is `crawlforge init`.
|
|
8
|
+
*/
|
|
9
|
+
import { randomBytes, createHash } from 'node:crypto';
|
|
10
|
+
import { hostname } from 'node:os';
|
|
11
|
+
import { existsSync } from 'node:fs';
|
|
12
|
+
import authManager from '../../core/authManager.js';
|
|
13
|
+
import { resolveApiEndpoint } from '../../core/endpointGuard.js';
|
|
14
|
+
|
|
15
|
+
const POLL_INTERVAL_MS = 3000;
|
|
16
|
+
const MAX_BACKOFF_MS = 30000;
|
|
17
|
+
const MAX_CONSECUTIVE_FAILURES = 10;
|
|
18
|
+
|
|
19
|
+
export function generateLoginParams() {
|
|
20
|
+
const codeVerifier = randomBytes(32).toString('base64url');
|
|
21
|
+
return {
|
|
22
|
+
sessionId: randomBytes(16).toString('hex'),
|
|
23
|
+
codeVerifier,
|
|
24
|
+
codeChallenge: createHash('sha256').update(codeVerifier).digest('base64url'),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function buildApprovalUrl(endpoint, params, name) {
|
|
29
|
+
return `${endpoint}/cli-auth?session_id=${params.sessionId}` +
|
|
30
|
+
`&code_challenge=${params.codeChallenge}&name=${encodeURIComponent(name)}`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function loginError(code, message) {
|
|
34
|
+
const err = new Error(message);
|
|
35
|
+
err.code = code;
|
|
36
|
+
return err;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Poll the status endpoint until the key arrives. Resolves with the `complete`
|
|
41
|
+
* payload; rejects with an error whose `code` names why (CLI_AUTH_TIMEOUT,
|
|
42
|
+
* CLI_AUTH_VERIFIER_MISMATCH, CLI_AUTH_UNREACHABLE). `sleep` and `now` are
|
|
43
|
+
* injectable so tests neither wait nor hit the network.
|
|
44
|
+
*/
|
|
45
|
+
export async function pollStatus(fetchImpl, endpoint, params, {
|
|
46
|
+
intervalMs = POLL_INTERVAL_MS,
|
|
47
|
+
timeoutMs = 600000,
|
|
48
|
+
requestTimeoutMs = 30000,
|
|
49
|
+
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
50
|
+
now = Date.now,
|
|
51
|
+
} = {}) {
|
|
52
|
+
const deadline = now() + timeoutMs;
|
|
53
|
+
let interval = intervalMs;
|
|
54
|
+
let failures = 0;
|
|
55
|
+
let lastFailure = '';
|
|
56
|
+
|
|
57
|
+
while (now() < deadline) {
|
|
58
|
+
let response;
|
|
59
|
+
try {
|
|
60
|
+
response = await fetchImpl(`${endpoint}/api/auth/cli/status`, {
|
|
61
|
+
method: 'POST',
|
|
62
|
+
headers: { 'Content-Type': 'application/json' },
|
|
63
|
+
body: JSON.stringify({ session_id: params.sessionId, code_verifier: params.codeVerifier }),
|
|
64
|
+
signal: AbortSignal.timeout(requestTimeoutMs),
|
|
65
|
+
});
|
|
66
|
+
} catch (err) {
|
|
67
|
+
response = null;
|
|
68
|
+
lastFailure = err.message;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (response && response.status === 200) {
|
|
72
|
+
const body = await response.json();
|
|
73
|
+
if (body.status === 'complete') return body;
|
|
74
|
+
if (body.status === 'pending') {
|
|
75
|
+
failures = 0;
|
|
76
|
+
} else {
|
|
77
|
+
failures++;
|
|
78
|
+
lastFailure = `unexpected status "${body.status}"`;
|
|
79
|
+
}
|
|
80
|
+
} else if (response && response.status === 403) {
|
|
81
|
+
let code = 'CLI_AUTH_FORBIDDEN';
|
|
82
|
+
try { code = (await response.json()).error?.code || code; } catch { /* keep default */ }
|
|
83
|
+
throw loginError(code, code === 'CLI_AUTH_VERIFIER_MISMATCH'
|
|
84
|
+
? 'The website rejected this session\'s verifier. Run crawlforge login again and open the new URL.'
|
|
85
|
+
: `The website refused the login session (${code}).`);
|
|
86
|
+
} else if (response && response.status === 429) {
|
|
87
|
+
interval = Math.min(interval * 2, MAX_BACKOFF_MS);
|
|
88
|
+
} else if (response) {
|
|
89
|
+
failures++;
|
|
90
|
+
lastFailure = `HTTP ${response.status}`;
|
|
91
|
+
} else {
|
|
92
|
+
failures++;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (failures >= MAX_CONSECUTIVE_FAILURES) {
|
|
96
|
+
throw loginError('CLI_AUTH_UNREACHABLE',
|
|
97
|
+
`Gave up after ${failures} consecutive failed status checks (last: ${lastFailure}).`);
|
|
98
|
+
}
|
|
99
|
+
await sleep(interval);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
throw loginError('CLI_AUTH_TIMEOUT',
|
|
103
|
+
`No approval within ${Math.round(timeoutMs / 1000)} seconds. Run crawlforge login again.`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function register(program) {
|
|
107
|
+
program
|
|
108
|
+
.command('login')
|
|
109
|
+
.description('Sign in through your browser and store an API key in ~/.crawlforge/config.json (does not touch client configs)')
|
|
110
|
+
.option('--name <name>', 'Name for the API key the approval creates', `CLI on ${hostname()}`)
|
|
111
|
+
// Not `--timeout`: the program-level `--timeout <ms>` parses argv first and
|
|
112
|
+
// would swallow the value, so a subcommand option of that name never gets one.
|
|
113
|
+
.option('--wait <seconds>', 'How long to wait for approval', '600')
|
|
114
|
+
.action(async (opts, cmd) => {
|
|
115
|
+
const json = cmd.parent.opts().json;
|
|
116
|
+
const out = (msg) => process.stderr.write(msg + '\n');
|
|
117
|
+
const fail = (code, message) => {
|
|
118
|
+
if (json) {
|
|
119
|
+
process.stdout.write(JSON.stringify({ status: 'error', code, message }) + '\n', () => process.exit(1));
|
|
120
|
+
} else {
|
|
121
|
+
out('Error: ' + message);
|
|
122
|
+
process.exit(1);
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
process.on('SIGINT', () => fail('CLI_AUTH_CANCELLED', 'Login cancelled.'));
|
|
126
|
+
|
|
127
|
+
// resolveApiEndpoint() returns the origin with a trailing slash; strip it
|
|
128
|
+
// so the approval URL and the status endpoint are not `host//path`.
|
|
129
|
+
const endpoint = resolveApiEndpoint(process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev').replace(/\/+$/, '');
|
|
130
|
+
const params = generateLoginParams();
|
|
131
|
+
const hadConfig = existsSync(authManager.configPath);
|
|
132
|
+
|
|
133
|
+
out('Open this URL in your browser and approve the key (session ' + params.sessionId.slice(0, 8) + '):');
|
|
134
|
+
out('');
|
|
135
|
+
out(' ' + buildApprovalUrl(endpoint, params, opts.name));
|
|
136
|
+
out('');
|
|
137
|
+
out('Waiting for approval… (Ctrl-C to cancel)');
|
|
138
|
+
|
|
139
|
+
let result;
|
|
140
|
+
try {
|
|
141
|
+
result = await pollStatus(fetch, endpoint, params, {
|
|
142
|
+
timeoutMs: parseInt(opts.wait, 10) * 1000,
|
|
143
|
+
requestTimeoutMs: parseInt(process.env.CRAWLFORGE_CLI_TIMEOUT || '30000', 10),
|
|
144
|
+
});
|
|
145
|
+
} catch (err) {
|
|
146
|
+
return fail(err.code || 'CLI_AUTH_FAILED', err.message);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const validation = await authManager.validateApiKey(result.api_key);
|
|
150
|
+
if (!validation.valid) {
|
|
151
|
+
return fail('CLI_AUTH_KEY_INVALID', 'The delivered API key failed validation: ' + validation.error);
|
|
152
|
+
}
|
|
153
|
+
await authManager.saveConfig(result.api_key, validation.userId, validation.email);
|
|
154
|
+
|
|
155
|
+
if (json) {
|
|
156
|
+
const line = JSON.stringify({
|
|
157
|
+
status: 'complete',
|
|
158
|
+
email: validation.email,
|
|
159
|
+
key_name: result.key_name,
|
|
160
|
+
config_path: authManager.configPath,
|
|
161
|
+
credits_remaining: validation.creditsRemaining,
|
|
162
|
+
plan: validation.planId,
|
|
163
|
+
});
|
|
164
|
+
process.stdout.write(line + '\n', () => process.exit(0));
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
out('Signed in as ' + validation.email);
|
|
169
|
+
out('Credits remaining: ' + validation.creditsRemaining);
|
|
170
|
+
out('Plan: ' + validation.planId);
|
|
171
|
+
out('API key "' + result.key_name + '" saved to ' + authManager.configPath +
|
|
172
|
+
(hadConfig ? ' (replaced the previous config)' : ''));
|
|
173
|
+
out('Next: crawlforge init --client claude-code|claude-desktop|cursor registers the MCP server with a client (this command did not modify any client config).');
|
|
174
|
+
process.exit(0);
|
|
175
|
+
});
|
|
176
|
+
}
|
|
@@ -61,19 +61,33 @@ export function register(program) {
|
|
|
61
61
|
.option('--threshold <level>', 'Notification threshold: minor|moderate|major|critical', 'moderate')
|
|
62
62
|
.option('--cron <expr>', 'Optional cron expression (advanced)')
|
|
63
63
|
.option('--selector <css>', 'CSS selector to scope monitoring')
|
|
64
|
+
.option('--hosted', "Run the monitor on CrawlForge's servers (fires without this process; email + signed webhooks; 3 credits per compared target per check)")
|
|
65
|
+
.option('--email <addresses>', 'Comma-separated notification emails (sent by hosted monitors only)')
|
|
66
|
+
.option('--name <text>', 'Display name for a hosted monitor (default: the URL host)')
|
|
64
67
|
.action(async (url, opts) => {
|
|
65
68
|
const tool = new TrackChangesTool(getToolConfig('track_changes'));
|
|
69
|
+
const notificationOptions = {
|
|
70
|
+
...(opts.webhook ? { webhook: { enabled: true, url: opts.webhook } } : {}),
|
|
71
|
+
...(opts.email ? { email: { enabled: true, recipients: opts.email.split(',').map((s) => s.trim()).filter(Boolean) } } : {})
|
|
72
|
+
};
|
|
73
|
+
if (opts.email && !opts.hosted) {
|
|
74
|
+
process.stderr.write('Warning: local monitors do not send email; add --hosted for --email to take effect.\n');
|
|
75
|
+
}
|
|
66
76
|
try {
|
|
67
77
|
const res = await tool.execute({
|
|
68
78
|
url,
|
|
69
79
|
operation: 'create_scheduled_monitor',
|
|
70
80
|
...(opts.selector ? { trackingOptions: { customSelectors: [opts.selector] } } : {}),
|
|
71
|
-
...(
|
|
81
|
+
...(Object.keys(notificationOptions).length ? { notificationOptions } : {}),
|
|
72
82
|
scheduledMonitorOptions: {
|
|
73
83
|
interval: Math.max(parseInt(opts.every, 10), 60) * 1000,
|
|
74
84
|
...(opts.goal ? { goal: opts.goal } : {}),
|
|
75
85
|
...(opts.cron ? { schedule: opts.cron } : {}),
|
|
76
|
-
|
|
86
|
+
...(opts.hosted ? { hosted: true } : {}),
|
|
87
|
+
...(opts.name ? { name: opts.name } : {}),
|
|
88
|
+
// Local only: a hosted check has no significance threshold, and
|
|
89
|
+
// the option's default would otherwise warn on every hosted create.
|
|
90
|
+
...(opts.hosted ? {} : { notificationThreshold: opts.threshold })
|
|
77
91
|
}
|
|
78
92
|
});
|
|
79
93
|
emit(res);
|
|
@@ -86,7 +100,7 @@ export function register(program) {
|
|
|
86
100
|
|
|
87
101
|
program
|
|
88
102
|
.command('monitor:list')
|
|
89
|
-
.description('List
|
|
103
|
+
.description('List scheduled monitors (local and hosted)')
|
|
90
104
|
.action(async () => {
|
|
91
105
|
const tool = new TrackChangesTool(getToolConfig('track_changes'));
|
|
92
106
|
try {
|
|
@@ -100,7 +114,7 @@ export function register(program) {
|
|
|
100
114
|
|
|
101
115
|
program
|
|
102
116
|
.command('monitor:stop <id>')
|
|
103
|
-
.description('Stop and remove a scheduled monitor by id')
|
|
117
|
+
.description('Stop and remove a scheduled monitor by id (local or hosted)')
|
|
104
118
|
.action(async (id) => {
|
|
105
119
|
const tool = new TrackChangesTool(getToolConfig('track_changes'));
|
|
106
120
|
try {
|
package/src/cli/index.js
CHANGED
|
@@ -59,6 +59,7 @@ import { register as registerMonitor } from './commands/monitor.js';
|
|
|
59
59
|
import { register as registerInstallSkills } from './commands/install-skills.js';
|
|
60
60
|
import { register as registerUninstallSkills } from './commands/uninstall-skills.js';
|
|
61
61
|
import { register as registerInit } from './commands/init.js';
|
|
62
|
+
import { register as registerLogin } from './commands/login.js';
|
|
62
63
|
|
|
63
64
|
// ─── MCP stdio server mode (backward compatibility) ──────────────────────────
|
|
64
65
|
// Before v4.1.0 the `crawlforge` bin WAS the MCP server. v4.1.0 turned it into
|
|
@@ -138,6 +139,7 @@ registerMonitor(program);
|
|
|
138
139
|
registerInstallSkills(program);
|
|
139
140
|
registerUninstallSkills(program);
|
|
140
141
|
registerInit(program);
|
|
142
|
+
registerLogin(program);
|
|
141
143
|
|
|
142
144
|
// `crawlforge mcp` / `crawlforge serve` — explicitly start the MCP server over
|
|
143
145
|
// stdio. Extra args (e.g. --http) are read directly by server.js from argv.
|
package/src/core/AuthManager.js
CHANGED
|
@@ -241,9 +241,16 @@ class AuthManager {
|
|
|
241
241
|
}
|
|
242
242
|
|
|
243
243
|
/**
|
|
244
|
-
* Check if user has enough credits for a tool
|
|
244
|
+
* Check if user has enough credits for a tool.
|
|
245
|
+
*
|
|
246
|
+
* @param {number} estimatedCredits
|
|
247
|
+
* @param {object} [ctx] the SDK per-request context, so the low-credit
|
|
248
|
+
* warning can ask as a multi-round-trip step (Phase 4.4).
|
|
249
|
+
* @returns {Promise<boolean|object>} `true`/`false` as before, or an
|
|
250
|
+
* `input_required` result the caller must RETURN verbatim — `withAuth`
|
|
251
|
+
* detects it, bills nothing and lets the SDK gather the answer.
|
|
245
252
|
*/
|
|
246
|
-
async checkCredits(estimatedCredits = 1) {
|
|
253
|
+
async checkCredits(estimatedCredits = 1, ctx) {
|
|
247
254
|
// Creator mode has unlimited credits
|
|
248
255
|
if (this.isCreatorMode()) {
|
|
249
256
|
return true;
|
|
@@ -277,10 +284,16 @@ class AuthManager {
|
|
|
277
284
|
this.lastCreditCheck = now;
|
|
278
285
|
this.lastSuccessfulCreditCheck.set(this.config.userId, now);
|
|
279
286
|
|
|
280
|
-
// D1.4: If credits are close to running out, elicit confirmation instead
|
|
287
|
+
// D1.4: If credits are close to running out, elicit confirmation instead
|
|
288
|
+
// of hard-failing. Phase 4.4: the ask is a round trip now — `ask` hands
|
|
289
|
+
// an `input_required` result back to withAuth, which returns it unbilled
|
|
290
|
+
// and is re-entered here with the answer. A client that cannot be asked
|
|
291
|
+
// still proceeds, which is what the inline helper did.
|
|
281
292
|
if (data.creditsRemaining < estimatedCredits) {
|
|
282
293
|
if (this._elicitation) {
|
|
283
|
-
const
|
|
294
|
+
const gate = this._elicitation.confirm(
|
|
295
|
+
ctx,
|
|
296
|
+
'credits:low',
|
|
284
297
|
`Low credits: ${data.creditsRemaining} remaining, this tool needs ~${estimatedCredits}. Proceed anyway?`,
|
|
285
298
|
{
|
|
286
299
|
credits_remaining: data.creditsRemaining,
|
|
@@ -288,8 +301,8 @@ class AuthManager {
|
|
|
288
301
|
note: 'Top up at https://www.crawlforge.dev/dashboard',
|
|
289
302
|
}
|
|
290
303
|
);
|
|
291
|
-
if (
|
|
292
|
-
return
|
|
304
|
+
if (gate.status === 'ask') return gate.result;
|
|
305
|
+
return gate.status === 'proceed'; // confirmed (or unaskable) — let tool attempt it
|
|
293
306
|
}
|
|
294
307
|
return false; // no elicitation — standard hard-fail behavior
|
|
295
308
|
}
|