crawlforge-mcp-server 6.0.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -69,7 +69,13 @@ npm install -g crawlforge-mcp-server
69
69
 
70
70
  ### 2. Setup Your API Key (required)
71
71
 
72
- Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start:
72
+ Every tool requires a CrawlForge API key — new accounts get 1,000 free trial credits to start. The recommended path signs you in through the browser, so the key is never pasted into a terminal (a coding agent can run this for you and relay the URL):
73
+
74
+ ```bash
75
+ crawlforge login
76
+ ```
77
+
78
+ It prints an approval URL; open it, approve, and the key is stored in `~/.crawlforge/config.json`. Then run `crawlforge init` to register the MCP server with your client. Or use the interactive wizard, which also configures your clients:
73
79
 
74
80
  ```bash
75
81
  npx crawlforge-setup
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crawlforge-mcp-server",
3
- "version": "6.0.0",
3
+ "version": "6.2.0",
4
4
  "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
5
5
  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
6
6
  "main": "server.js",
package/server.js CHANGED
@@ -5,7 +5,7 @@
5
5
  export { isCreatorModeVerified } from './src/core/creatorMode.js';
6
6
 
7
7
  // Import everything else
8
- import { McpServer, ResourceTemplate } from "@modelcontextprotocol/server";
8
+ import { McpServer, ResourceTemplate, isInputRequiredResult } from "@modelcontextprotocol/server";
9
9
  import { z } from "zod";
10
10
  import { logger } from "./src/utils/Logger.js";
11
11
  import { SearchWebTool } from "./src/tools/search/searchWeb.js";
@@ -23,7 +23,7 @@ import { ListOllamaModelsTool } from "./src/tools/extract/listOllamaModels.js";
23
23
  import { BatchScrapeTool } from "./src/tools/advanced/BatchScrapeTool.js";
24
24
  import { ScrapeWithActionsTool } from "./src/tools/advanced/ScrapeWithActionsTool.js";
25
25
  import { DeepResearchTool } from "./src/tools/research/deepResearch.js";
26
- import { TrackChangesTool } from "./src/tools/tracking/trackChanges/index.js";
26
+ import { TrackChangesTool, TRACK_CHANGES_INPUT_SHAPE } from "./src/tools/tracking/trackChanges/index.js";
27
27
  import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
28
28
  import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
29
29
  import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
107
107
  // Create the server
108
108
  const server = new McpServer({
109
109
  name: "crawlforge",
110
- version: "6.0.0",
110
+ version: "6.2.0",
111
111
  description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
112
112
  homepage: "https://www.crawlforge.dev",
113
113
  icon: "https://www.crawlforge.dev/icon.png",
@@ -607,12 +607,15 @@ registerToolIfEnabled("crawl_deep", {
607
607
  ...REDACT_PII_PARAM
608
608
  },
609
609
  outputSchema: OUTPUT_SCHEMAS.crawl_deep
610
- }, withAuth("crawl_deep", async ({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }) => {
610
+ }, withAuth("crawl_deep", async ({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }, ctx) => {
611
611
  try {
612
612
  if (!url) {
613
613
  return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
614
614
  }
615
- const result = await crawlDeepTool.execute({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session });
615
+ const result = await crawlDeepTool.execute({ url, max_depth, max_pages, include_patterns, exclude_patterns, follow_external, respect_robots, extract_content, content_max_length, concurrency, enable_link_analysis, link_analysis_options, domain_filter, import_filter_config, session }, ctx);
616
+ // A confirmation round trip is the SDK's result shape, not a tool payload:
617
+ // it must reach the transport unwrapped (Phase 4.4).
618
+ if (isInputRequiredResult(result)) return result;
616
619
  return dualOutput(result);
617
620
  } catch (error) {
618
621
  return { content: [{ type: "text", text: `Crawl failed: ${error.message}` }], isError: true };
@@ -763,12 +766,13 @@ registerToolIfEnabled("extract_structured", {
763
766
  ...VERIFY_NUMBERS_PARAM
764
767
  },
765
768
  outputSchema: OUTPUT_SCHEMAS.extract_structured
766
- }, withAuth("extract_structured", async (params) => {
769
+ }, withAuth("extract_structured", async (params, ctx) => {
767
770
  try {
768
771
  // Forward params whole. This wrapper used to destructure a fixed six, which
769
772
  // silently dropped respect_robots and user_agent — both declared here and
770
773
  // read by the tool, so the G5 override was accepted and ignored.
771
- const result = await extractStructuredTool.execute(params);
774
+ const result = await extractStructuredTool.execute(params, ctx);
775
+ if (isInputRequiredResult(result)) return result;
772
776
  return dualOutput(result);
773
777
  } catch (error) {
774
778
  return { content: [{ type: "text", text: `Structured extraction failed: ${error.message}` }], isError: true };
@@ -855,9 +859,10 @@ registerToolIfEnabled("batch_scrape", {
855
859
  ...MAX_INLINE_CHARS_PARAM,
856
860
  ...REDACT_PII_PARAM
857
861
  }
858
- }, withAuth("batch_scrape", async (params) => {
862
+ }, withAuth("batch_scrape", async (params, ctx) => {
859
863
  try {
860
- const result = await batchScrapeTool.execute(params);
864
+ const result = await batchScrapeTool.execute(params, ctx);
865
+ if (isInputRequiredResult(result)) return result;
861
866
  return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
862
867
  } catch (error) {
863
868
  return { content: [{ type: "text", text: `Batch scrape failed: ${error.message}` }], isError: true };
@@ -1056,9 +1061,10 @@ registerToolIfEnabled("deep_research", {
1056
1061
  }).optional().describe("Webhook for progress and completion notifications"),
1057
1062
  ...MAX_INLINE_CHARS_PARAM
1058
1063
  }
1059
- }, withAuth("deep_research", async (params) => {
1064
+ }, withAuth("deep_research", async (params, ctx) => {
1060
1065
  try {
1061
- const result = await deepResearchTool.execute(params);
1066
+ const result = await deepResearchTool.execute(params, ctx);
1067
+ if (isInputRequiredResult(result)) return result;
1062
1068
  return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
1063
1069
  } catch (error) {
1064
1070
  return { content: [{ type: "text", text: `Deep research failed: ${error.message}` }], isError: true };
@@ -1120,9 +1126,10 @@ registerToolIfEnabled("agent", {
1120
1126
  maxSteps: z.number().min(1).max(10).optional().default(5).describe("Max fetch iterations (hard cap: 10)"),
1121
1127
  maxUrls: z.number().min(1).max(20).optional().default(10).describe("Max URLs to fetch (hard cap: 20)")
1122
1128
  }
1123
- }, withAuth("agent", async (params) => {
1129
+ }, withAuth("agent", async (params, ctx) => {
1124
1130
  try {
1125
- const result = await agentTool.execute(params);
1131
+ const result = await agentTool.execute(params, ctx);
1132
+ if (isInputRequiredResult(result)) return result;
1126
1133
  return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
1127
1134
  } catch (error) {
1128
1135
  return { content: [{ type: "text", text: `Agent failed: ${error.message}` }], isError: true };
@@ -1131,103 +1138,11 @@ registerToolIfEnabled("agent", {
1131
1138
 
1132
1139
  // Tool: track_changes
1133
1140
  registerToolIfEnabled("track_changes", {
1134
- description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
1141
+ description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
1135
1142
  annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
1143
+ // The tool module owns the schema (G5); this is the same shape it validates with.
1136
1144
  inputSchema: {
1137
- url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
1138
- operation: z.enum([
1139
- 'create_baseline', 'compare', 'monitor', 'get_history', 'get_stats',
1140
- 'create_scheduled_monitor', 'stop_scheduled_monitor', 'list_scheduled_monitors', 'get_dashboard',
1141
- 'export_history', 'create_alert_rule', 'generate_trend_report', 'get_monitoring_templates'
1142
- ]).default('compare').describe("Tracking operation to perform"),
1143
- content: z.string().optional().describe("Content to compare against baseline"),
1144
- html: z.string().optional().describe("HTML content to compare against baseline"),
1145
- trackingOptions: z.object({
1146
- granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
1147
- trackText: z.boolean().default(true),
1148
- trackStructure: z.boolean().default(true),
1149
- trackAttributes: z.boolean().default(false),
1150
- trackImages: z.boolean().default(false),
1151
- trackLinks: z.boolean().default(true),
1152
- ignoreWhitespace: z.boolean().default(true),
1153
- ignoreCase: z.boolean().default(false),
1154
- customSelectors: z.array(z.string()).optional(),
1155
- excludeSelectors: z.array(z.string()).optional(),
1156
- significanceThresholds: z.object({
1157
- minor: z.number().min(0).max(1).default(0.1),
1158
- moderate: z.number().min(0).max(1).default(0.3),
1159
- major: z.number().min(0).max(1).default(0.7)
1160
- }).optional()
1161
- }).optional().describe("Options for how changes are tracked and compared"),
1162
- monitoringOptions: z.object({
1163
- enabled: z.boolean().default(false),
1164
- interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
1165
- maxRetries: z.number().min(0).max(5).default(3),
1166
- retryDelay: z.number().min(1000).max(60000).default(5000),
1167
- notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).default('moderate'),
1168
- enableWebhook: z.boolean().default(false),
1169
- webhookUrl: z.string().url().optional(),
1170
- webhookSecret: z.string().optional()
1171
- }).optional().describe("Monitoring schedule and notification settings"),
1172
- storageOptions: z.object({
1173
- enableSnapshots: z.boolean().default(true),
1174
- retainHistory: z.boolean().default(true),
1175
- maxHistoryEntries: z.number().min(1).max(1000).default(100),
1176
- compressionEnabled: z.boolean().default(true),
1177
- deltaStorageEnabled: z.boolean().default(true)
1178
- }).optional().describe("Storage and history retention settings"),
1179
- queryOptions: z.object({
1180
- limit: z.number().min(1).max(500).default(50),
1181
- offset: z.number().min(0).default(0),
1182
- startTime: z.number().optional(),
1183
- endTime: z.number().optional(),
1184
- includeContent: z.boolean().default(false),
1185
- significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
1186
- }).optional().describe("Query options for history and stats retrieval"),
1187
- notificationOptions: z.object({
1188
- webhook: z.object({
1189
- enabled: z.boolean().default(false),
1190
- url: z.string().url().optional(),
1191
- method: z.enum(['POST', 'PUT']).default('POST'),
1192
- headers: z.record(z.string()).optional(),
1193
- signingSecret: z.string().optional(),
1194
- includeContent: z.boolean().default(false)
1195
- }).optional(),
1196
- slack: z.object({
1197
- enabled: z.boolean().default(false),
1198
- webhookUrl: z.string().url().optional(),
1199
- channel: z.string().optional(),
1200
- username: z.string().optional()
1201
- }).optional()
1202
- }).optional().describe("Notification configuration for webhooks and Slack"),
1203
- scheduledMonitorOptions: z.object({
1204
- schedule: z.string().optional().describe("Optional cron expression (power users)"),
1205
- templateId: z.string().optional(),
1206
- enabled: z.boolean().default(true),
1207
- interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
1208
- goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
1209
- monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
1210
- notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional()
1211
- }).optional().describe("Scheduled monitoring: recurring compare + notify, optional plain-English goal"),
1212
- alertRuleOptions: z.object({
1213
- ruleId: z.string().optional(),
1214
- condition: z.string().optional(),
1215
- actions: z.array(z.enum(['webhook', 'email', 'slack'])).optional(),
1216
- throttle: z.number().min(0).optional(),
1217
- priority: z.enum(['low', 'medium', 'high']).optional()
1218
- }).optional().describe("Alert rule configuration for change notifications"),
1219
- exportOptions: z.object({
1220
- format: z.enum(['json', 'csv']).default('json'),
1221
- startTime: z.number().optional(),
1222
- endTime: z.number().optional(),
1223
- includeContent: z.boolean().default(false),
1224
- includeSnapshots: z.boolean().default(false)
1225
- }).optional().describe("Export options for change history data"),
1226
- dashboardOptions: z.object({
1227
- includeRecentAlerts: z.boolean().default(true),
1228
- includeTrends: z.boolean().default(true),
1229
- includeMonitorStatus: z.boolean().default(true)
1230
- }).optional().describe("Dashboard display options"),
1145
+ ...TRACK_CHANGES_INPUT_SHAPE,
1231
1146
  ...COMPLIANCE_PARAMS
1232
1147
  }
1233
1148
  }, withAuth("track_changes", async (params) => {
@@ -0,0 +1,176 @@
1
+ /**
2
+ * login command — browser handoff that stores an API key in ~/.crawlforge/config.json.
3
+ *
4
+ * The CLI mints PKCE parameters, prints an approval URL for the human, and polls
5
+ * the website until the signed-in user approves; the key is delivered once, over
6
+ * the poll, and never typed into a terminal. This command stores the credential
7
+ * ONLY — registering the MCP server with a client is `crawlforge init`.
8
+ */
9
+ import { randomBytes, createHash } from 'node:crypto';
10
+ import { hostname } from 'node:os';
11
+ import { existsSync } from 'node:fs';
12
+ import authManager from '../../core/authManager.js';
13
+ import { resolveApiEndpoint } from '../../core/endpointGuard.js';
14
+
15
+ const POLL_INTERVAL_MS = 3000;
16
+ const MAX_BACKOFF_MS = 30000;
17
+ const MAX_CONSECUTIVE_FAILURES = 10;
18
+
19
+ export function generateLoginParams() {
20
+ const codeVerifier = randomBytes(32).toString('base64url');
21
+ return {
22
+ sessionId: randomBytes(16).toString('hex'),
23
+ codeVerifier,
24
+ codeChallenge: createHash('sha256').update(codeVerifier).digest('base64url'),
25
+ };
26
+ }
27
+
28
+ export function buildApprovalUrl(endpoint, params, name) {
29
+ return `${endpoint}/cli-auth?session_id=${params.sessionId}` +
30
+ `&code_challenge=${params.codeChallenge}&name=${encodeURIComponent(name)}`;
31
+ }
32
+
33
+ function loginError(code, message) {
34
+ const err = new Error(message);
35
+ err.code = code;
36
+ return err;
37
+ }
38
+
39
+ /**
40
+ * Poll the status endpoint until the key arrives. Resolves with the `complete`
41
+ * payload; rejects with an error whose `code` names why (CLI_AUTH_TIMEOUT,
42
+ * CLI_AUTH_VERIFIER_MISMATCH, CLI_AUTH_UNREACHABLE). `sleep` and `now` are
43
+ * injectable so tests neither wait nor hit the network.
44
+ */
45
+ export async function pollStatus(fetchImpl, endpoint, params, {
46
+ intervalMs = POLL_INTERVAL_MS,
47
+ timeoutMs = 600000,
48
+ requestTimeoutMs = 30000,
49
+ sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
50
+ now = Date.now,
51
+ } = {}) {
52
+ const deadline = now() + timeoutMs;
53
+ let interval = intervalMs;
54
+ let failures = 0;
55
+ let lastFailure = '';
56
+
57
+ while (now() < deadline) {
58
+ let response;
59
+ try {
60
+ response = await fetchImpl(`${endpoint}/api/auth/cli/status`, {
61
+ method: 'POST',
62
+ headers: { 'Content-Type': 'application/json' },
63
+ body: JSON.stringify({ session_id: params.sessionId, code_verifier: params.codeVerifier }),
64
+ signal: AbortSignal.timeout(requestTimeoutMs),
65
+ });
66
+ } catch (err) {
67
+ response = null;
68
+ lastFailure = err.message;
69
+ }
70
+
71
+ if (response && response.status === 200) {
72
+ const body = await response.json();
73
+ if (body.status === 'complete') return body;
74
+ if (body.status === 'pending') {
75
+ failures = 0;
76
+ } else {
77
+ failures++;
78
+ lastFailure = `unexpected status "${body.status}"`;
79
+ }
80
+ } else if (response && response.status === 403) {
81
+ let code = 'CLI_AUTH_FORBIDDEN';
82
+ try { code = (await response.json()).error?.code || code; } catch { /* keep default */ }
83
+ throw loginError(code, code === 'CLI_AUTH_VERIFIER_MISMATCH'
84
+ ? 'The website rejected this session\'s verifier. Run crawlforge login again and open the new URL.'
85
+ : `The website refused the login session (${code}).`);
86
+ } else if (response && response.status === 429) {
87
+ interval = Math.min(interval * 2, MAX_BACKOFF_MS);
88
+ } else if (response) {
89
+ failures++;
90
+ lastFailure = `HTTP ${response.status}`;
91
+ } else {
92
+ failures++;
93
+ }
94
+
95
+ if (failures >= MAX_CONSECUTIVE_FAILURES) {
96
+ throw loginError('CLI_AUTH_UNREACHABLE',
97
+ `Gave up after ${failures} consecutive failed status checks (last: ${lastFailure}).`);
98
+ }
99
+ await sleep(interval);
100
+ }
101
+
102
+ throw loginError('CLI_AUTH_TIMEOUT',
103
+ `No approval within ${Math.round(timeoutMs / 1000)} seconds. Run crawlforge login again.`);
104
+ }
105
+
106
+ export function register(program) {
107
+ program
108
+ .command('login')
109
+ .description('Sign in through your browser and store an API key in ~/.crawlforge/config.json (does not touch client configs)')
110
+ .option('--name <name>', 'Name for the API key the approval creates', `CLI on ${hostname()}`)
111
+ // Not `--timeout`: the program-level `--timeout <ms>` parses argv first and
112
+ // would swallow the value, so a subcommand option of that name never gets one.
113
+ .option('--wait <seconds>', 'How long to wait for approval', '600')
114
+ .action(async (opts, cmd) => {
115
+ const json = cmd.parent.opts().json;
116
+ const out = (msg) => process.stderr.write(msg + '\n');
117
+ const fail = (code, message) => {
118
+ if (json) {
119
+ process.stdout.write(JSON.stringify({ status: 'error', code, message }) + '\n', () => process.exit(1));
120
+ } else {
121
+ out('Error: ' + message);
122
+ process.exit(1);
123
+ }
124
+ };
125
+ process.on('SIGINT', () => fail('CLI_AUTH_CANCELLED', 'Login cancelled.'));
126
+
127
+ // resolveApiEndpoint() returns the origin with a trailing slash; strip it
128
+ // so the approval URL and the status endpoint are not `host//path`.
129
+ const endpoint = resolveApiEndpoint(process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev').replace(/\/+$/, '');
130
+ const params = generateLoginParams();
131
+ const hadConfig = existsSync(authManager.configPath);
132
+
133
+ out('Open this URL in your browser and approve the key (session ' + params.sessionId.slice(0, 8) + '):');
134
+ out('');
135
+ out(' ' + buildApprovalUrl(endpoint, params, opts.name));
136
+ out('');
137
+ out('Waiting for approval… (Ctrl-C to cancel)');
138
+
139
+ let result;
140
+ try {
141
+ result = await pollStatus(fetch, endpoint, params, {
142
+ timeoutMs: parseInt(opts.wait, 10) * 1000,
143
+ requestTimeoutMs: parseInt(process.env.CRAWLFORGE_CLI_TIMEOUT || '30000', 10),
144
+ });
145
+ } catch (err) {
146
+ return fail(err.code || 'CLI_AUTH_FAILED', err.message);
147
+ }
148
+
149
+ const validation = await authManager.validateApiKey(result.api_key);
150
+ if (!validation.valid) {
151
+ return fail('CLI_AUTH_KEY_INVALID', 'The delivered API key failed validation: ' + validation.error);
152
+ }
153
+ await authManager.saveConfig(result.api_key, validation.userId, validation.email);
154
+
155
+ if (json) {
156
+ const line = JSON.stringify({
157
+ status: 'complete',
158
+ email: validation.email,
159
+ key_name: result.key_name,
160
+ config_path: authManager.configPath,
161
+ credits_remaining: validation.creditsRemaining,
162
+ plan: validation.planId,
163
+ });
164
+ process.stdout.write(line + '\n', () => process.exit(0));
165
+ return;
166
+ }
167
+
168
+ out('Signed in as ' + validation.email);
169
+ out('Credits remaining: ' + validation.creditsRemaining);
170
+ out('Plan: ' + validation.planId);
171
+ out('API key "' + result.key_name + '" saved to ' + authManager.configPath +
172
+ (hadConfig ? ' (replaced the previous config)' : ''));
173
+ out('Next: crawlforge init --client claude-code|claude-desktop|cursor registers the MCP server with a client (this command did not modify any client config).');
174
+ process.exit(0);
175
+ });
176
+ }
@@ -61,19 +61,33 @@ export function register(program) {
61
61
  .option('--threshold <level>', 'Notification threshold: minor|moderate|major|critical', 'moderate')
62
62
  .option('--cron <expr>', 'Optional cron expression (advanced)')
63
63
  .option('--selector <css>', 'CSS selector to scope monitoring')
64
+ .option('--hosted', "Run the monitor on CrawlForge's servers (fires without this process; email + signed webhooks; 3 credits per compared target per check)")
65
+ .option('--email <addresses>', 'Comma-separated notification emails (sent by hosted monitors only)')
66
+ .option('--name <text>', 'Display name for a hosted monitor (default: the URL host)')
64
67
  .action(async (url, opts) => {
65
68
  const tool = new TrackChangesTool(getToolConfig('track_changes'));
69
+ const notificationOptions = {
70
+ ...(opts.webhook ? { webhook: { enabled: true, url: opts.webhook } } : {}),
71
+ ...(opts.email ? { email: { enabled: true, recipients: opts.email.split(',').map((s) => s.trim()).filter(Boolean) } } : {})
72
+ };
73
+ if (opts.email && !opts.hosted) {
74
+ process.stderr.write('Warning: local monitors do not send email; add --hosted for --email to take effect.\n');
75
+ }
66
76
  try {
67
77
  const res = await tool.execute({
68
78
  url,
69
79
  operation: 'create_scheduled_monitor',
70
80
  ...(opts.selector ? { trackingOptions: { customSelectors: [opts.selector] } } : {}),
71
- ...(opts.webhook ? { notificationOptions: { webhook: { enabled: true, url: opts.webhook } } } : {}),
81
+ ...(Object.keys(notificationOptions).length ? { notificationOptions } : {}),
72
82
  scheduledMonitorOptions: {
73
83
  interval: Math.max(parseInt(opts.every, 10), 60) * 1000,
74
84
  ...(opts.goal ? { goal: opts.goal } : {}),
75
85
  ...(opts.cron ? { schedule: opts.cron } : {}),
76
- notificationThreshold: opts.threshold
86
+ ...(opts.hosted ? { hosted: true } : {}),
87
+ ...(opts.name ? { name: opts.name } : {}),
88
+ // Local only: a hosted check has no significance threshold, and
89
+ // the option's default would otherwise warn on every hosted create.
90
+ ...(opts.hosted ? {} : { notificationThreshold: opts.threshold })
77
91
  }
78
92
  });
79
93
  emit(res);
@@ -86,7 +100,7 @@ export function register(program) {
86
100
 
87
101
  program
88
102
  .command('monitor:list')
89
- .description('List persisted scheduled monitors')
103
+ .description('List scheduled monitors (local and hosted)')
90
104
  .action(async () => {
91
105
  const tool = new TrackChangesTool(getToolConfig('track_changes'));
92
106
  try {
@@ -100,7 +114,7 @@ export function register(program) {
100
114
 
101
115
  program
102
116
  .command('monitor:stop <id>')
103
- .description('Stop and remove a scheduled monitor by id')
117
+ .description('Stop and remove a scheduled monitor by id (local or hosted)')
104
118
  .action(async (id) => {
105
119
  const tool = new TrackChangesTool(getToolConfig('track_changes'));
106
120
  try {
package/src/cli/index.js CHANGED
@@ -59,6 +59,7 @@ import { register as registerMonitor } from './commands/monitor.js';
59
59
  import { register as registerInstallSkills } from './commands/install-skills.js';
60
60
  import { register as registerUninstallSkills } from './commands/uninstall-skills.js';
61
61
  import { register as registerInit } from './commands/init.js';
62
+ import { register as registerLogin } from './commands/login.js';
62
63
 
63
64
  // ─── MCP stdio server mode (backward compatibility) ──────────────────────────
64
65
  // Before v4.1.0 the `crawlforge` bin WAS the MCP server. v4.1.0 turned it into
@@ -138,6 +139,7 @@ registerMonitor(program);
138
139
  registerInstallSkills(program);
139
140
  registerUninstallSkills(program);
140
141
  registerInit(program);
142
+ registerLogin(program);
141
143
 
142
144
  // `crawlforge mcp` / `crawlforge serve` — explicitly start the MCP server over
143
145
  // stdio. Extra args (e.g. --http) are read directly by server.js from argv.
@@ -241,9 +241,16 @@ class AuthManager {
241
241
  }
242
242
 
243
243
  /**
244
- * Check if user has enough credits for a tool
244
+ * Check if user has enough credits for a tool.
245
+ *
246
+ * @param {number} estimatedCredits
247
+ * @param {object} [ctx] the SDK per-request context, so the low-credit
248
+ * warning can ask as a multi-round-trip step (Phase 4.4).
249
+ * @returns {Promise<boolean|object>} `true`/`false` as before, or an
250
+ * `input_required` result the caller must RETURN verbatim — `withAuth`
251
+ * detects it, bills nothing and lets the SDK gather the answer.
245
252
  */
246
- async checkCredits(estimatedCredits = 1) {
253
+ async checkCredits(estimatedCredits = 1, ctx) {
247
254
  // Creator mode has unlimited credits
248
255
  if (this.isCreatorMode()) {
249
256
  return true;
@@ -277,10 +284,16 @@ class AuthManager {
277
284
  this.lastCreditCheck = now;
278
285
  this.lastSuccessfulCreditCheck.set(this.config.userId, now);
279
286
 
280
- // D1.4: If credits are close to running out, elicit confirmation instead of hard-failing
287
+ // D1.4: If credits are close to running out, elicit confirmation instead
288
+ // of hard-failing. Phase 4.4: the ask is a round trip now — `ask` hands
289
+ // an `input_required` result back to withAuth, which returns it unbilled
290
+ // and is re-entered here with the answer. A client that cannot be asked
291
+ // still proceeds, which is what the inline helper did.
281
292
  if (data.creditsRemaining < estimatedCredits) {
282
293
  if (this._elicitation) {
283
- const proceed = await this._elicitation.confirm(
294
+ const gate = this._elicitation.confirm(
295
+ ctx,
296
+ 'credits:low',
284
297
  `Low credits: ${data.creditsRemaining} remaining, this tool needs ~${estimatedCredits}. Proceed anyway?`,
285
298
  {
286
299
  credits_remaining: data.creditsRemaining,
@@ -288,8 +301,8 @@ class AuthManager {
288
301
  note: 'Top up at https://www.crawlforge.dev/dashboard',
289
302
  }
290
303
  );
291
- if (!proceed) return false;
292
- return true; // user confirmed — let tool attempt it
304
+ if (gate.status === 'ask') return gate.result;
305
+ return gate.status === 'proceed'; // confirmed (or unaskable) — let tool attempt it
293
306
  }
294
307
  return false; // no elicitation — standard hard-fail behavior
295
308
  }