crawlforge-mcp-server 6.0.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -83,41 +83,51 @@ export class BatchScrapeTool extends EventEmitter {
83
83
  this._elicitation = new ElicitationHelper({ mcpServer });
84
84
  }
85
85
 
86
- async execute(params) {
86
+ async execute(params, ctx) {
87
87
  try {
88
88
  const validated = BatchScrapeSchema.parse(params);
89
- this.stats.totalBatches++;
90
89
  const batchId = this._generateBatchId();
91
- const startTime = Date.now();
92
-
93
- this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
94
-
95
- const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
96
90
 
97
- let webhookConfig = null;
98
- if (validated.webhook && this.enableWebhookNotifications) {
99
- webhookConfig = this._registerWebhook(validated.webhook, batchId);
100
- }
101
-
102
- // D1.4: Elicitation warn when batch is large in sync mode
103
- if (validated.mode === 'sync' && urlConfigs.length > 25) {
104
- const proceed = await this._elicitation.confirm(
105
- `batch_scrape (sync mode) will fetch ${urlConfigs.length} URLs synchronously. This may take a while and consume significant credits.`,
91
+ // D1.4: Elicitation — warn when batch is large in sync mode. An
92
+ // unanswered gate returns an input-required result the SDK answers by
93
+ // re-entering this handler from the top, so it runs before anything that
94
+ // leaves a trace: below it are the batch counter and the webhook
95
+ // registration, which a second entry would repeat (a second webhook
96
+ // registration for one batch). The count comes from validated.urls,
97
+ // which _normalizeUrlConfigs maps one-for-one.
98
+ if (validated.mode === 'sync' && validated.urls.length > 25) {
99
+ const gate = this._elicitation.confirm(
100
+ ctx,
101
+ 'batch_scrape:large_sync',
102
+ `batch_scrape (sync mode) will fetch ${validated.urls.length} URLs synchronously. This may take a while and consume significant credits.`,
106
103
  {
107
- url_count: urlConfigs.length,
104
+ url_count: validated.urls.length,
108
105
  mode: 'sync',
109
106
  suggestion: 'Consider using mode:"async" for large batches.',
110
107
  }
111
108
  );
112
- if (!proceed) {
109
+ if (gate.status === 'ask') return gate.result;
110
+ if (gate.status === 'cancelled') {
113
111
  return {
114
112
  batchId, mode: 'sync', success: false,
115
113
  error: 'Batch scrape cancelled by user (elicitation declined).',
116
- totalUrls: urlConfigs.length,
114
+ totalUrls: validated.urls.length,
117
115
  };
118
116
  }
119
117
  }
120
118
 
119
+ this.stats.totalBatches++;
120
+ const startTime = Date.now();
121
+
122
+ this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
123
+
124
+ const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
125
+
126
+ let webhookConfig = null;
127
+ if (validated.webhook && this.enableWebhookNotifications) {
128
+ webhookConfig = this._registerWebhook(validated.webhook, batchId);
129
+ }
130
+
121
131
  if (validated.mode === 'sync') {
122
132
  return await this._processBatchSync(batchId, urlConfigs, validated, webhookConfig, startTime);
123
133
  } else {
@@ -35,16 +35,21 @@ export class AgentTool {
35
35
  this._elicitation = new ElicitationHelper({ mcpServer });
36
36
  }
37
37
 
38
- async execute(params) {
38
+ async execute(params, ctx) {
39
39
  const validated = AgentInputSchema.parse(params);
40
40
 
41
- // Request confirmation before a pro run (expensive)
41
+ // Request confirmation before a pro run (expensive). An unanswered gate
42
+ // returns an input-required result the SDK answers by re-entering this
43
+ // handler from the top, so nothing above it may fetch or leave a trace.
42
44
  if (validated.model === 'pro') {
43
- const proceed = await this._elicitation.confirm(
45
+ const gate = this._elicitation.confirm(
46
+ ctx,
47
+ 'agent:pro_model',
44
48
  'agent tool: pro model uses ResearchOrchestrator and may incur significant costs.',
45
49
  { model: 'pro', maxUrls: validated.maxUrls, note: 'External LLM API costs billed separately if keys are set.' }
46
50
  );
47
- if (!proceed) {
51
+ if (gate.status === 'ask') return gate.result;
52
+ if (gate.status === 'cancelled') {
48
53
  return {
49
54
  success: false,
50
55
  cancelled: true,
@@ -117,7 +117,7 @@ export class CrawlDeepTool {
117
117
  this._elicitation = new ElicitationHelper({ mcpServer });
118
118
  }
119
119
 
120
- async execute(params) {
120
+ async execute(params, ctx) {
121
121
  try {
122
122
  const validated = CrawlDeepSchema.parse(params);
123
123
 
@@ -147,9 +147,14 @@ export class CrawlDeepTool {
147
147
  if (cached) return { ...cached, cached: true };
148
148
  }
149
149
 
150
- // D1.4: Elicitation — warn when max_pages is very high
150
+ // D1.4: Elicitation — warn when max_pages is very high. An unanswered
151
+ // gate returns an input-required result the SDK answers by re-entering
152
+ // this handler from the top; everything above is the clamp arithmetic and
153
+ // a cache read, so a second entry fetches nothing and leaves no trace.
151
154
  if (effectiveMaxPages > 500) {
152
- const proceed = await this._elicitation.confirm(
155
+ const gate = this._elicitation.confirm(
156
+ ctx,
157
+ 'crawl_deep:max_pages',
153
158
  `crawl_deep will crawl up to ${effectiveMaxPages} pages from ${validated.url}. Large crawls consume many credits.`,
154
159
  {
155
160
  url: validated.url,
@@ -157,7 +162,8 @@ export class CrawlDeepTool {
157
162
  max_depth: effectiveMaxDepth,
158
163
  }
159
164
  );
160
- if (!proceed) {
165
+ if (gate.status === 'ask') return gate.result;
166
+ if (gate.status === 'cancelled') {
161
167
  return {
162
168
  success: false,
163
169
  error: 'Crawl cancelled by user (elicitation declined).',
@@ -174,36 +174,78 @@ export class ExtractStructuredTool {
174
174
  * @param {Object} params - Extraction parameters
175
175
  * @returns {Promise<Object>} Extraction result
176
176
  */
177
- async execute(params) {
177
+ async execute(params, ctx) {
178
178
  const startTime = Date.now();
179
179
 
180
180
  try {
181
181
  const validated = ExtractStructuredSchema.parse(params);
182
182
  const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
183
183
 
184
- // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
185
- const { html, $, textContent, warnings } = await fetchAndParse(url, {
186
- userAgent: user_agent || this.userAgent,
187
- respectRobots: respect_robots,
188
- tool: 'extract_structured'
189
- });
190
-
191
- // What the model reads — see shownText().
192
- const shown = shownText($, html, url, textContent);
193
-
194
- // Step 3: Try LLM extraction first
195
184
  let extractionResult = null;
196
185
  let extractionMethod = 'llm';
197
186
  let llmErrorMessage = null;
198
187
  let llmAvailable = false;
188
+ let llm = null;
199
189
 
190
+ // Step 0: LLM readiness, resolved before the fetch so the D1.4 gate below
191
+ // can ask before any network work — an unanswered gate returns an
192
+ // input-required result the SDK answers by re-entering this handler from
193
+ // the top, and everything above the gate runs a second time.
200
194
  try {
201
- const llm = this._ensureLLMManager(llmConfig || {});
195
+ llm = this._ensureLLMManager(llmConfig || {});
202
196
  // ready() probes Ollama, which has no API key to gate on. isAvailable()
203
197
  // alone reported false on any machine without a cloud key, so a running
204
198
  // local Ollama was never used.
205
199
  llmAvailable = await llm.ready();
206
- if (llmAvailable) {
200
+ } catch (llmError) {
201
+ // No usable LLM — this falls through to the CSS fallback. Keep the
202
+ // message so callers can tell "LLM broken" apart from "no LLM
203
+ // configured".
204
+ llmErrorMessage = llmError.message;
205
+ }
206
+
207
+ // D1.4: no LLM configured and the schema demands more than 3 required
208
+ // fields — confirm before running the lower-fidelity CSS fallback. With
209
+ // no LLM, step 3 extracts nothing, so landing on that fallback is already
210
+ // settled here, before the page is fetched.
211
+ const requiredCount = (schema.required || []).length;
212
+ if (fallbackToSelectors !== false && !llmAvailable && requiredCount > 3) {
213
+ const gate = this._elicitation.confirm(
214
+ ctx,
215
+ 'extract_structured:no_llm_required_fields',
216
+ `No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
217
+ `extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
218
+ { url, required_fields: requiredCount }
219
+ );
220
+ if (gate.status === 'ask') return gate.result;
221
+ if (gate.status === 'cancelled') {
222
+ return {
223
+ success: false,
224
+ url,
225
+ data: {},
226
+ extraction_method: 'none',
227
+ confidence: 0,
228
+ schema_used: schema,
229
+ processingTime: Date.now() - startTime,
230
+ error: 'Extraction cancelled by user (elicitation declined).',
231
+ validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
232
+ };
233
+ }
234
+ }
235
+
236
+ // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
237
+ const { html, $, textContent, warnings } = await fetchAndParse(url, {
238
+ userAgent: user_agent || this.userAgent,
239
+ respectRobots: respect_robots,
240
+ tool: 'extract_structured'
241
+ });
242
+
243
+ // What the model reads — see shownText().
244
+ const shown = shownText($, html, url, textContent);
245
+
246
+ // Step 3: Try LLM extraction first (readiness resolved in step 0)
247
+ if (llmAvailable) {
248
+ try {
207
249
  const result = await llm.extractStructured(shown, schema, {
208
250
  prompt: prompt || '',
209
251
  maxContentLength: SHOWN_TEXT_BUDGET
@@ -218,12 +260,11 @@ export class ExtractStructuredTool {
218
260
  } else {
219
261
  llmErrorMessage = result?.error || 'LLM did not return usable JSON';
220
262
  }
263
+ } catch (llmError) {
264
+ // LLM failed — will fall through to CSS fallback.
265
+ extractionResult = null;
266
+ llmErrorMessage = llmError.message;
221
267
  }
222
- } catch (llmError) {
223
- // LLM failed — will fall through to CSS fallback. Keep the message so
224
- // callers can tell "LLM broken" apart from "no LLM configured".
225
- extractionResult = null;
226
- llmErrorMessage = llmError.message;
227
268
  }
228
269
 
229
270
  // Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
@@ -318,31 +359,9 @@ export class ExtractStructuredTool {
318
359
  if (guarded.checked.skipped) provenance.skipped = guarded.checked.skipped;
319
360
  }
320
361
 
321
- // Step 4: CSS selector fallback if LLM unavailable or failed
362
+ // Step 4: CSS selector fallback if LLM unavailable or failed (the D1.4
363
+ // confirmation for this path is gated above, before the fetch)
322
364
  if (!extractionResult && fallbackToSelectors !== false) {
323
- // D1.4: no LLM configured and the schema demands more than 3 required
324
- // fields — confirm before running the lower-fidelity CSS fallback.
325
- const requiredCount = (schema.required || []).length;
326
- if (!llmAvailable && requiredCount > 3) {
327
- const proceed = await this._elicitation.confirm(
328
- `No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
329
- `extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
330
- { url, required_fields: requiredCount }
331
- );
332
- if (!proceed) {
333
- return {
334
- success: false,
335
- url,
336
- data: {},
337
- extraction_method: 'none',
338
- confidence: 0,
339
- schema_used: schema,
340
- processingTime: Date.now() - startTime,
341
- error: 'Extraction cancelled by user (elicitation declined).',
342
- validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
343
- };
344
- }
345
- }
346
365
  extractionResult = this._cssExtraction($, schema, selectorHints || {});
347
366
  extractionMethod = 'css_fallback';
348
367
  }
@@ -115,7 +115,7 @@ export class DeepResearchTool {
115
115
  this._elicitation = new ElicitationHelper({ mcpServer });
116
116
  }
117
117
 
118
- async execute(params) {
118
+ async execute(params, ctx) {
119
119
  try {
120
120
  const validated = DeepResearchSchema.parse(params);
121
121
  const sessionId = this.generateSessionId();
@@ -137,10 +137,15 @@ export class DeepResearchTool {
137
137
  }
138
138
 
139
139
  // D1.4: Elicitation — warn user if projected cost exceeds 50 credits
140
- // deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm
140
+ // deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm.
141
+ // An unanswered gate returns an input-required result the SDK answers by
142
+ // re-entering this handler from the top, so it sits above the session
143
+ // registration below — a second entry would otherwise leak a session.
141
144
  if (validated.maxUrls > 50) {
142
145
  const projectedCredits = validated.maxUrls;
143
- const proceed = await this._elicitation.confirm(
146
+ const gate = this._elicitation.confirm(
147
+ ctx,
148
+ 'deep_research:max_urls',
144
149
  `deep_research will scan up to ${validated.maxUrls} URLs, projecting ~${projectedCredits} credits.`,
145
150
  {
146
151
  topic: validated.topic,
@@ -148,7 +153,8 @@ export class DeepResearchTool {
148
153
  max_urls: validated.maxUrls,
149
154
  }
150
155
  );
151
- if (!proceed) {
156
+ if (gate.status === 'ask') return gate.result;
157
+ if (gate.status === 'cancelled') {
152
158
  return {
153
159
  success: false,
154
160
  error: 'Research cancelled by user before starting (elicitation declined).',
@@ -0,0 +1,176 @@
1
+ /**
2
+ * TrackChanges — hosted monitors (Phase 6.1).
3
+ *
4
+ * `scheduledMonitorOptions.hosted: true` registers a monitor with the
5
+ * website's /api/v1/monitors instead of the local MonitorStore. The website's
6
+ * cron then fetches, compares, bills the account and sends the notifications
7
+ * (email, signed webhooks), so the monitor fires whether or not this process
8
+ * is alive. This module is the thin client; index.js decides when to use it
9
+ * and shapes the tool results.
10
+ *
11
+ * These calls go to our own configured backend (AuthManager.apiEndpoint, from
12
+ * CRAWLFORGE_API_URL through endpointGuard), not to a caller-supplied URL, so
13
+ * they use bare fetch with the X-API-Key header exactly as AuthManager does.
14
+ * The SSRF guard is for pages a caller names; the endpoint is legitimately
15
+ * localhost in development.
16
+ */
17
+ import authManager from '../../../core/AuthManager.js';
18
+
19
+ const HOSTED_TIMEOUT_MS = 30_000;
20
+ const MINUTE = 60_000;
21
+ const HOUR = 60 * MINUTE;
22
+
23
+ export const HOSTED_FIRING_GUARANTEE_NOTE =
24
+ "Runs from CrawlForge's scheduler whether or not this process is alive. Each check bills " +
25
+ '3 credits per compared target from the account; blocked and errored targets are free.';
26
+
27
+ export const NO_KEY_MESSAGE =
28
+ 'hosted monitors need a CrawlForge API key — run `crawlforge-setup` or `crawlforge login`';
29
+
30
+ /**
31
+ * The endpoint and key the hosted calls authenticate with, from the sources
32
+ * the server and the CLI already use: CRAWLFORGE_API_KEY (the CLI's preAction
33
+ * hook fills it from --api-key or the stored config), then the key AuthManager
34
+ * loaded at startup, then the stored config read directly — the server skips
35
+ * loading it in creator mode, and a hosted monitor is billed to an account
36
+ * either way. No network: initialize() would re-validate the key.
37
+ */
38
+ export async function resolveHostedCredentials() {
39
+ let apiKey = process.env.CRAWLFORGE_API_KEY || authManager.getConfig()?.apiKey;
40
+ if (!apiKey) {
41
+ try {
42
+ await authManager.loadConfig();
43
+ apiKey = authManager.getConfig()?.apiKey;
44
+ } catch {
45
+ /* no stored config */
46
+ }
47
+ }
48
+ if (!apiKey) throw new Error(NO_KEY_MESSAGE);
49
+ return { endpoint: authManager.apiEndpoint, apiKey };
50
+ }
51
+
52
+ // resolveApiEndpoint keeps a trailing slash on the configured endpoint; joined
53
+ // to an absolute path that is `//api/...`, a redirect on every call and a
54
+ // dashboard link with a double slash.
55
+ const base = (endpoint) => String(endpoint).replace(/\/+$/, '');
56
+
57
+ async function request(method, pathname, creds, body) {
58
+ const response = await fetch(`${base(creds.endpoint)}${pathname}`, {
59
+ method,
60
+ headers: {
61
+ 'X-API-Key': creds.apiKey,
62
+ ...(body ? { 'Content-Type': 'application/json' } : {})
63
+ },
64
+ ...(body ? { body: JSON.stringify(body) } : {}),
65
+ signal: AbortSignal.timeout(HOSTED_TIMEOUT_MS)
66
+ });
67
+ let payload = null;
68
+ try {
69
+ payload = await response.json();
70
+ } catch {
71
+ /* no JSON body */
72
+ }
73
+ if (!response.ok) {
74
+ // `{ error: { code, message, details? } }` from the monitors API; the
75
+ // API-key middleware's 401 is the same envelope. The website's own words
76
+ // reach the caller so a validation or robots refusal is readable.
77
+ const err = payload?.error;
78
+ const code = err?.code || `HTTP_${response.status}`;
79
+ const message = (typeof err === 'string' ? err : err?.message) || response.statusText || 'request failed';
80
+ const details = err?.details !== undefined ? ` ${JSON.stringify(err.details)}` : '';
81
+ const failure = new Error(`${code}: ${message}${details}`);
82
+ failure.code = code;
83
+ failure.status = response.status;
84
+ throw failure;
85
+ }
86
+ return payload?.data;
87
+ }
88
+
89
+ export function createHostedMonitor(input, creds) {
90
+ return request('POST', '/api/v1/monitors', creds, input);
91
+ }
92
+
93
+ export async function listHostedMonitors(creds) {
94
+ // An account holds at most 50 monitors, so one page is the whole list.
95
+ return (await request('GET', '/api/v1/monitors?limit=100', creds)) ?? [];
96
+ }
97
+
98
+ export function deleteHostedMonitor(id, creds) {
99
+ return request('DELETE', `/api/v1/monitors/${encodeURIComponent(id)}`, creds);
100
+ }
101
+
102
+ // The cron slots the website accepts: consecutive runs at least 5 minutes
103
+ // apart. For a `*/N` minute step that means N must divide 60 (`*/7` has a
104
+ // 4-minute gap at the top of every hour); for an hour step, H must divide 24.
105
+ const SLOTS = [
106
+ ...[5, 6, 10, 12, 15, 20, 30].map((m) => ({ ms: m * MINUTE, cron: `*/${m} * * * *` })),
107
+ { ms: HOUR, cron: '0 * * * *' },
108
+ ...[2, 3, 4, 6, 8, 12].map((h) => ({ ms: h * HOUR, cron: `0 */${h} * * *` })),
109
+ { ms: 24 * HOUR, cron: '0 0 * * *' }
110
+ ];
111
+
112
+ /**
113
+ * The hosted schedule for a polling interval in ms.
114
+ * @returns {{ cron: string, effectiveIntervalMs: number, adjusted: boolean }}
115
+ * `adjusted` is true when the interval was not an accepted slot and the
116
+ * nearest one was used; a tie goes to the longer interval (fewer billed checks).
117
+ */
118
+ export function intervalToCron(ms) {
119
+ let best = SLOTS[0];
120
+ for (const slot of SLOTS) {
121
+ const d = Math.abs(slot.ms - ms);
122
+ const bestD = Math.abs(best.ms - ms);
123
+ if (d < bestD || (d === bestD && slot.ms > best.ms)) best = slot;
124
+ }
125
+ return { cron: best.cron, effectiveIntervalMs: best.ms, adjusted: best.ms !== ms };
126
+ }
127
+
128
+ export function formatInterval(ms) {
129
+ return ms % HOUR === 0 ? `${ms / HOUR} h` : `${Math.round(ms / MINUTE)} min`;
130
+ }
131
+
132
+ const parseIso = (iso) => (iso ? Date.parse(iso) || null : null);
133
+
134
+ export function hostedDashboardUrl(endpoint, id) {
135
+ return `${base(endpoint)}/dashboard/monitors/${id}`;
136
+ }
137
+
138
+ /** The `monitor` a hosted create_scheduled_monitor returns. */
139
+ export function createdHostedMonitor(record, endpoint) {
140
+ return {
141
+ id: record.id,
142
+ hosted: true,
143
+ name: record.name,
144
+ targets: record.targets,
145
+ schedule: record.schedule_cron,
146
+ timezone: record.timezone,
147
+ notifyEmails: record.notify_emails,
148
+ webhookUrl: record.webhook_url,
149
+ webhookSecret: record.webhook_secret,
150
+ status: record.status,
151
+ nextRunAt: parseIso(record.next_run_at),
152
+ estimatedCreditsPerMonth: record.estimated_credits_per_month,
153
+ dashboardUrl: hostedDashboardUrl(endpoint, record.id)
154
+ };
155
+ }
156
+
157
+ /** A hosted monitor as list_scheduled_monitors shows it, beside the local ones. */
158
+ export function listedHostedMonitor(record, endpoint) {
159
+ const active = record.status === 'active';
160
+ return {
161
+ id: record.id,
162
+ hosted: true,
163
+ url: record.targets?.[0]?.url,
164
+ targets: record.targets,
165
+ name: record.name,
166
+ schedule: record.schedule_cron,
167
+ timezone: record.timezone,
168
+ enabled: active,
169
+ nextDueAt: parseIso(record.next_run_at),
170
+ lastCheckAt: parseIso(record.last_check_at),
171
+ lastCheck: record.last_check ?? null,
172
+ estimatedCreditsPerMonth: record.estimated_credits_per_month,
173
+ dashboardUrl: hostedDashboardUrl(endpoint, record.id),
174
+ scheduled: active
175
+ };
176
+ }
@@ -21,10 +21,18 @@ import SnapshotManager from '../../../core/SnapshotManager.js';
21
21
  import CacheManager from '../../../core/cache/CacheManager.js';
22
22
  import { MonitorStore } from '../../../core/MonitorStore.js';
23
23
  import { MonitorScheduler } from '../../../core/MonitorScheduler.js';
24
+ import { setActualCost } from '../../../server/requestContext.js';
24
25
  import { TrackChangesSchema } from './schema.js';
25
26
  import { fetchContent, mergeHistoryData, matchesSignificanceFilter, calculateAverageInterval, calculateSignificanceDistribution } from './differ.js';
26
27
  import { performMonitoringCheck, stopMonitor } from './monitor.js';
27
28
  import { sendNotifications } from './notifier.js';
29
+ import {
30
+ HOSTED_FIRING_GUARANTEE_NOTE, createHostedMonitor, createdHostedMonitor, deleteHostedMonitor,
31
+ formatInterval, intervalToCron, listHostedMonitors, listedHostedMonitor, resolveHostedCredentials
32
+ } from './hosted.js';
33
+
34
+ // server.js spreads this into the registered inputSchema (G5: one declaration).
35
+ export { TRACK_CHANGES_INPUT_SHAPE } from './schema.js';
28
36
 
29
37
  export class TrackChangesTool extends EventEmitter {
30
38
  constructor(options = {}) {
@@ -42,6 +50,9 @@ export class TrackChangesTool extends EventEmitter {
42
50
  enableRealTimeMonitoring: true,
43
51
  maxConcurrentMonitors: 50,
44
52
  defaultPollingInterval: 300000,
53
+ // The key and endpoint hosted monitors authenticate with; tests inject
54
+ // a stub so nothing reads ~/.crawlforge or reaches the website.
55
+ resolveHostedCredentials,
45
56
  ...options
46
57
  };
47
58
 
@@ -404,6 +415,9 @@ export class TrackChangesTool extends EventEmitter {
404
415
  );
405
416
  }
406
417
  }
418
+ if (opts.hosted) {
419
+ return this._createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions });
420
+ }
407
421
  // Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
408
422
  // schema fills monitoringOptions.interval/notificationThreshold with
409
423
  // defaults, so they cannot sit above a preset without always winning.
@@ -423,25 +437,127 @@ export class TrackChangesTool extends EventEmitter {
423
437
  };
424
438
  }
425
439
 
440
+ /**
441
+ * Hosted (6.1): the website's /api/v1/monitors owns the monitor — its cron
442
+ * fetches, compares, bills and notifies — so nothing is stored or fetched
443
+ * here. The interval precedence matches the local path except that
444
+ * monitoringOptions.interval (schema-defaulted to 5 min) is not consulted:
445
+ * a hosted check is billed, and the website's own default is hourly.
446
+ */
447
+ async _createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions }) {
448
+ const creds = await this.options.resolveHostedCredentials();
449
+ const warnings = [];
450
+ let scheduleCron = opts.schedule;
451
+ const interval = opts.interval ?? preset?.frequency;
452
+ if (!scheduleCron && interval) {
453
+ const slot = intervalToCron(interval);
454
+ scheduleCron = slot.cron;
455
+ if (slot.adjusted) {
456
+ warnings.push(
457
+ `interval ${formatInterval(interval)} is not a hosted schedule slot; the monitor runs every ` +
458
+ `${formatInterval(slot.effectiveIntervalMs)} (${slot.cron}). Hosted runs are at least 5 minutes apart ` +
459
+ 'and divide the hour or the day evenly.'
460
+ );
461
+ }
462
+ }
463
+ if (opts.goal ?? preset?.goal) {
464
+ warnings.push('goal is judged by the local goal judge only and is not applied to a hosted monitor, which notifies on every changed, new, blocked or errored page');
465
+ }
466
+ if (opts.notificationThreshold) {
467
+ warnings.push('notificationThreshold has no effect on a hosted monitor; hosted checks have no significance threshold');
468
+ }
469
+
470
+ const tracking = preset ? { ...preset.options, ...(trackingOptions || {}) } : (trackingOptions || {});
471
+ const selectors = tracking.customSelectors || [];
472
+ const email = notificationOptions?.email;
473
+ const webhook = notificationOptions?.webhook;
474
+ const secret = webhook?.signingSecret;
475
+ const record = await createHostedMonitor({
476
+ name: opts.name || new URL(url).host.slice(0, 80),
477
+ targets: selectors.length ? selectors.map((selector) => ({ url, selector })) : [{ url }],
478
+ ...(scheduleCron ? { schedule_cron: scheduleCron } : {}),
479
+ timezone: 'UTC',
480
+ ...(email?.enabled && email.recipients?.length ? { notify_emails: email.recipients } : {}),
481
+ ...(webhook?.enabled && webhook.url ? { webhook_url: webhook.url } : {}),
482
+ // A secret outside 16-128 chars is left out so the website generates one.
483
+ ...(webhook?.enabled && webhook.url && secret?.length >= 16 && secret.length <= 128 ? { webhook_secret: secret } : {}),
484
+ status: 'active'
485
+ }, creds);
486
+ // Nothing ran on this machine and the monitors API is free (G4).
487
+ setActualCost(0);
488
+ return {
489
+ success: true, operation: 'create_scheduled_monitor', url, hosted: true,
490
+ ...(preset ? { templateId: preset.id } : {}),
491
+ monitor: createdHostedMonitor(record, creds.endpoint),
492
+ firingGuarantee: HOSTED_FIRING_GUARANTEE_NOTE,
493
+ ...(warnings.length ? { warnings } : {}),
494
+ timestamp: Date.now()
495
+ };
496
+ }
497
+
426
498
  async stopScheduledMonitor(params) {
427
499
  const { url, scheduledMonitorOptions } = params;
428
500
  const monitorId = scheduledMonitorOptions?.monitorId;
429
501
  if (monitorId) {
430
- const result = await this.scheduler.stopMonitor(monitorId);
431
- if (!result.stopped) {
432
- return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}`, timestamp: Date.now() };
502
+ if (!this.monitorStore._loaded) await this.monitorStore.load();
503
+ if (this.monitorStore.get(monitorId)) {
504
+ await this.scheduler.stopMonitor(monitorId);
505
+ return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
433
506
  }
434
- return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
507
+ // Not in the local store: it may be hosted.
508
+ try {
509
+ await deleteHostedMonitor(monitorId, await this.options.resolveHostedCredentials());
510
+ } catch (error) {
511
+ const reason = error.status === 404 ? '' : ` (hosted lookup failed: ${error.message})`;
512
+ return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}${reason}`, timestamp: Date.now() };
513
+ }
514
+ // Nothing ran on this machine and the monitors API is free (G4).
515
+ setActualCost(0);
516
+ return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, hosted: true, timestamp: Date.now() };
435
517
  }
436
518
  if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
437
519
  const result = await this.scheduler.stopByUrl(url);
438
- return { success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, timestamp: Date.now() };
520
+ // Only a hosted monitor whose every target is this URL — never a
521
+ // multi-target monitor that merely includes it.
522
+ let stoppedHosted = 0;
523
+ let hostedError = null;
524
+ try {
525
+ const creds = await this.options.resolveHostedCredentials();
526
+ for (const m of await listHostedMonitors(creds)) {
527
+ if (m.targets?.length && m.targets.every((t) => t.url === url)) {
528
+ await deleteHostedMonitor(m.id, creds);
529
+ stoppedHosted++;
530
+ }
531
+ }
532
+ } catch (error) {
533
+ hostedError = error.message;
534
+ }
535
+ return {
536
+ success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
537
+ ...(hostedError ? { hostedError } : {}),
538
+ timestamp: Date.now()
539
+ };
439
540
  }
440
541
 
441
542
  async listScheduledMonitors() {
442
543
  if (!this.monitorStore._loaded) await this.monitorStore.load();
443
- const monitors = this.scheduler.list();
444
- return { success: true, operation: 'list_scheduled_monitors', monitors, count: monitors.length, timestamp: Date.now() };
544
+ const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
545
+ // The local list never fails because the website is unreachable.
546
+ let hosted = [];
547
+ let hostedError = null;
548
+ try {
549
+ const creds = await this.options.resolveHostedCredentials();
550
+ hosted = (await listHostedMonitors(creds)).map((r) => listedHostedMonitor(r, creds.endpoint));
551
+ } catch (error) {
552
+ hostedError = error.message;
553
+ }
554
+ const monitors = [...local, ...hosted];
555
+ return {
556
+ success: true, operation: 'list_scheduled_monitors', monitors,
557
+ count: monitors.length, localCount: local.length, hostedCount: hosted.length,
558
+ ...(hostedError ? { hostedError } : {}),
559
+ timestamp: Date.now()
560
+ };
445
561
  }
446
562
 
447
563
  async getMonitoringDashboard(params) {