crawlforge-mcp-server 6.1.0 → 6.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.1.0",
+  "version": "6.2.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -23,7 +23,7 @@ import { ListOllamaModelsTool } from "./src/tools/extract/listOllamaModels.js";
 import { BatchScrapeTool } from "./src/tools/advanced/BatchScrapeTool.js";
 import { ScrapeWithActionsTool } from "./src/tools/advanced/ScrapeWithActionsTool.js";
 import { DeepResearchTool } from "./src/tools/research/deepResearch.js";
-import { TrackChangesTool } from "./src/tools/tracking/trackChanges/index.js";
+import { TrackChangesTool, TRACK_CHANGES_INPUT_SHAPE } from "./src/tools/tracking/trackChanges/index.js";
 import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
 import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
 import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.1.0",
+  version: "6.2.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1138,103 +1138,11 @@ registerToolIfEnabled("agent", {
 
 // Tool: track_changes
 registerToolIfEnabled("track_changes", {
-  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
+  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
   annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
+  // The tool module owns the schema (G5); this is the same shape it validates with.
   inputSchema: {
-    url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
-    operation: z.enum([
-      'create_baseline', 'compare', 'monitor', 'get_history', 'get_stats',
-      'create_scheduled_monitor', 'stop_scheduled_monitor', 'list_scheduled_monitors', 'get_dashboard',
-      'export_history', 'create_alert_rule', 'generate_trend_report', 'get_monitoring_templates'
-    ]).default('compare').describe("Tracking operation to perform"),
-    content: z.string().optional().describe("Content to compare against baseline"),
-    html: z.string().optional().describe("HTML content to compare against baseline"),
-    trackingOptions: z.object({
-      granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
-      trackText: z.boolean().default(true),
-      trackStructure: z.boolean().default(true),
-      trackAttributes: z.boolean().default(false),
-      trackImages: z.boolean().default(false),
-      trackLinks: z.boolean().default(true),
-      ignoreWhitespace: z.boolean().default(true),
-      ignoreCase: z.boolean().default(false),
-      customSelectors: z.array(z.string()).optional(),
-      excludeSelectors: z.array(z.string()).optional(),
-      significanceThresholds: z.object({
-        minor: z.number().min(0).max(1).default(0.1),
-        moderate: z.number().min(0).max(1).default(0.3),
-        major: z.number().min(0).max(1).default(0.7)
-      }).optional()
-    }).optional().describe("Options for how changes are tracked and compared"),
-    monitoringOptions: z.object({
-      enabled: z.boolean().default(false),
-      interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
-      maxRetries: z.number().min(0).max(5).default(3),
-      retryDelay: z.number().min(1000).max(60000).default(5000),
-      notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).default('moderate'),
-      enableWebhook: z.boolean().default(false),
-      webhookUrl: z.string().url().optional(),
-      webhookSecret: z.string().optional()
-    }).optional().describe("Monitoring schedule and notification settings"),
-    storageOptions: z.object({
-      enableSnapshots: z.boolean().default(true),
-      retainHistory: z.boolean().default(true),
-      maxHistoryEntries: z.number().min(1).max(1000).default(100),
-      compressionEnabled: z.boolean().default(true),
-      deltaStorageEnabled: z.boolean().default(true)
-    }).optional().describe("Storage and history retention settings"),
-    queryOptions: z.object({
-      limit: z.number().min(1).max(500).default(50),
-      offset: z.number().min(0).default(0),
-      startTime: z.number().optional(),
-      endTime: z.number().optional(),
-      includeContent: z.boolean().default(false),
-      significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
-    }).optional().describe("Query options for history and stats retrieval"),
-    notificationOptions: z.object({
-      webhook: z.object({
-        enabled: z.boolean().default(false),
-        url: z.string().url().optional(),
-        method: z.enum(['POST', 'PUT']).default('POST'),
-        headers: z.record(z.string()).optional(),
-        signingSecret: z.string().optional(),
-        includeContent: z.boolean().default(false)
-      }).optional(),
-      slack: z.object({
-        enabled: z.boolean().default(false),
-        webhookUrl: z.string().url().optional(),
-        channel: z.string().optional(),
-        username: z.string().optional()
-      }).optional()
-    }).optional().describe("Notification configuration for webhooks and Slack"),
-    scheduledMonitorOptions: z.object({
-      schedule: z.string().optional().describe("Optional cron expression (power users)"),
-      templateId: z.string().optional(),
-      enabled: z.boolean().default(true),
-      interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
-      goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
-      monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
-      notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional()
-    }).optional().describe("Scheduled monitoring: recurring compare + notify, optional plain-English goal"),
-    alertRuleOptions: z.object({
-      ruleId: z.string().optional(),
-      condition: z.string().optional(),
-      actions: z.array(z.enum(['webhook', 'email', 'slack'])).optional(),
-      throttle: z.number().min(0).optional(),
-      priority: z.enum(['low', 'medium', 'high']).optional()
-    }).optional().describe("Alert rule configuration for change notifications"),
-    exportOptions: z.object({
-      format: z.enum(['json', 'csv']).default('json'),
-      startTime: z.number().optional(),
-      endTime: z.number().optional(),
-      includeContent: z.boolean().default(false),
-      includeSnapshots: z.boolean().default(false)
-    }).optional().describe("Export options for change history data"),
-    dashboardOptions: z.object({
-      includeRecentAlerts: z.boolean().default(true),
-      includeTrends: z.boolean().default(true),
-      includeMonitorStatus: z.boolean().default(true)
-    }).optional().describe("Dashboard display options"),
+    ...TRACK_CHANGES_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS
   }
 }, withAuth("track_changes", async (params) => {

package/src/cli/commands/monitor.js

@@ -61,19 +61,33 @@ export function register(program) {
     .option('--threshold <level>', 'Notification threshold: minor|moderate|major|critical', 'moderate')
     .option('--cron <expr>', 'Optional cron expression (advanced)')
     .option('--selector <css>', 'CSS selector to scope monitoring')
+    .option('--hosted', "Run the monitor on CrawlForge's servers (fires without this process; email + signed webhooks; 3 credits per compared target per check)")
+    .option('--email <addresses>', 'Comma-separated notification emails (sent by hosted monitors only)')
+    .option('--name <text>', 'Display name for a hosted monitor (default: the URL host)')
     .action(async (url, opts) => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      const notificationOptions = {
+        ...(opts.webhook ? { webhook: { enabled: true, url: opts.webhook } } : {}),
+        ...(opts.email ? { email: { enabled: true, recipients: opts.email.split(',').map((s) => s.trim()).filter(Boolean) } } : {})
+      };
+      if (opts.email && !opts.hosted) {
+        process.stderr.write('Warning: local monitors do not send email; add --hosted for --email to take effect.\n');
+      }
       try {
         const res = await tool.execute({
           url,
           operation: 'create_scheduled_monitor',
           ...(opts.selector ? { trackingOptions: { customSelectors: [opts.selector] } } : {}),
-          ...(opts.webhook ? { notificationOptions: { webhook: { enabled: true, url: opts.webhook } } } : {}),
+          ...(Object.keys(notificationOptions).length ? { notificationOptions } : {}),
           scheduledMonitorOptions: {
             interval: Math.max(parseInt(opts.every, 10), 60) * 1000,
             ...(opts.goal ? { goal: opts.goal } : {}),
             ...(opts.cron ? { schedule: opts.cron } : {}),
-            notificationThreshold: opts.threshold
+            ...(opts.hosted ? { hosted: true } : {}),
+            ...(opts.name ? { name: opts.name } : {}),
+            // Local only: a hosted check has no significance threshold, and
+            // the option's default would otherwise warn on every hosted create.
+            ...(opts.hosted ? {} : { notificationThreshold: opts.threshold })
           }
         });
         emit(res);
@@ -86,7 +100,7 @@ export function register(program) {
 
   program
     .command('monitor:list')
-    .description('List persisted scheduled monitors')
+    .description('List scheduled monitors (local and hosted)')
     .action(async () => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
       try {
@@ -100,7 +114,7 @@ export function register(program) {
 
   program
     .command('monitor:stop <id>')
-    .description('Stop and remove a scheduled monitor by id')
+    .description('Stop and remove a scheduled monitor by id (local or hosted)')
     .action(async (id) => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
       try {

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -68,7 +68,14 @@ CLI: `crawlforge track https://example.com --selector ".price" --threshold 1`.
 
 ## Scheduled monitoring & notifications
 
-Run continuous monitoring with webhooks instead of polling manually:
+`create_scheduled_monitor` repeats `compare` on a schedule and notifies on
+change. It comes in two kinds.
+
+**Local** (default): persisted in `~/.crawlforge/monitors/`; fires in-process
+only while this MCP server runs (missed runs catch up on restart; `crawlforge
+monitor:run-due` from system cron guarantees firing). Notifies by webhook or
+Slack — never email. `goal` (plain-English LLM judge) and
+`notificationThreshold` apply to local monitors only.
 
 ```json
 {
@@ -76,19 +83,47 @@ Run continuous monitoring with webhooks instead of polling manually:
   "params": {
     "url": "https://example.com/pricing",
     "operation": "create_scheduled_monitor",
-    "monitoringOptions": {
-      "enabled": true,
-      "interval": 300000,
-      "notificationThreshold": "moderate",
-      "enableWebhook": true,
-      "webhookUrl": "https://my-site.com/notify"
+    "scheduledMonitorOptions": { "interval": 1800000, "notificationThreshold": "moderate" },
+    "notificationOptions": { "webhook": { "enabled": true, "url": "https://my-site.com/notify" } }
+  }
+}
+```
+
+**Hosted** (`scheduledMonitorOptions.hosted: true`): registered with
+CrawlForge's hosted monitors API under your API key; nothing is created locally
+and this process never fetches the page. CrawlForge's own scheduler runs every
+check whether or not this process is alive, records it, and notifies by email
+and signed webhook on every changed, new, blocked or errored page. Passing
+`goal` or `notificationThreshold` adds a `warnings` entry.
+
+```json
+{
+  "tool": "track_changes",
+  "params": {
+    "url": "https://example.com/pricing",
+    "operation": "create_scheduled_monitor",
+    "trackingOptions": { "customSelectors": [".price"] },
+    "scheduledMonitorOptions": { "hosted": true, "interval": 21600000 },
+    "notificationOptions": {
+      "email": { "enabled": true, "recipients": ["you@example.com"] },
+      "webhook": { "enabled": true, "url": "https://my-site.com/notify" }
     }
   }
 }
 ```
 
-CLI (runs until Ctrl+C):
-`crawlforge monitor https://example.com --interval 60 --webhook https://my-site.com/notify`.
+Hosted mapping: each `customSelectors` entry becomes a target selector on the
+URL; `schedule` (cron) passes through, otherwise `interval` becomes a cron of
+5–60 minutes dividing 60, whole hours dividing 24, or daily — other values
+round to the nearest and `warnings` says what they became. Up to 5 email
+recipients. `webhook.signingSecret` (16–128 chars) becomes the webhook secret;
+omit it and the response returns a generated `webhookSecret`. The response's
+`monitor` carries the hosted `id`, `nextRunAt`, `estimatedCreditsPerMonth` and
+a `dashboardUrl` for managing it.
+
+CLI: `crawlforge monitor:create <url> --every 1800 --webhook <url>` (local) or
+`crawlforge monitor:create <url> --hosted --email you@example.com` (hosted);
+`monitor:list` shows both kinds and `monitor:stop <id>` removes either.
 
 ## Other operations
 
@@ -99,10 +134,12 @@ CLI (runs until Ctrl+C):
 | `monitor` | One monitoring pass. |
 | `get_history` | Retrieve past change records (`queryOptions`). |
 | `get_stats` | Summary statistics for a tracked URL. |
-| `create_scheduled_monitor` / `stop_scheduled_monitor` | Manage cron-style monitors. |
+| `create_scheduled_monitor` | Recurring `compare` + notify; local by default, `scheduledMonitorOptions.hosted: true` for a CrawlForge-run monitor (see above). |
+| `list_scheduled_monitors` | Local monitors (`hosted: false`) then hosted ones (`hosted: true`), with `localCount`/`hostedCount`; `hostedError` if the website is unreachable. |
+| `stop_scheduled_monitor` | By `scheduledMonitorOptions.monitorId`: stops a local monitor, or deletes the hosted one with that id. By `url` alone: stops every local monitor on it and deletes hosted monitors whose targets are all that exact URL. |
 | `get_dashboard` | Aggregate status, recent alerts, trends. |
 | `export_history` | Export change history as `json` or `csv`. |
-| `create_alert_rule` | Conditional alerts (webhook / email / slack). |
+| `create_alert_rule` | Conditional alerts fired from `compare` (webhook / slack; the email action is not sent by a local process — use a hosted monitor for email). |
 | `generate_trend_report` | Trend analysis over time. |
 | `get_monitoring_templates` | List built-in monitoring presets. |
 
@@ -111,6 +148,8 @@ against the baseline without re-fetching.
 
 ## Cost note
 
-`track_changes` = 3 credits per call. A typical watch is one `create_baseline`
-plus periodic `compare` calls; scheduled monitors run server-side and notify via
-webhook/Slack so you don't pay for manual polling loops.
+`track_changes` = 3 credits per call, except that creating a hosted monitor or
+stopping a hosted-only one charges 0 (3 is the projected ceiling). A typical
+watch is one `create_baseline` plus periodic `compare` calls, or one scheduled
+monitor. Each hosted check bills 3 credits per compared target to your account;
+blocked and errored targets are free.

package/src/tools/tracking/trackChanges/hosted.js

@@ -0,0 +1,176 @@
+/**
+ * TrackChanges — hosted monitors (Phase 6.1).
+ *
+ * `scheduledMonitorOptions.hosted: true` registers a monitor with the
+ * website's /api/v1/monitors instead of the local MonitorStore. The website's
+ * cron then fetches, compares, bills the account and sends the notifications
+ * (email, signed webhooks), so the monitor fires whether or not this process
+ * is alive. This module is the thin client; index.js decides when to use it
+ * and shapes the tool results.
+ *
+ * These calls go to our own configured backend (AuthManager.apiEndpoint, from
+ * CRAWLFORGE_API_URL through endpointGuard), not to a caller-supplied URL, so
+ * they use bare fetch with the X-API-Key header exactly as AuthManager does.
+ * The SSRF guard is for pages a caller names; the endpoint is legitimately
+ * localhost in development.
+ */
+import authManager from '../../../core/AuthManager.js';
+
+const HOSTED_TIMEOUT_MS = 30_000;
+const MINUTE = 60_000;
+const HOUR = 60 * MINUTE;
+
+export const HOSTED_FIRING_GUARANTEE_NOTE =
+  "Runs from CrawlForge's scheduler whether or not this process is alive. Each check bills " +
+  '3 credits per compared target from the account; blocked and errored targets are free.';
+
+export const NO_KEY_MESSAGE =
+  'hosted monitors need a CrawlForge API key — run `crawlforge-setup` or `crawlforge login`';
+
+/**
+ * The endpoint and key the hosted calls authenticate with, from the sources
+ * the server and the CLI already use: CRAWLFORGE_API_KEY (the CLI's preAction
+ * hook fills it from --api-key or the stored config), then the key AuthManager
+ * loaded at startup, then the stored config read directly — the server skips
+ * loading it in creator mode, and a hosted monitor is billed to an account
+ * either way. No network: initialize() would re-validate the key.
+ */
+export async function resolveHostedCredentials() {
+  let apiKey = process.env.CRAWLFORGE_API_KEY || authManager.getConfig()?.apiKey;
+  if (!apiKey) {
+    try {
+      await authManager.loadConfig();
+      apiKey = authManager.getConfig()?.apiKey;
+    } catch {
+      /* no stored config */
+    }
+  }
+  if (!apiKey) throw new Error(NO_KEY_MESSAGE);
+  return { endpoint: authManager.apiEndpoint, apiKey };
+}
+
+// resolveApiEndpoint keeps a trailing slash on the configured endpoint; joined
+// to an absolute path that is `//api/...`, a redirect on every call and a
+// dashboard link with a double slash.
+const base = (endpoint) => String(endpoint).replace(/\/+$/, '');
+
+async function request(method, pathname, creds, body) {
+  const response = await fetch(`${base(creds.endpoint)}${pathname}`, {
+    method,
+    headers: {
+      'X-API-Key': creds.apiKey,
+      ...(body ? { 'Content-Type': 'application/json' } : {})
+    },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+    signal: AbortSignal.timeout(HOSTED_TIMEOUT_MS)
+  });
+  let payload = null;
+  try {
+    payload = await response.json();
+  } catch {
+    /* no JSON body */
+  }
+  if (!response.ok) {
+    // `{ error: { code, message, details? } }` from the monitors API; the
+    // API-key middleware's 401 is the same envelope. The website's own words
+    // reach the caller so a validation or robots refusal is readable.
+    const err = payload?.error;
+    const code = err?.code || `HTTP_${response.status}`;
+    const message = (typeof err === 'string' ? err : err?.message) || response.statusText || 'request failed';
+    const details = err?.details !== undefined ? ` ${JSON.stringify(err.details)}` : '';
+    const failure = new Error(`${code}: ${message}${details}`);
+    failure.code = code;
+    failure.status = response.status;
+    throw failure;
+  }
+  return payload?.data;
+}
+
+export function createHostedMonitor(input, creds) {
+  return request('POST', '/api/v1/monitors', creds, input);
+}
+
+export async function listHostedMonitors(creds) {
+  // An account holds at most 50 monitors, so one page is the whole list.
+  return (await request('GET', '/api/v1/monitors?limit=100', creds)) ?? [];
+}
+
+export function deleteHostedMonitor(id, creds) {
+  return request('DELETE', `/api/v1/monitors/${encodeURIComponent(id)}`, creds);
+}
+
+// The cron slots the website accepts: consecutive runs at least 5 minutes
+// apart. For a `*/N` minute step that means N must divide 60 (`*/7` has a
+// 4-minute gap at the top of every hour); for an hour step, H must divide 24.
+const SLOTS = [
+  ...[5, 6, 10, 12, 15, 20, 30].map((m) => ({ ms: m * MINUTE, cron: `*/${m} * * * *` })),
+  { ms: HOUR, cron: '0 * * * *' },
+  ...[2, 3, 4, 6, 8, 12].map((h) => ({ ms: h * HOUR, cron: `0 */${h} * * *` })),
+  { ms: 24 * HOUR, cron: '0 0 * * *' }
+];
+
+/**
+ * The hosted schedule for a polling interval in ms.
+ * @returns {{ cron: string, effectiveIntervalMs: number, adjusted: boolean }}
+ *   `adjusted` is true when the interval was not an accepted slot and the
+ *   nearest one was used; a tie goes to the longer interval (fewer billed checks).
+ */
+export function intervalToCron(ms) {
+  let best = SLOTS[0];
+  for (const slot of SLOTS) {
+    const d = Math.abs(slot.ms - ms);
+    const bestD = Math.abs(best.ms - ms);
+    if (d < bestD || (d === bestD && slot.ms > best.ms)) best = slot;
+  }
+  return { cron: best.cron, effectiveIntervalMs: best.ms, adjusted: best.ms !== ms };
+}
+
+export function formatInterval(ms) {
+  return ms % HOUR === 0 ? `${ms / HOUR} h` : `${Math.round(ms / MINUTE)} min`;
+}
+
+const parseIso = (iso) => (iso ? Date.parse(iso) || null : null);
+
+export function hostedDashboardUrl(endpoint, id) {
+  return `${base(endpoint)}/dashboard/monitors/${id}`;
+}
+
+/** The `monitor` a hosted create_scheduled_monitor returns. */
+export function createdHostedMonitor(record, endpoint) {
+  return {
+    id: record.id,
+    hosted: true,
+    name: record.name,
+    targets: record.targets,
+    schedule: record.schedule_cron,
+    timezone: record.timezone,
+    notifyEmails: record.notify_emails,
+    webhookUrl: record.webhook_url,
+    webhookSecret: record.webhook_secret,
+    status: record.status,
+    nextRunAt: parseIso(record.next_run_at),
+    estimatedCreditsPerMonth: record.estimated_credits_per_month,
+    dashboardUrl: hostedDashboardUrl(endpoint, record.id)
+  };
+}
+
+/** A hosted monitor as list_scheduled_monitors shows it, beside the local ones. */
+export function listedHostedMonitor(record, endpoint) {
+  const active = record.status === 'active';
+  return {
+    id: record.id,
+    hosted: true,
+    url: record.targets?.[0]?.url,
+    targets: record.targets,
+    name: record.name,
+    schedule: record.schedule_cron,
+    timezone: record.timezone,
+    enabled: active,
+    nextDueAt: parseIso(record.next_run_at),
+    lastCheckAt: parseIso(record.last_check_at),
+    lastCheck: record.last_check ?? null,
+    estimatedCreditsPerMonth: record.estimated_credits_per_month,
+    dashboardUrl: hostedDashboardUrl(endpoint, record.id),
+    scheduled: active
+  };
+}

package/src/tools/tracking/trackChanges/index.js

@@ -21,10 +21,18 @@ import SnapshotManager from '../../../core/SnapshotManager.js';
 import CacheManager from '../../../core/cache/CacheManager.js';
 import { MonitorStore } from '../../../core/MonitorStore.js';
 import { MonitorScheduler } from '../../../core/MonitorScheduler.js';
+import { setActualCost } from '../../../server/requestContext.js';
 import { TrackChangesSchema } from './schema.js';
 import { fetchContent, mergeHistoryData, matchesSignificanceFilter, calculateAverageInterval, calculateSignificanceDistribution } from './differ.js';
 import { performMonitoringCheck, stopMonitor } from './monitor.js';
 import { sendNotifications } from './notifier.js';
+import {
+  HOSTED_FIRING_GUARANTEE_NOTE, createHostedMonitor, createdHostedMonitor, deleteHostedMonitor,
+  formatInterval, intervalToCron, listHostedMonitors, listedHostedMonitor, resolveHostedCredentials
+} from './hosted.js';
+
+// server.js spreads this into the registered inputSchema (G5: one declaration).
+export { TRACK_CHANGES_INPUT_SHAPE } from './schema.js';
 
 export class TrackChangesTool extends EventEmitter {
   constructor(options = {}) {
@@ -42,6 +50,9 @@ export class TrackChangesTool extends EventEmitter {
       enableRealTimeMonitoring: true,
       maxConcurrentMonitors: 50,
       defaultPollingInterval: 300000,
+      // The key and endpoint hosted monitors authenticate with; tests inject
+      // a stub so nothing reads ~/.crawlforge or reaches the website.
+      resolveHostedCredentials,
       ...options
     };
 
@@ -404,6 +415,9 @@ export class TrackChangesTool extends EventEmitter {
         );
       }
     }
+    if (opts.hosted) {
+      return this._createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions });
+    }
     // Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
     // schema fills monitoringOptions.interval/notificationThreshold with
     // defaults, so they cannot sit above a preset without always winning.
@@ -423,25 +437,127 @@ export class TrackChangesTool extends EventEmitter {
     };
   }
 
+  /**
+   * Hosted (6.1): the website's /api/v1/monitors owns the monitor — its cron
+   * fetches, compares, bills and notifies — so nothing is stored or fetched
+   * here. The interval precedence matches the local path except that
+   * monitoringOptions.interval (schema-defaulted to 5 min) is not consulted:
+   * a hosted check is billed, and the website's own default is hourly.
+   */
+  async _createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions }) {
+    const creds = await this.options.resolveHostedCredentials();
+    const warnings = [];
+    let scheduleCron = opts.schedule;
+    const interval = opts.interval ?? preset?.frequency;
+    if (!scheduleCron && interval) {
+      const slot = intervalToCron(interval);
+      scheduleCron = slot.cron;
+      if (slot.adjusted) {
+        warnings.push(
+          `interval ${formatInterval(interval)} is not a hosted schedule slot; the monitor runs every ` +
+          `${formatInterval(slot.effectiveIntervalMs)} (${slot.cron}). Hosted runs are at least 5 minutes apart ` +
+          'and divide the hour or the day evenly.'
+        );
+      }
+    }
+    if (opts.goal ?? preset?.goal) {
+      warnings.push('goal is judged by the local goal judge only and is not applied to a hosted monitor, which notifies on every changed, new, blocked or errored page');
+    }
+    if (opts.notificationThreshold) {
+      warnings.push('notificationThreshold has no effect on a hosted monitor; hosted checks have no significance threshold');
+    }
+
+    const tracking = preset ? { ...preset.options, ...(trackingOptions || {}) } : (trackingOptions || {});
+    const selectors = tracking.customSelectors || [];
+    const email = notificationOptions?.email;
+    const webhook = notificationOptions?.webhook;
+    const secret = webhook?.signingSecret;
+    const record = await createHostedMonitor({
+      name: opts.name || new URL(url).host.slice(0, 80),
+      targets: selectors.length ? selectors.map((selector) => ({ url, selector })) : [{ url }],
+      ...(scheduleCron ? { schedule_cron: scheduleCron } : {}),
+      timezone: 'UTC',
+      ...(email?.enabled && email.recipients?.length ? { notify_emails: email.recipients } : {}),
+      ...(webhook?.enabled && webhook.url ? { webhook_url: webhook.url } : {}),
+      // A secret outside 16-128 chars is left out so the website generates one.
+      ...(webhook?.enabled && webhook.url && secret?.length >= 16 && secret.length <= 128 ? { webhook_secret: secret } : {}),
+      status: 'active'
+    }, creds);
+    // Nothing ran on this machine and the monitors API is free (G4).
+    setActualCost(0);
+    return {
+      success: true, operation: 'create_scheduled_monitor', url, hosted: true,
+      ...(preset ? { templateId: preset.id } : {}),
+      monitor: createdHostedMonitor(record, creds.endpoint),
+      firingGuarantee: HOSTED_FIRING_GUARANTEE_NOTE,
+      ...(warnings.length ? { warnings } : {}),
+      timestamp: Date.now()
+    };
+  }
+
   async stopScheduledMonitor(params) {
     const { url, scheduledMonitorOptions } = params;
     const monitorId = scheduledMonitorOptions?.monitorId;
     if (monitorId) {
-      const result = await this.scheduler.stopMonitor(monitorId);
-      if (!result.stopped) {
-        return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}`, timestamp: Date.now() };
+      if (!this.monitorStore._loaded) await this.monitorStore.load();
+      if (this.monitorStore.get(monitorId)) {
+        await this.scheduler.stopMonitor(monitorId);
+        return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
       }
-      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
+      // Not in the local store: it may be hosted.
+      try {
+        await deleteHostedMonitor(monitorId, await this.options.resolveHostedCredentials());
+      } catch (error) {
+        const reason = error.status === 404 ? '' : ` (hosted lookup failed: ${error.message})`;
+        return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}${reason}`, timestamp: Date.now() };
+      }
+      // Nothing ran on this machine and the monitors API is free (G4).
+      setActualCost(0);
+      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, hosted: true, timestamp: Date.now() };
     }
     if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
     const result = await this.scheduler.stopByUrl(url);
-    return { success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, timestamp: Date.now() };
+    // Only a hosted monitor whose every target is this URL — never a
+    // multi-target monitor that merely includes it.
+    let stoppedHosted = 0;
+    let hostedError = null;
+    try {
+      const creds = await this.options.resolveHostedCredentials();
+      for (const m of await listHostedMonitors(creds)) {
+        if (m.targets?.length && m.targets.every((t) => t.url === url)) {
+          await deleteHostedMonitor(m.id, creds);
+          stoppedHosted++;
+        }
+      }
+    } catch (error) {
+      hostedError = error.message;
+    }
+    return {
+      success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
+      ...(hostedError ? { hostedError } : {}),
+      timestamp: Date.now()
+    };
   }
 
   async listScheduledMonitors() {
     if (!this.monitorStore._loaded) await this.monitorStore.load();
-    const monitors = this.scheduler.list();
-    return { success: true, operation: 'list_scheduled_monitors', monitors, count: monitors.length, timestamp: Date.now() };
+    const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
+    // The local list never fails because the website is unreachable.
+    let hosted = [];
+    let hostedError = null;
+    try {
+      const creds = await this.options.resolveHostedCredentials();
+      hosted = (await listHostedMonitors(creds)).map((r) => listedHostedMonitor(r, creds.endpoint));
+    } catch (error) {
+      hostedError = error.message;
+    }
+    const monitors = [...local, ...hosted];
+    return {
+      success: true, operation: 'list_scheduled_monitors', monitors,
+      count: monitors.length, localCount: local.length, hostedCount: hosted.length,
+      ...(hostedError ? { hostedError } : {}),
+      timestamp: Date.now()
+    };
   }
 
   async getMonitoringDashboard(params) {

package/src/tools/tracking/trackChanges/notifier.js

@@ -79,12 +79,13 @@ export async function sendWebhookNotification(url, changeResult, webhookConfig,
 }
 
 export async function sendEmailNotification(url, changeResult, emailConfig, emitter) {
-  // Email integration placeholder — requires external service
-  emitter?.emit('notificationSent', {
+  // This process has no mail service. Until 6.2.0 this emitted
+  // notificationSent { success: true } for a message that was never sent.
+  // Email is sent by the website's cron for hosted monitors.
+  emitter?.emit('notificationError', {
     type: 'email',
     url,
-    success: true,
-    note: 'Email notifications require external service integration'
+    error: 'Local monitors do not send email; create the monitor with scheduledMonitorOptions.hosted: true (or in the website dashboard) for email notifications'
   });
 }