crawlforge-mcp-server 6.1.0 → 6.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server.js +5 -97
- package/src/cli/commands/monitor.js +18 -4
- package/src/core/AgentOrchestrator.js +106 -1
- package/src/core/ResearchOrchestrator.js +7 -1
- package/src/core/analysis/ContentAnalyzer.js +69 -13
- package/src/core/llm/LLMManager.js +94 -66
- package/src/schemas/toolOutputSchemas.js +1 -1
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +53 -14
- package/src/tools/extract/extractStructured.js +28 -11
- package/src/tools/extract/extractWithLlm.js +1 -62
- package/src/tools/research/deepResearch.js +3 -2
- package/src/tools/tracking/trackChanges/hosted.js +176 -0
- package/src/tools/tracking/trackChanges/index.js +123 -7
- package/src/tools/tracking/trackChanges/notifier.js +5 -4
- package/src/tools/tracking/trackChanges/schema.js +36 -22
- package/src/utils/schemaValidate.js +139 -0
- package/src/core/AlertNotificationSystem.js +0 -602
|
@@ -332,7 +332,7 @@ const searchWebShape = {
|
|
|
332
332
|
// ── extract_structured ───────────────────────────────────────────────────────
|
|
333
333
|
|
|
334
334
|
const extractStructuredShape = {
|
|
335
|
-
success: z.boolean().optional().describe('False when the extraction errored or a required field came back missing or
|
|
335
|
+
success: z.boolean().optional().describe('False when the extraction errored, or a required field came back missing, empty, or in the wrong shape'),
|
|
336
336
|
url: z.string().optional(),
|
|
337
337
|
data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
|
|
338
338
|
extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "keyword_fallback" | "none"'),
|
|
@@ -68,7 +68,14 @@ CLI: `crawlforge track https://example.com --selector ".price" --threshold 1`.
|
|
|
68
68
|
|
|
69
69
|
## Scheduled monitoring & notifications
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
`create_scheduled_monitor` repeats `compare` on a schedule and notifies on
|
|
72
|
+
change. It comes in two kinds.
|
|
73
|
+
|
|
74
|
+
**Local** (default): persisted in `~/.crawlforge/monitors/`; fires in-process
|
|
75
|
+
only while this MCP server runs (missed runs catch up on restart; `crawlforge
|
|
76
|
+
monitor:run-due` from system cron guarantees firing). Notifies by webhook or
|
|
77
|
+
Slack — never email. `goal` (plain-English LLM judge) and
|
|
78
|
+
`notificationThreshold` apply to local monitors only.
|
|
72
79
|
|
|
73
80
|
```json
|
|
74
81
|
{
|
|
@@ -76,19 +83,47 @@ Run continuous monitoring with webhooks instead of polling manually:
|
|
|
76
83
|
"params": {
|
|
77
84
|
"url": "https://example.com/pricing",
|
|
78
85
|
"operation": "create_scheduled_monitor",
|
|
79
|
-
"
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
86
|
+
"scheduledMonitorOptions": { "interval": 1800000, "notificationThreshold": "moderate" },
|
|
87
|
+
"notificationOptions": { "webhook": { "enabled": true, "url": "https://my-site.com/notify" } }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
**Hosted** (`scheduledMonitorOptions.hosted: true`): registered with
|
|
93
|
+
CrawlForge's hosted monitors API under your API key; nothing is created locally
|
|
94
|
+
and this process never fetches the page. CrawlForge's own scheduler runs every
|
|
95
|
+
check whether or not this process is alive, records it, and notifies by email
|
|
96
|
+
and signed webhook on every changed, new, blocked or errored page. Passing
|
|
97
|
+
`goal` or `notificationThreshold` adds a `warnings` entry.
|
|
98
|
+
|
|
99
|
+
```json
|
|
100
|
+
{
|
|
101
|
+
"tool": "track_changes",
|
|
102
|
+
"params": {
|
|
103
|
+
"url": "https://example.com/pricing",
|
|
104
|
+
"operation": "create_scheduled_monitor",
|
|
105
|
+
"trackingOptions": { "customSelectors": [".price"] },
|
|
106
|
+
"scheduledMonitorOptions": { "hosted": true, "interval": 21600000 },
|
|
107
|
+
"notificationOptions": {
|
|
108
|
+
"email": { "enabled": true, "recipients": ["you@example.com"] },
|
|
109
|
+
"webhook": { "enabled": true, "url": "https://my-site.com/notify" }
|
|
85
110
|
}
|
|
86
111
|
}
|
|
87
112
|
}
|
|
88
113
|
```
|
|
89
114
|
|
|
90
|
-
|
|
91
|
-
`
|
|
115
|
+
Hosted mapping: each `customSelectors` entry becomes a target selector on the
|
|
116
|
+
URL; `schedule` (cron) passes through, otherwise `interval` becomes a cron of
|
|
117
|
+
5–60 minutes dividing 60, whole hours dividing 24, or daily — other values
|
|
118
|
+
round to the nearest and `warnings` says what they became. Up to 5 email
|
|
119
|
+
recipients. `webhook.signingSecret` (16–128 chars) becomes the webhook secret;
|
|
120
|
+
omit it and the response returns a generated `webhookSecret`. The response's
|
|
121
|
+
`monitor` carries the hosted `id`, `nextRunAt`, `estimatedCreditsPerMonth` and
|
|
122
|
+
a `dashboardUrl` for managing it.
|
|
123
|
+
|
|
124
|
+
CLI: `crawlforge monitor:create <url> --every 1800 --webhook <url>` (local) or
|
|
125
|
+
`crawlforge monitor:create <url> --hosted --email you@example.com` (hosted);
|
|
126
|
+
`monitor:list` shows both kinds and `monitor:stop <id>` removes either.
|
|
92
127
|
|
|
93
128
|
## Other operations
|
|
94
129
|
|
|
@@ -99,10 +134,12 @@ CLI (runs until Ctrl+C):
|
|
|
99
134
|
| `monitor` | One monitoring pass. |
|
|
100
135
|
| `get_history` | Retrieve past change records (`queryOptions`). |
|
|
101
136
|
| `get_stats` | Summary statistics for a tracked URL. |
|
|
102
|
-
| `create_scheduled_monitor`
|
|
137
|
+
| `create_scheduled_monitor` | Recurring `compare` + notify; local by default, `scheduledMonitorOptions.hosted: true` for a CrawlForge-run monitor (see above). |
|
|
138
|
+
| `list_scheduled_monitors` | Local monitors (`hosted: false`) then hosted ones (`hosted: true`), with `localCount`/`hostedCount`; `hostedError` if the website is unreachable. |
|
|
139
|
+
| `stop_scheduled_monitor` | By `scheduledMonitorOptions.monitorId`: stops a local monitor, or deletes the hosted one with that id. By `url` alone: stops every local monitor on it and deletes hosted monitors whose targets are all that exact URL. |
|
|
103
140
|
| `get_dashboard` | Aggregate status, recent alerts, trends. |
|
|
104
141
|
| `export_history` | Export change history as `json` or `csv`. |
|
|
105
|
-
| `create_alert_rule` | Conditional alerts (webhook / email
|
|
142
|
+
| `create_alert_rule` | Conditional alerts fired from `compare` (webhook / slack; the email action is not sent by a local process — use a hosted monitor for email). |
|
|
106
143
|
| `generate_trend_report` | Trend analysis over time. |
|
|
107
144
|
| `get_monitoring_templates` | List built-in monitoring presets. |
|
|
108
145
|
|
|
@@ -111,6 +148,8 @@ against the baseline without re-fetching.
|
|
|
111
148
|
|
|
112
149
|
## Cost note
|
|
113
150
|
|
|
114
|
-
`track_changes` = 3 credits per call
|
|
115
|
-
|
|
116
|
-
|
|
151
|
+
`track_changes` = 3 credits per call, except that creating a hosted monitor or
|
|
152
|
+
stopping a hosted-only one charges 0 (3 is the projected ceiling). A typical
|
|
153
|
+
watch is one `create_baseline` plus periodic `compare` calls, or one scheduled
|
|
154
|
+
monitor. Each hosted check bills 3 credits per compared target to your account;
|
|
155
|
+
blocked and errored targets are free.
|
|
@@ -12,6 +12,7 @@ import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
|
|
|
12
12
|
import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
|
|
13
13
|
import { extractMainContent } from '../scrape/_mainContent.js';
|
|
14
14
|
import { verifyNumericProvenance } from '../../utils/provenance.js';
|
|
15
|
+
import { validateAgainstSchema, validateFieldsAgainstSchema } from '../../utils/schemaValidate.js';
|
|
15
16
|
|
|
16
17
|
// Semantic element selectors for well-known field names, tried as a last
|
|
17
18
|
// resort in the CSS fallback so common fields (e.g. "title") still resolve when
|
|
@@ -394,7 +395,21 @@ export class ExtractStructuredTool {
|
|
|
394
395
|
const missingRequired = (schema.required || []).filter(
|
|
395
396
|
(field) => isEmptyValue((extractionResult.data || {})[field])
|
|
396
397
|
);
|
|
397
|
-
|
|
398
|
+
// Same reasoning for a required field that arrived in the wrong shape:
|
|
399
|
+
// `{countries: ["a stray line of page text"]}` for an array of objects
|
|
400
|
+
// is a failed extraction too, and it is neither missing nor empty, so
|
|
401
|
+
// the check above waves it through (R19). Re-validated per field rather
|
|
402
|
+
// than parsed out of the error strings.
|
|
403
|
+
const invalidRequired = extractionResult.valid !== true
|
|
404
|
+
? (schema.required || []).filter((field) => {
|
|
405
|
+
if (missingRequired.includes(field)) return false;
|
|
406
|
+
const fieldSchema = schema.properties?.[field];
|
|
407
|
+
if (!fieldSchema) return false;
|
|
408
|
+
return validateAgainstSchema((extractionResult.data || {})[field], fieldSchema).valid !== true;
|
|
409
|
+
})
|
|
410
|
+
: [];
|
|
411
|
+
const failedRequired = extractionResult.valid !== true
|
|
412
|
+
&& (missingRequired.length > 0 || invalidRequired.length > 0);
|
|
398
413
|
|
|
399
414
|
return {
|
|
400
415
|
success: !failedRequired,
|
|
@@ -405,7 +420,12 @@ export class ExtractStructuredTool {
|
|
|
405
420
|
schema_used: schema,
|
|
406
421
|
processingTime: Date.now() - startTime,
|
|
407
422
|
...(failedRequired
|
|
408
|
-
? {
|
|
423
|
+
? {
|
|
424
|
+
error: `Required field(s) ${[
|
|
425
|
+
missingRequired.length ? `missing or empty: ${missingRequired.join(', ')}` : '',
|
|
426
|
+
invalidRequired.length ? `wrong shape: ${invalidRequired.join(', ')}` : ''
|
|
427
|
+
].filter(Boolean).join('; ')}`
|
|
428
|
+
}
|
|
409
429
|
: {}),
|
|
410
430
|
validation: {
|
|
411
431
|
valid: extractionResult.valid || false,
|
|
@@ -566,18 +586,15 @@ export class ExtractStructuredTool {
|
|
|
566
586
|
return null; // No fields found via CSS, let keyword fallback handle it
|
|
567
587
|
}
|
|
568
588
|
|
|
569
|
-
// Validate
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
errors.push(`Missing required field: ${field}`);
|
|
575
|
-
}
|
|
576
|
-
}
|
|
589
|
+
// Validate the extracted fields. This used to check only that each
|
|
590
|
+
// required key was present, which is why a fallback that swept three
|
|
591
|
+
// stray <p> elements into an array-of-objects field reported valid: true
|
|
592
|
+
// (R19) — presence says nothing about shape.
|
|
593
|
+
const { valid, errors } = validateFieldsAgainstSchema(extracted, schema);
|
|
577
594
|
|
|
578
595
|
return {
|
|
579
596
|
data: extracted,
|
|
580
|
-
valid
|
|
597
|
+
valid,
|
|
581
598
|
validationErrors: errors,
|
|
582
599
|
extractionNotes: ['Used CSS selector fallback extraction']
|
|
583
600
|
};
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
* Pass provider: "openai" | "anthropic" with the matching API key to use a cloud model.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { z } from 'zod';
|
|
11
10
|
import { fetchAndParse } from './_fetchAndParse.js';
|
|
12
11
|
import { ollamaBaseUrl, ollamaHeaders, selectOllamaModel } from '../../utils/ollamaConfig.js';
|
|
13
12
|
import { verifyNumericProvenance } from '../../utils/provenance.js';
|
|
14
13
|
import { extractionFormat } from '../../utils/extractionFormat.js';
|
|
14
|
+
import { validateAgainstSchema } from '../../utils/schemaValidate.js';
|
|
15
15
|
import { fenceUntrusted } from '../../utils/untrustedContent.js';
|
|
16
16
|
// D1.3: SamplingClient for MCP sampling fallback (lazy — only imported if needed)
|
|
17
17
|
let _SamplingClient = null;
|
|
@@ -173,48 +173,6 @@ function buildInputSchema(schema) {
|
|
|
173
173
|
return format === 'json' ? { type: 'object', properties: {}, additionalProperties: true } : format;
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
/**
|
|
177
|
-
* Build a zod validator from a JSON-Schema-like hint. Best-effort: unknown
|
|
178
|
-
* shapes fall back to `z.any()` so validation never rejects on constructs the
|
|
179
|
-
* converter does not understand.
|
|
180
|
-
*/
|
|
181
|
-
function jsonSchemaToZod(schema) {
|
|
182
|
-
if (!schema || typeof schema !== 'object') return z.any();
|
|
183
|
-
|
|
184
|
-
// Flat hint map (no `type`/`properties`) → treat values as field hints.
|
|
185
|
-
const isJsonSchema = schema.type || schema.properties || schema.items;
|
|
186
|
-
if (!isJsonSchema) {
|
|
187
|
-
const shape = {};
|
|
188
|
-
for (const [key, val] of Object.entries(schema)) {
|
|
189
|
-
shape[key] = jsonSchemaToZod(typeof val === 'string' ? { type: val } : val).nullable().optional();
|
|
190
|
-
}
|
|
191
|
-
return z.object(shape).passthrough();
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
switch (schema.type) {
|
|
195
|
-
case 'string': return z.string();
|
|
196
|
-
case 'number':
|
|
197
|
-
case 'integer': return z.number();
|
|
198
|
-
case 'boolean': return z.boolean();
|
|
199
|
-
case 'null': return z.null();
|
|
200
|
-
case 'array': return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());
|
|
201
|
-
case 'object': {
|
|
202
|
-
const shape = {};
|
|
203
|
-
const required = Array.isArray(schema.required) ? schema.required : [];
|
|
204
|
-
for (const [key, val] of Object.entries(schema.properties || {})) {
|
|
205
|
-
const field = jsonSchemaToZod(val);
|
|
206
|
-
// The model is told to answer null for a field the content never
|
|
207
|
-
// states, so null is the honest answer for a field the schema does
|
|
208
|
-
// not require — not a type violation. A required field stays strict:
|
|
209
|
-
// null there is exactly what the caller needs to hear about.
|
|
210
|
-
shape[key] = required.includes(key) ? field : field.nullable().optional();
|
|
211
|
-
}
|
|
212
|
-
return z.object(shape).passthrough();
|
|
213
|
-
}
|
|
214
|
-
default: return z.any();
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
|
|
218
176
|
/** JSON Schema type keywords, used to spot a type declaration posing as a value. */
|
|
219
177
|
const SCHEMA_TYPE_KEYWORDS = new Set(['string', 'number', 'integer', 'boolean', 'object', 'array', 'null']);
|
|
220
178
|
|
|
@@ -281,25 +239,6 @@ function hasNoExtractableData(parsed) {
|
|
|
281
239
|
return Object.values(parsed).every(hasNoExtractableData);
|
|
282
240
|
}
|
|
283
241
|
|
|
284
|
-
/**
|
|
285
|
-
* Validate parsed output against the schema hint.
|
|
286
|
-
* @returns {{ valid: boolean, errors: string[] }}
|
|
287
|
-
*/
|
|
288
|
-
function validateAgainstSchema(parsed, schema) {
|
|
289
|
-
try {
|
|
290
|
-
const validator = jsonSchemaToZod(schema);
|
|
291
|
-
const result = validator.safeParse(parsed);
|
|
292
|
-
if (result.success) return { valid: true, errors: [] };
|
|
293
|
-
return {
|
|
294
|
-
valid: false,
|
|
295
|
-
errors: result.error.issues.map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`)
|
|
296
|
-
};
|
|
297
|
-
} catch {
|
|
298
|
-
// Converter failure should not block extraction — treat as unvalidated.
|
|
299
|
-
return { valid: true, errors: [] };
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
242
|
// ── OpenAI call ───────────────────────────────────────────────────────────────
|
|
304
243
|
|
|
305
244
|
async function callOpenAI({ apiKey, model, systemMessage, userMessage, maxTokens }) {
|
|
@@ -451,8 +451,9 @@ export class DeepResearchTool {
|
|
|
451
451
|
* Format research results according to output preferences
|
|
452
452
|
*/
|
|
453
453
|
formatResults(results, params) {
|
|
454
|
-
// Raw evidence mode (no LLM configured
|
|
455
|
-
//
|
|
454
|
+
// Raw evidence mode (no LLM configured, or the token budget ran out
|
|
455
|
+
// mid-run): apply lightweight formatting so outputFormat is not silently
|
|
456
|
+
// ignored, and rank sources by credibility. `results.note` says which.
|
|
456
457
|
if (results.synthesisMode === 'raw_evidence') {
|
|
457
458
|
const rankedSources = (results.sources || [])
|
|
458
459
|
.slice()
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TrackChanges — hosted monitors (Phase 6.1).
|
|
3
|
+
*
|
|
4
|
+
* `scheduledMonitorOptions.hosted: true` registers a monitor with the
|
|
5
|
+
* website's /api/v1/monitors instead of the local MonitorStore. The website's
|
|
6
|
+
* cron then fetches, compares, bills the account and sends the notifications
|
|
7
|
+
* (email, signed webhooks), so the monitor fires whether or not this process
|
|
8
|
+
* is alive. This module is the thin client; index.js decides when to use it
|
|
9
|
+
* and shapes the tool results.
|
|
10
|
+
*
|
|
11
|
+
* These calls go to our own configured backend (AuthManager.apiEndpoint, from
|
|
12
|
+
* CRAWLFORGE_API_URL through endpointGuard), not to a caller-supplied URL, so
|
|
13
|
+
* they use bare fetch with the X-API-Key header exactly as AuthManager does.
|
|
14
|
+
* The SSRF guard is for pages a caller names; the endpoint is legitimately
|
|
15
|
+
* localhost in development.
|
|
16
|
+
*/
|
|
17
|
+
import authManager from '../../../core/AuthManager.js';
|
|
18
|
+
|
|
19
|
+
const HOSTED_TIMEOUT_MS = 30_000;
|
|
20
|
+
const MINUTE = 60_000;
|
|
21
|
+
const HOUR = 60 * MINUTE;
|
|
22
|
+
|
|
23
|
+
export const HOSTED_FIRING_GUARANTEE_NOTE =
|
|
24
|
+
"Runs from CrawlForge's scheduler whether or not this process is alive. Each check bills " +
|
|
25
|
+
'3 credits per compared target from the account; blocked and errored targets are free.';
|
|
26
|
+
|
|
27
|
+
export const NO_KEY_MESSAGE =
|
|
28
|
+
'hosted monitors need a CrawlForge API key — run `crawlforge-setup` or `crawlforge login`';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The endpoint and key the hosted calls authenticate with, from the sources
|
|
32
|
+
* the server and the CLI already use: CRAWLFORGE_API_KEY (the CLI's preAction
|
|
33
|
+
* hook fills it from --api-key or the stored config), then the key AuthManager
|
|
34
|
+
* loaded at startup, then the stored config read directly — the server skips
|
|
35
|
+
* loading it in creator mode, and a hosted monitor is billed to an account
|
|
36
|
+
* either way. No network: initialize() would re-validate the key.
|
|
37
|
+
*/
|
|
38
|
+
export async function resolveHostedCredentials() {
|
|
39
|
+
let apiKey = process.env.CRAWLFORGE_API_KEY || authManager.getConfig()?.apiKey;
|
|
40
|
+
if (!apiKey) {
|
|
41
|
+
try {
|
|
42
|
+
await authManager.loadConfig();
|
|
43
|
+
apiKey = authManager.getConfig()?.apiKey;
|
|
44
|
+
} catch {
|
|
45
|
+
/* no stored config */
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (!apiKey) throw new Error(NO_KEY_MESSAGE);
|
|
49
|
+
return { endpoint: authManager.apiEndpoint, apiKey };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// resolveApiEndpoint keeps a trailing slash on the configured endpoint; joined
|
|
53
|
+
// to an absolute path that is `//api/...`, a redirect on every call and a
|
|
54
|
+
// dashboard link with a double slash.
|
|
55
|
+
const base = (endpoint) => String(endpoint).replace(/\/+$/, '');
|
|
56
|
+
|
|
57
|
+
async function request(method, pathname, creds, body) {
|
|
58
|
+
const response = await fetch(`${base(creds.endpoint)}${pathname}`, {
|
|
59
|
+
method,
|
|
60
|
+
headers: {
|
|
61
|
+
'X-API-Key': creds.apiKey,
|
|
62
|
+
...(body ? { 'Content-Type': 'application/json' } : {})
|
|
63
|
+
},
|
|
64
|
+
...(body ? { body: JSON.stringify(body) } : {}),
|
|
65
|
+
signal: AbortSignal.timeout(HOSTED_TIMEOUT_MS)
|
|
66
|
+
});
|
|
67
|
+
let payload = null;
|
|
68
|
+
try {
|
|
69
|
+
payload = await response.json();
|
|
70
|
+
} catch {
|
|
71
|
+
/* no JSON body */
|
|
72
|
+
}
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
// `{ error: { code, message, details? } }` from the monitors API; the
|
|
75
|
+
// API-key middleware's 401 is the same envelope. The website's own words
|
|
76
|
+
// reach the caller so a validation or robots refusal is readable.
|
|
77
|
+
const err = payload?.error;
|
|
78
|
+
const code = err?.code || `HTTP_${response.status}`;
|
|
79
|
+
const message = (typeof err === 'string' ? err : err?.message) || response.statusText || 'request failed';
|
|
80
|
+
const details = err?.details !== undefined ? ` ${JSON.stringify(err.details)}` : '';
|
|
81
|
+
const failure = new Error(`${code}: ${message}${details}`);
|
|
82
|
+
failure.code = code;
|
|
83
|
+
failure.status = response.status;
|
|
84
|
+
throw failure;
|
|
85
|
+
}
|
|
86
|
+
return payload?.data;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function createHostedMonitor(input, creds) {
|
|
90
|
+
return request('POST', '/api/v1/monitors', creds, input);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export async function listHostedMonitors(creds) {
|
|
94
|
+
// An account holds at most 50 monitors, so one page is the whole list.
|
|
95
|
+
return (await request('GET', '/api/v1/monitors?limit=100', creds)) ?? [];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function deleteHostedMonitor(id, creds) {
|
|
99
|
+
return request('DELETE', `/api/v1/monitors/${encodeURIComponent(id)}`, creds);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The cron slots the website accepts: consecutive runs at least 5 minutes
|
|
103
|
+
// apart. For a `*/N` minute step that means N must divide 60 (`*/7` has a
|
|
104
|
+
// 4-minute gap at the top of every hour); for an hour step, H must divide 24.
|
|
105
|
+
const SLOTS = [
|
|
106
|
+
...[5, 6, 10, 12, 15, 20, 30].map((m) => ({ ms: m * MINUTE, cron: `*/${m} * * * *` })),
|
|
107
|
+
{ ms: HOUR, cron: '0 * * * *' },
|
|
108
|
+
...[2, 3, 4, 6, 8, 12].map((h) => ({ ms: h * HOUR, cron: `0 */${h} * * *` })),
|
|
109
|
+
{ ms: 24 * HOUR, cron: '0 0 * * *' }
|
|
110
|
+
];
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The hosted schedule for a polling interval in ms.
|
|
114
|
+
* @returns {{ cron: string, effectiveIntervalMs: number, adjusted: boolean }}
|
|
115
|
+
* `adjusted` is true when the interval was not an accepted slot and the
|
|
116
|
+
* nearest one was used; a tie goes to the longer interval (fewer billed checks).
|
|
117
|
+
*/
|
|
118
|
+
export function intervalToCron(ms) {
|
|
119
|
+
let best = SLOTS[0];
|
|
120
|
+
for (const slot of SLOTS) {
|
|
121
|
+
const d = Math.abs(slot.ms - ms);
|
|
122
|
+
const bestD = Math.abs(best.ms - ms);
|
|
123
|
+
if (d < bestD || (d === bestD && slot.ms > best.ms)) best = slot;
|
|
124
|
+
}
|
|
125
|
+
return { cron: best.cron, effectiveIntervalMs: best.ms, adjusted: best.ms !== ms };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function formatInterval(ms) {
|
|
129
|
+
return ms % HOUR === 0 ? `${ms / HOUR} h` : `${Math.round(ms / MINUTE)} min`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const parseIso = (iso) => (iso ? Date.parse(iso) || null : null);
|
|
133
|
+
|
|
134
|
+
export function hostedDashboardUrl(endpoint, id) {
|
|
135
|
+
return `${base(endpoint)}/dashboard/monitors/${id}`;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The `monitor` a hosted create_scheduled_monitor returns. */
|
|
139
|
+
export function createdHostedMonitor(record, endpoint) {
|
|
140
|
+
return {
|
|
141
|
+
id: record.id,
|
|
142
|
+
hosted: true,
|
|
143
|
+
name: record.name,
|
|
144
|
+
targets: record.targets,
|
|
145
|
+
schedule: record.schedule_cron,
|
|
146
|
+
timezone: record.timezone,
|
|
147
|
+
notifyEmails: record.notify_emails,
|
|
148
|
+
webhookUrl: record.webhook_url,
|
|
149
|
+
webhookSecret: record.webhook_secret,
|
|
150
|
+
status: record.status,
|
|
151
|
+
nextRunAt: parseIso(record.next_run_at),
|
|
152
|
+
estimatedCreditsPerMonth: record.estimated_credits_per_month,
|
|
153
|
+
dashboardUrl: hostedDashboardUrl(endpoint, record.id)
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A hosted monitor as list_scheduled_monitors shows it, beside the local ones. */
|
|
158
|
+
export function listedHostedMonitor(record, endpoint) {
|
|
159
|
+
const active = record.status === 'active';
|
|
160
|
+
return {
|
|
161
|
+
id: record.id,
|
|
162
|
+
hosted: true,
|
|
163
|
+
url: record.targets?.[0]?.url,
|
|
164
|
+
targets: record.targets,
|
|
165
|
+
name: record.name,
|
|
166
|
+
schedule: record.schedule_cron,
|
|
167
|
+
timezone: record.timezone,
|
|
168
|
+
enabled: active,
|
|
169
|
+
nextDueAt: parseIso(record.next_run_at),
|
|
170
|
+
lastCheckAt: parseIso(record.last_check_at),
|
|
171
|
+
lastCheck: record.last_check ?? null,
|
|
172
|
+
estimatedCreditsPerMonth: record.estimated_credits_per_month,
|
|
173
|
+
dashboardUrl: hostedDashboardUrl(endpoint, record.id),
|
|
174
|
+
scheduled: active
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -21,10 +21,18 @@ import SnapshotManager from '../../../core/SnapshotManager.js';
|
|
|
21
21
|
import CacheManager from '../../../core/cache/CacheManager.js';
|
|
22
22
|
import { MonitorStore } from '../../../core/MonitorStore.js';
|
|
23
23
|
import { MonitorScheduler } from '../../../core/MonitorScheduler.js';
|
|
24
|
+
import { setActualCost } from '../../../server/requestContext.js';
|
|
24
25
|
import { TrackChangesSchema } from './schema.js';
|
|
25
26
|
import { fetchContent, mergeHistoryData, matchesSignificanceFilter, calculateAverageInterval, calculateSignificanceDistribution } from './differ.js';
|
|
26
27
|
import { performMonitoringCheck, stopMonitor } from './monitor.js';
|
|
27
28
|
import { sendNotifications } from './notifier.js';
|
|
29
|
+
import {
|
|
30
|
+
HOSTED_FIRING_GUARANTEE_NOTE, createHostedMonitor, createdHostedMonitor, deleteHostedMonitor,
|
|
31
|
+
formatInterval, intervalToCron, listHostedMonitors, listedHostedMonitor, resolveHostedCredentials
|
|
32
|
+
} from './hosted.js';
|
|
33
|
+
|
|
34
|
+
// server.js spreads this into the registered inputSchema (G5: one declaration).
|
|
35
|
+
export { TRACK_CHANGES_INPUT_SHAPE } from './schema.js';
|
|
28
36
|
|
|
29
37
|
export class TrackChangesTool extends EventEmitter {
|
|
30
38
|
constructor(options = {}) {
|
|
@@ -42,6 +50,9 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
42
50
|
enableRealTimeMonitoring: true,
|
|
43
51
|
maxConcurrentMonitors: 50,
|
|
44
52
|
defaultPollingInterval: 300000,
|
|
53
|
+
// The key and endpoint hosted monitors authenticate with; tests inject
|
|
54
|
+
// a stub so nothing reads ~/.crawlforge or reaches the website.
|
|
55
|
+
resolveHostedCredentials,
|
|
45
56
|
...options
|
|
46
57
|
};
|
|
47
58
|
|
|
@@ -404,6 +415,9 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
404
415
|
);
|
|
405
416
|
}
|
|
406
417
|
}
|
|
418
|
+
if (opts.hosted) {
|
|
419
|
+
return this._createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions });
|
|
420
|
+
}
|
|
407
421
|
// Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
|
|
408
422
|
// schema fills monitoringOptions.interval/notificationThreshold with
|
|
409
423
|
// defaults, so they cannot sit above a preset without always winning.
|
|
@@ -423,25 +437,127 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
423
437
|
};
|
|
424
438
|
}
|
|
425
439
|
|
|
440
|
+
/**
|
|
441
|
+
* Hosted (6.1): the website's /api/v1/monitors owns the monitor — its cron
|
|
442
|
+
* fetches, compares, bills and notifies — so nothing is stored or fetched
|
|
443
|
+
* here. The interval precedence matches the local path except that
|
|
444
|
+
* monitoringOptions.interval (schema-defaulted to 5 min) is not consulted:
|
|
445
|
+
* a hosted check is billed, and the website's own default is hourly.
|
|
446
|
+
*/
|
|
447
|
+
async _createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions }) {
|
|
448
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
449
|
+
const warnings = [];
|
|
450
|
+
let scheduleCron = opts.schedule;
|
|
451
|
+
const interval = opts.interval ?? preset?.frequency;
|
|
452
|
+
if (!scheduleCron && interval) {
|
|
453
|
+
const slot = intervalToCron(interval);
|
|
454
|
+
scheduleCron = slot.cron;
|
|
455
|
+
if (slot.adjusted) {
|
|
456
|
+
warnings.push(
|
|
457
|
+
`interval ${formatInterval(interval)} is not a hosted schedule slot; the monitor runs every ` +
|
|
458
|
+
`${formatInterval(slot.effectiveIntervalMs)} (${slot.cron}). Hosted runs are at least 5 minutes apart ` +
|
|
459
|
+
'and divide the hour or the day evenly.'
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (opts.goal ?? preset?.goal) {
|
|
464
|
+
warnings.push('goal is judged by the local goal judge only and is not applied to a hosted monitor, which notifies on every changed, new, blocked or errored page');
|
|
465
|
+
}
|
|
466
|
+
if (opts.notificationThreshold) {
|
|
467
|
+
warnings.push('notificationThreshold has no effect on a hosted monitor; hosted checks have no significance threshold');
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const tracking = preset ? { ...preset.options, ...(trackingOptions || {}) } : (trackingOptions || {});
|
|
471
|
+
const selectors = tracking.customSelectors || [];
|
|
472
|
+
const email = notificationOptions?.email;
|
|
473
|
+
const webhook = notificationOptions?.webhook;
|
|
474
|
+
const secret = webhook?.signingSecret;
|
|
475
|
+
const record = await createHostedMonitor({
|
|
476
|
+
name: opts.name || new URL(url).host.slice(0, 80),
|
|
477
|
+
targets: selectors.length ? selectors.map((selector) => ({ url, selector })) : [{ url }],
|
|
478
|
+
...(scheduleCron ? { schedule_cron: scheduleCron } : {}),
|
|
479
|
+
timezone: 'UTC',
|
|
480
|
+
...(email?.enabled && email.recipients?.length ? { notify_emails: email.recipients } : {}),
|
|
481
|
+
...(webhook?.enabled && webhook.url ? { webhook_url: webhook.url } : {}),
|
|
482
|
+
// A secret outside 16-128 chars is left out so the website generates one.
|
|
483
|
+
...(webhook?.enabled && webhook.url && secret?.length >= 16 && secret.length <= 128 ? { webhook_secret: secret } : {}),
|
|
484
|
+
status: 'active'
|
|
485
|
+
}, creds);
|
|
486
|
+
// Nothing ran on this machine and the monitors API is free (G4).
|
|
487
|
+
setActualCost(0);
|
|
488
|
+
return {
|
|
489
|
+
success: true, operation: 'create_scheduled_monitor', url, hosted: true,
|
|
490
|
+
...(preset ? { templateId: preset.id } : {}),
|
|
491
|
+
monitor: createdHostedMonitor(record, creds.endpoint),
|
|
492
|
+
firingGuarantee: HOSTED_FIRING_GUARANTEE_NOTE,
|
|
493
|
+
...(warnings.length ? { warnings } : {}),
|
|
494
|
+
timestamp: Date.now()
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
|
|
426
498
|
async stopScheduledMonitor(params) {
|
|
427
499
|
const { url, scheduledMonitorOptions } = params;
|
|
428
500
|
const monitorId = scheduledMonitorOptions?.monitorId;
|
|
429
501
|
if (monitorId) {
|
|
430
|
-
|
|
431
|
-
if (
|
|
432
|
-
|
|
502
|
+
if (!this.monitorStore._loaded) await this.monitorStore.load();
|
|
503
|
+
if (this.monitorStore.get(monitorId)) {
|
|
504
|
+
await this.scheduler.stopMonitor(monitorId);
|
|
505
|
+
return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
|
|
433
506
|
}
|
|
434
|
-
|
|
507
|
+
// Not in the local store: it may be hosted.
|
|
508
|
+
try {
|
|
509
|
+
await deleteHostedMonitor(monitorId, await this.options.resolveHostedCredentials());
|
|
510
|
+
} catch (error) {
|
|
511
|
+
const reason = error.status === 404 ? '' : ` (hosted lookup failed: ${error.message})`;
|
|
512
|
+
return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}${reason}`, timestamp: Date.now() };
|
|
513
|
+
}
|
|
514
|
+
// Nothing ran on this machine and the monitors API is free (G4).
|
|
515
|
+
setActualCost(0);
|
|
516
|
+
return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, hosted: true, timestamp: Date.now() };
|
|
435
517
|
}
|
|
436
518
|
if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
|
|
437
519
|
const result = await this.scheduler.stopByUrl(url);
|
|
438
|
-
|
|
520
|
+
// Only a hosted monitor whose every target is this URL — never a
|
|
521
|
+
// multi-target monitor that merely includes it.
|
|
522
|
+
let stoppedHosted = 0;
|
|
523
|
+
let hostedError = null;
|
|
524
|
+
try {
|
|
525
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
526
|
+
for (const m of await listHostedMonitors(creds)) {
|
|
527
|
+
if (m.targets?.length && m.targets.every((t) => t.url === url)) {
|
|
528
|
+
await deleteHostedMonitor(m.id, creds);
|
|
529
|
+
stoppedHosted++;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
} catch (error) {
|
|
533
|
+
hostedError = error.message;
|
|
534
|
+
}
|
|
535
|
+
return {
|
|
536
|
+
success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
|
|
537
|
+
...(hostedError ? { hostedError } : {}),
|
|
538
|
+
timestamp: Date.now()
|
|
539
|
+
};
|
|
439
540
|
}
|
|
440
541
|
|
|
441
542
|
async listScheduledMonitors() {
|
|
442
543
|
if (!this.monitorStore._loaded) await this.monitorStore.load();
|
|
443
|
-
const
|
|
444
|
-
|
|
544
|
+
const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
|
|
545
|
+
// The local list never fails because the website is unreachable.
|
|
546
|
+
let hosted = [];
|
|
547
|
+
let hostedError = null;
|
|
548
|
+
try {
|
|
549
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
550
|
+
hosted = (await listHostedMonitors(creds)).map((r) => listedHostedMonitor(r, creds.endpoint));
|
|
551
|
+
} catch (error) {
|
|
552
|
+
hostedError = error.message;
|
|
553
|
+
}
|
|
554
|
+
const monitors = [...local, ...hosted];
|
|
555
|
+
return {
|
|
556
|
+
success: true, operation: 'list_scheduled_monitors', monitors,
|
|
557
|
+
count: monitors.length, localCount: local.length, hostedCount: hosted.length,
|
|
558
|
+
...(hostedError ? { hostedError } : {}),
|
|
559
|
+
timestamp: Date.now()
|
|
560
|
+
};
|
|
445
561
|
}
|
|
446
562
|
|
|
447
563
|
async getMonitoringDashboard(params) {
|
|
@@ -79,12 +79,13 @@ export async function sendWebhookNotification(url, changeResult, webhookConfig,
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
export async function sendEmailNotification(url, changeResult, emailConfig, emitter) {
|
|
82
|
-
//
|
|
83
|
-
|
|
82
|
+
// This process has no mail service. Until 6.2.0 this emitted
|
|
83
|
+
// notificationSent { success: true } for a message that was never sent.
|
|
84
|
+
// Email is sent by the website's cron for hosted monitors.
|
|
85
|
+
emitter?.emit('notificationError', {
|
|
84
86
|
type: 'email',
|
|
85
87
|
url,
|
|
86
|
-
|
|
87
|
-
note: 'Email notifications require external service integration'
|
|
88
|
+
error: 'Local monitors do not send email; create the monitor with scheduledMonitorOptions.hosted: true (or in the website dashboard) for email notifications'
|
|
88
89
|
});
|
|
89
90
|
}
|
|
90
91
|
|