crawlforge-mcp-server 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/package.json +1 -1
- package/server.js +23 -108
- package/src/cli/commands/login.js +176 -0
- package/src/cli/commands/monitor.js +18 -4
- package/src/cli/index.js +2 -0
- package/src/core/AuthManager.js +19 -6
- package/src/core/ElicitationHelper.js +157 -76
- package/src/server/requestContext.js +50 -0
- package/src/server/transports/streamableHttp.js +17 -4
- package/src/server/withAuth.js +47 -9
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +53 -14
- package/src/tools/advanced/batchScrape/index.js +29 -19
- package/src/tools/agent/agent.js +9 -4
- package/src/tools/crawl/crawlDeep.js +10 -4
- package/src/tools/extract/extractStructured.js +62 -43
- package/src/tools/research/deepResearch.js +10 -4
- package/src/tools/tracking/trackChanges/hosted.js +176 -0
- package/src/tools/tracking/trackChanges/index.js +123 -7
- package/src/tools/tracking/trackChanges/notifier.js +5 -4
- package/src/tools/tracking/trackChanges/schema.js +36 -22
- package/src/core/AlertNotificationSystem.js +0 -602
|
@@ -83,41 +83,51 @@ export class BatchScrapeTool extends EventEmitter {
|
|
|
83
83
|
this._elicitation = new ElicitationHelper({ mcpServer });
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
-
async execute(params) {
|
|
86
|
+
async execute(params, ctx) {
|
|
87
87
|
try {
|
|
88
88
|
const validated = BatchScrapeSchema.parse(params);
|
|
89
|
-
this.stats.totalBatches++;
|
|
90
89
|
const batchId = this._generateBatchId();
|
|
91
|
-
const startTime = Date.now();
|
|
92
|
-
|
|
93
|
-
this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
|
|
94
|
-
|
|
95
|
-
const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
|
|
96
90
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
//
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
91
|
+
// D1.4: Elicitation — warn when batch is large in sync mode. An
|
|
92
|
+
// unanswered gate returns an input-required result the SDK answers by
|
|
93
|
+
// re-entering this handler from the top, so it runs before anything that
|
|
94
|
+
// leaves a trace: below it are the batch counter and the webhook
|
|
95
|
+
// registration, which a second entry would repeat (a second webhook
|
|
96
|
+
// registration for one batch). The count comes from validated.urls,
|
|
97
|
+
// which _normalizeUrlConfigs maps one-for-one.
|
|
98
|
+
if (validated.mode === 'sync' && validated.urls.length > 25) {
|
|
99
|
+
const gate = this._elicitation.confirm(
|
|
100
|
+
ctx,
|
|
101
|
+
'batch_scrape:large_sync',
|
|
102
|
+
`batch_scrape (sync mode) will fetch ${validated.urls.length} URLs synchronously. This may take a while and consume significant credits.`,
|
|
106
103
|
{
|
|
107
|
-
url_count:
|
|
104
|
+
url_count: validated.urls.length,
|
|
108
105
|
mode: 'sync',
|
|
109
106
|
suggestion: 'Consider using mode:"async" for large batches.',
|
|
110
107
|
}
|
|
111
108
|
);
|
|
112
|
-
if (
|
|
109
|
+
if (gate.status === 'ask') return gate.result;
|
|
110
|
+
if (gate.status === 'cancelled') {
|
|
113
111
|
return {
|
|
114
112
|
batchId, mode: 'sync', success: false,
|
|
115
113
|
error: 'Batch scrape cancelled by user (elicitation declined).',
|
|
116
|
-
totalUrls:
|
|
114
|
+
totalUrls: validated.urls.length,
|
|
117
115
|
};
|
|
118
116
|
}
|
|
119
117
|
}
|
|
120
118
|
|
|
119
|
+
this.stats.totalBatches++;
|
|
120
|
+
const startTime = Date.now();
|
|
121
|
+
|
|
122
|
+
this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
|
|
123
|
+
|
|
124
|
+
const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
|
|
125
|
+
|
|
126
|
+
let webhookConfig = null;
|
|
127
|
+
if (validated.webhook && this.enableWebhookNotifications) {
|
|
128
|
+
webhookConfig = this._registerWebhook(validated.webhook, batchId);
|
|
129
|
+
}
|
|
130
|
+
|
|
121
131
|
if (validated.mode === 'sync') {
|
|
122
132
|
return await this._processBatchSync(batchId, urlConfigs, validated, webhookConfig, startTime);
|
|
123
133
|
} else {
|
package/src/tools/agent/agent.js
CHANGED
|
@@ -35,16 +35,21 @@ export class AgentTool {
|
|
|
35
35
|
this._elicitation = new ElicitationHelper({ mcpServer });
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
async execute(params) {
|
|
38
|
+
async execute(params, ctx) {
|
|
39
39
|
const validated = AgentInputSchema.parse(params);
|
|
40
40
|
|
|
41
|
-
// Request confirmation before a pro run (expensive)
|
|
41
|
+
// Request confirmation before a pro run (expensive). An unanswered gate
|
|
42
|
+
// returns an input-required result the SDK answers by re-entering this
|
|
43
|
+
// handler from the top, so nothing above it may fetch or leave a trace.
|
|
42
44
|
if (validated.model === 'pro') {
|
|
43
|
-
const
|
|
45
|
+
const gate = this._elicitation.confirm(
|
|
46
|
+
ctx,
|
|
47
|
+
'agent:pro_model',
|
|
44
48
|
'agent tool: pro model uses ResearchOrchestrator and may incur significant costs.',
|
|
45
49
|
{ model: 'pro', maxUrls: validated.maxUrls, note: 'External LLM API costs billed separately if keys are set.' }
|
|
46
50
|
);
|
|
47
|
-
if (
|
|
51
|
+
if (gate.status === 'ask') return gate.result;
|
|
52
|
+
if (gate.status === 'cancelled') {
|
|
48
53
|
return {
|
|
49
54
|
success: false,
|
|
50
55
|
cancelled: true,
|
|
@@ -117,7 +117,7 @@ export class CrawlDeepTool {
|
|
|
117
117
|
this._elicitation = new ElicitationHelper({ mcpServer });
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
-
async execute(params) {
|
|
120
|
+
async execute(params, ctx) {
|
|
121
121
|
try {
|
|
122
122
|
const validated = CrawlDeepSchema.parse(params);
|
|
123
123
|
|
|
@@ -147,9 +147,14 @@ export class CrawlDeepTool {
|
|
|
147
147
|
if (cached) return { ...cached, cached: true };
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
-
// D1.4: Elicitation — warn when max_pages is very high
|
|
150
|
+
// D1.4: Elicitation — warn when max_pages is very high. An unanswered
|
|
151
|
+
// gate returns an input-required result the SDK answers by re-entering
|
|
152
|
+
// this handler from the top; everything above is the clamp arithmetic and
|
|
153
|
+
// a cache read, so a second entry fetches nothing and leaves no trace.
|
|
151
154
|
if (effectiveMaxPages > 500) {
|
|
152
|
-
const
|
|
155
|
+
const gate = this._elicitation.confirm(
|
|
156
|
+
ctx,
|
|
157
|
+
'crawl_deep:max_pages',
|
|
153
158
|
`crawl_deep will crawl up to ${effectiveMaxPages} pages from ${validated.url}. Large crawls consume many credits.`,
|
|
154
159
|
{
|
|
155
160
|
url: validated.url,
|
|
@@ -157,7 +162,8 @@ export class CrawlDeepTool {
|
|
|
157
162
|
max_depth: effectiveMaxDepth,
|
|
158
163
|
}
|
|
159
164
|
);
|
|
160
|
-
if (
|
|
165
|
+
if (gate.status === 'ask') return gate.result;
|
|
166
|
+
if (gate.status === 'cancelled') {
|
|
161
167
|
return {
|
|
162
168
|
success: false,
|
|
163
169
|
error: 'Crawl cancelled by user (elicitation declined).',
|
|
@@ -174,36 +174,78 @@ export class ExtractStructuredTool {
|
|
|
174
174
|
* @param {Object} params - Extraction parameters
|
|
175
175
|
* @returns {Promise<Object>} Extraction result
|
|
176
176
|
*/
|
|
177
|
-
async execute(params) {
|
|
177
|
+
async execute(params, ctx) {
|
|
178
178
|
const startTime = Date.now();
|
|
179
179
|
|
|
180
180
|
try {
|
|
181
181
|
const validated = ExtractStructuredSchema.parse(params);
|
|
182
182
|
const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
|
|
183
183
|
|
|
184
|
-
// Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
|
|
185
|
-
const { html, $, textContent, warnings } = await fetchAndParse(url, {
|
|
186
|
-
userAgent: user_agent || this.userAgent,
|
|
187
|
-
respectRobots: respect_robots,
|
|
188
|
-
tool: 'extract_structured'
|
|
189
|
-
});
|
|
190
|
-
|
|
191
|
-
// What the model reads — see shownText().
|
|
192
|
-
const shown = shownText($, html, url, textContent);
|
|
193
|
-
|
|
194
|
-
// Step 3: Try LLM extraction first
|
|
195
184
|
let extractionResult = null;
|
|
196
185
|
let extractionMethod = 'llm';
|
|
197
186
|
let llmErrorMessage = null;
|
|
198
187
|
let llmAvailable = false;
|
|
188
|
+
let llm = null;
|
|
199
189
|
|
|
190
|
+
// Step 0: LLM readiness, resolved before the fetch so the D1.4 gate below
|
|
191
|
+
// can ask before any network work — an unanswered gate returns an
|
|
192
|
+
// input-required result the SDK answers by re-entering this handler from
|
|
193
|
+
// the top, and everything above the gate runs a second time.
|
|
200
194
|
try {
|
|
201
|
-
|
|
195
|
+
llm = this._ensureLLMManager(llmConfig || {});
|
|
202
196
|
// ready() probes Ollama, which has no API key to gate on. isAvailable()
|
|
203
197
|
// alone reported false on any machine without a cloud key, so a running
|
|
204
198
|
// local Ollama was never used.
|
|
205
199
|
llmAvailable = await llm.ready();
|
|
206
|
-
|
|
200
|
+
} catch (llmError) {
|
|
201
|
+
// No usable LLM — this falls through to the CSS fallback. Keep the
|
|
202
|
+
// message so callers can tell "LLM broken" apart from "no LLM
|
|
203
|
+
// configured".
|
|
204
|
+
llmErrorMessage = llmError.message;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// D1.4: no LLM configured and the schema demands more than 3 required
|
|
208
|
+
// fields — confirm before running the lower-fidelity CSS fallback. With
|
|
209
|
+
// no LLM, step 3 extracts nothing, so landing on that fallback is already
|
|
210
|
+
// settled here, before the page is fetched.
|
|
211
|
+
const requiredCount = (schema.required || []).length;
|
|
212
|
+
if (fallbackToSelectors !== false && !llmAvailable && requiredCount > 3) {
|
|
213
|
+
const gate = this._elicitation.confirm(
|
|
214
|
+
ctx,
|
|
215
|
+
'extract_structured:no_llm_required_fields',
|
|
216
|
+
`No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
|
|
217
|
+
`extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
|
|
218
|
+
{ url, required_fields: requiredCount }
|
|
219
|
+
);
|
|
220
|
+
if (gate.status === 'ask') return gate.result;
|
|
221
|
+
if (gate.status === 'cancelled') {
|
|
222
|
+
return {
|
|
223
|
+
success: false,
|
|
224
|
+
url,
|
|
225
|
+
data: {},
|
|
226
|
+
extraction_method: 'none',
|
|
227
|
+
confidence: 0,
|
|
228
|
+
schema_used: schema,
|
|
229
|
+
processingTime: Date.now() - startTime,
|
|
230
|
+
error: 'Extraction cancelled by user (elicitation declined).',
|
|
231
|
+
validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
|
|
237
|
+
const { html, $, textContent, warnings } = await fetchAndParse(url, {
|
|
238
|
+
userAgent: user_agent || this.userAgent,
|
|
239
|
+
respectRobots: respect_robots,
|
|
240
|
+
tool: 'extract_structured'
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
// What the model reads — see shownText().
|
|
244
|
+
const shown = shownText($, html, url, textContent);
|
|
245
|
+
|
|
246
|
+
// Step 3: Try LLM extraction first (readiness resolved in step 0)
|
|
247
|
+
if (llmAvailable) {
|
|
248
|
+
try {
|
|
207
249
|
const result = await llm.extractStructured(shown, schema, {
|
|
208
250
|
prompt: prompt || '',
|
|
209
251
|
maxContentLength: SHOWN_TEXT_BUDGET
|
|
@@ -218,12 +260,11 @@ export class ExtractStructuredTool {
|
|
|
218
260
|
} else {
|
|
219
261
|
llmErrorMessage = result?.error || 'LLM did not return usable JSON';
|
|
220
262
|
}
|
|
263
|
+
} catch (llmError) {
|
|
264
|
+
// LLM failed — will fall through to CSS fallback.
|
|
265
|
+
extractionResult = null;
|
|
266
|
+
llmErrorMessage = llmError.message;
|
|
221
267
|
}
|
|
222
|
-
} catch (llmError) {
|
|
223
|
-
// LLM failed — will fall through to CSS fallback. Keep the message so
|
|
224
|
-
// callers can tell "LLM broken" apart from "no LLM configured".
|
|
225
|
-
extractionResult = null;
|
|
226
|
-
llmErrorMessage = llmError.message;
|
|
227
268
|
}
|
|
228
269
|
|
|
229
270
|
// Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
|
|
@@ -318,31 +359,9 @@ export class ExtractStructuredTool {
|
|
|
318
359
|
if (guarded.checked.skipped) provenance.skipped = guarded.checked.skipped;
|
|
319
360
|
}
|
|
320
361
|
|
|
321
|
-
// Step 4: CSS selector fallback if LLM unavailable or failed
|
|
362
|
+
// Step 4: CSS selector fallback if LLM unavailable or failed (the D1.4
|
|
363
|
+
// confirmation for this path is gated above, before the fetch)
|
|
322
364
|
if (!extractionResult && fallbackToSelectors !== false) {
|
|
323
|
-
// D1.4: no LLM configured and the schema demands more than 3 required
|
|
324
|
-
// fields — confirm before running the lower-fidelity CSS fallback.
|
|
325
|
-
const requiredCount = (schema.required || []).length;
|
|
326
|
-
if (!llmAvailable && requiredCount > 3) {
|
|
327
|
-
const proceed = await this._elicitation.confirm(
|
|
328
|
-
`No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
|
|
329
|
-
`extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
|
|
330
|
-
{ url, required_fields: requiredCount }
|
|
331
|
-
);
|
|
332
|
-
if (!proceed) {
|
|
333
|
-
return {
|
|
334
|
-
success: false,
|
|
335
|
-
url,
|
|
336
|
-
data: {},
|
|
337
|
-
extraction_method: 'none',
|
|
338
|
-
confidence: 0,
|
|
339
|
-
schema_used: schema,
|
|
340
|
-
processingTime: Date.now() - startTime,
|
|
341
|
-
error: 'Extraction cancelled by user (elicitation declined).',
|
|
342
|
-
validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
365
|
extractionResult = this._cssExtraction($, schema, selectorHints || {});
|
|
347
366
|
extractionMethod = 'css_fallback';
|
|
348
367
|
}
|
|
@@ -115,7 +115,7 @@ export class DeepResearchTool {
|
|
|
115
115
|
this._elicitation = new ElicitationHelper({ mcpServer });
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
async execute(params) {
|
|
118
|
+
async execute(params, ctx) {
|
|
119
119
|
try {
|
|
120
120
|
const validated = DeepResearchSchema.parse(params);
|
|
121
121
|
const sessionId = this.generateSessionId();
|
|
@@ -137,10 +137,15 @@ export class DeepResearchTool {
|
|
|
137
137
|
}
|
|
138
138
|
|
|
139
139
|
// D1.4: Elicitation — warn user if projected cost exceeds 50 credits
|
|
140
|
-
// deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm
|
|
140
|
+
// deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm.
|
|
141
|
+
// An unanswered gate returns an input-required result the SDK answers by
|
|
142
|
+
// re-entering this handler from the top, so it sits above the session
|
|
143
|
+
// registration below — a second entry would otherwise leak a session.
|
|
141
144
|
if (validated.maxUrls > 50) {
|
|
142
145
|
const projectedCredits = validated.maxUrls;
|
|
143
|
-
const
|
|
146
|
+
const gate = this._elicitation.confirm(
|
|
147
|
+
ctx,
|
|
148
|
+
'deep_research:max_urls',
|
|
144
149
|
`deep_research will scan up to ${validated.maxUrls} URLs, projecting ~${projectedCredits} credits.`,
|
|
145
150
|
{
|
|
146
151
|
topic: validated.topic,
|
|
@@ -148,7 +153,8 @@ export class DeepResearchTool {
|
|
|
148
153
|
max_urls: validated.maxUrls,
|
|
149
154
|
}
|
|
150
155
|
);
|
|
151
|
-
if (
|
|
156
|
+
if (gate.status === 'ask') return gate.result;
|
|
157
|
+
if (gate.status === 'cancelled') {
|
|
152
158
|
return {
|
|
153
159
|
success: false,
|
|
154
160
|
error: 'Research cancelled by user before starting (elicitation declined).',
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TrackChanges — hosted monitors (Phase 6.1).
|
|
3
|
+
*
|
|
4
|
+
* `scheduledMonitorOptions.hosted: true` registers a monitor with the
|
|
5
|
+
* website's /api/v1/monitors instead of the local MonitorStore. The website's
|
|
6
|
+
* cron then fetches, compares, bills the account and sends the notifications
|
|
7
|
+
* (email, signed webhooks), so the monitor fires whether or not this process
|
|
8
|
+
* is alive. This module is the thin client; index.js decides when to use it
|
|
9
|
+
* and shapes the tool results.
|
|
10
|
+
*
|
|
11
|
+
* These calls go to our own configured backend (AuthManager.apiEndpoint, from
|
|
12
|
+
* CRAWLFORGE_API_URL through endpointGuard), not to a caller-supplied URL, so
|
|
13
|
+
* they use bare fetch with the X-API-Key header exactly as AuthManager does.
|
|
14
|
+
* The SSRF guard is for pages a caller names; the endpoint is legitimately
|
|
15
|
+
* localhost in development.
|
|
16
|
+
*/
|
|
17
|
+
import authManager from '../../../core/AuthManager.js';
|
|
18
|
+
|
|
19
|
+
const HOSTED_TIMEOUT_MS = 30_000;
|
|
20
|
+
const MINUTE = 60_000;
|
|
21
|
+
const HOUR = 60 * MINUTE;
|
|
22
|
+
|
|
23
|
+
export const HOSTED_FIRING_GUARANTEE_NOTE =
|
|
24
|
+
"Runs from CrawlForge's scheduler whether or not this process is alive. Each check bills " +
|
|
25
|
+
'3 credits per compared target from the account; blocked and errored targets are free.';
|
|
26
|
+
|
|
27
|
+
export const NO_KEY_MESSAGE =
|
|
28
|
+
'hosted monitors need a CrawlForge API key — run `crawlforge-setup` or `crawlforge login`';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The endpoint and key the hosted calls authenticate with, from the sources
|
|
32
|
+
* the server and the CLI already use: CRAWLFORGE_API_KEY (the CLI's preAction
|
|
33
|
+
* hook fills it from --api-key or the stored config), then the key AuthManager
|
|
34
|
+
* loaded at startup, then the stored config read directly — the server skips
|
|
35
|
+
* loading it in creator mode, and a hosted monitor is billed to an account
|
|
36
|
+
* either way. No network: initialize() would re-validate the key.
|
|
37
|
+
*/
|
|
38
|
+
export async function resolveHostedCredentials() {
|
|
39
|
+
let apiKey = process.env.CRAWLFORGE_API_KEY || authManager.getConfig()?.apiKey;
|
|
40
|
+
if (!apiKey) {
|
|
41
|
+
try {
|
|
42
|
+
await authManager.loadConfig();
|
|
43
|
+
apiKey = authManager.getConfig()?.apiKey;
|
|
44
|
+
} catch {
|
|
45
|
+
/* no stored config */
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (!apiKey) throw new Error(NO_KEY_MESSAGE);
|
|
49
|
+
return { endpoint: authManager.apiEndpoint, apiKey };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// resolveApiEndpoint keeps a trailing slash on the configured endpoint; joined
|
|
53
|
+
// to an absolute path that is `//api/...`, a redirect on every call and a
|
|
54
|
+
// dashboard link with a double slash.
|
|
55
|
+
const base = (endpoint) => String(endpoint).replace(/\/+$/, '');
|
|
56
|
+
|
|
57
|
+
async function request(method, pathname, creds, body) {
|
|
58
|
+
const response = await fetch(`${base(creds.endpoint)}${pathname}`, {
|
|
59
|
+
method,
|
|
60
|
+
headers: {
|
|
61
|
+
'X-API-Key': creds.apiKey,
|
|
62
|
+
...(body ? { 'Content-Type': 'application/json' } : {})
|
|
63
|
+
},
|
|
64
|
+
...(body ? { body: JSON.stringify(body) } : {}),
|
|
65
|
+
signal: AbortSignal.timeout(HOSTED_TIMEOUT_MS)
|
|
66
|
+
});
|
|
67
|
+
let payload = null;
|
|
68
|
+
try {
|
|
69
|
+
payload = await response.json();
|
|
70
|
+
} catch {
|
|
71
|
+
/* no JSON body */
|
|
72
|
+
}
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
// `{ error: { code, message, details? } }` from the monitors API; the
|
|
75
|
+
// API-key middleware's 401 is the same envelope. The website's own words
|
|
76
|
+
// reach the caller so a validation or robots refusal is readable.
|
|
77
|
+
const err = payload?.error;
|
|
78
|
+
const code = err?.code || `HTTP_${response.status}`;
|
|
79
|
+
const message = (typeof err === 'string' ? err : err?.message) || response.statusText || 'request failed';
|
|
80
|
+
const details = err?.details !== undefined ? ` ${JSON.stringify(err.details)}` : '';
|
|
81
|
+
const failure = new Error(`${code}: ${message}${details}`);
|
|
82
|
+
failure.code = code;
|
|
83
|
+
failure.status = response.status;
|
|
84
|
+
throw failure;
|
|
85
|
+
}
|
|
86
|
+
return payload?.data;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function createHostedMonitor(input, creds) {
|
|
90
|
+
return request('POST', '/api/v1/monitors', creds, input);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export async function listHostedMonitors(creds) {
|
|
94
|
+
// An account holds at most 50 monitors, so one page is the whole list.
|
|
95
|
+
return (await request('GET', '/api/v1/monitors?limit=100', creds)) ?? [];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function deleteHostedMonitor(id, creds) {
|
|
99
|
+
return request('DELETE', `/api/v1/monitors/${encodeURIComponent(id)}`, creds);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The cron slots the website accepts: consecutive runs at least 5 minutes
|
|
103
|
+
// apart. For a `*/N` minute step that means N must divide 60 (`*/7` has a
|
|
104
|
+
// 4-minute gap at the top of every hour); for an hour step, H must divide 24.
|
|
105
|
+
const SLOTS = [
|
|
106
|
+
...[5, 6, 10, 12, 15, 20, 30].map((m) => ({ ms: m * MINUTE, cron: `*/${m} * * * *` })),
|
|
107
|
+
{ ms: HOUR, cron: '0 * * * *' },
|
|
108
|
+
...[2, 3, 4, 6, 8, 12].map((h) => ({ ms: h * HOUR, cron: `0 */${h} * * *` })),
|
|
109
|
+
{ ms: 24 * HOUR, cron: '0 0 * * *' }
|
|
110
|
+
];
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The hosted schedule for a polling interval in ms.
|
|
114
|
+
* @returns {{ cron: string, effectiveIntervalMs: number, adjusted: boolean }}
|
|
115
|
+
* `adjusted` is true when the interval was not an accepted slot and the
|
|
116
|
+
* nearest one was used; a tie goes to the longer interval (fewer billed checks).
|
|
117
|
+
*/
|
|
118
|
+
export function intervalToCron(ms) {
|
|
119
|
+
let best = SLOTS[0];
|
|
120
|
+
for (const slot of SLOTS) {
|
|
121
|
+
const d = Math.abs(slot.ms - ms);
|
|
122
|
+
const bestD = Math.abs(best.ms - ms);
|
|
123
|
+
if (d < bestD || (d === bestD && slot.ms > best.ms)) best = slot;
|
|
124
|
+
}
|
|
125
|
+
return { cron: best.cron, effectiveIntervalMs: best.ms, adjusted: best.ms !== ms };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function formatInterval(ms) {
|
|
129
|
+
return ms % HOUR === 0 ? `${ms / HOUR} h` : `${Math.round(ms / MINUTE)} min`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const parseIso = (iso) => (iso ? Date.parse(iso) || null : null);
|
|
133
|
+
|
|
134
|
+
export function hostedDashboardUrl(endpoint, id) {
|
|
135
|
+
return `${base(endpoint)}/dashboard/monitors/${id}`;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The `monitor` a hosted create_scheduled_monitor returns. */
|
|
139
|
+
export function createdHostedMonitor(record, endpoint) {
|
|
140
|
+
return {
|
|
141
|
+
id: record.id,
|
|
142
|
+
hosted: true,
|
|
143
|
+
name: record.name,
|
|
144
|
+
targets: record.targets,
|
|
145
|
+
schedule: record.schedule_cron,
|
|
146
|
+
timezone: record.timezone,
|
|
147
|
+
notifyEmails: record.notify_emails,
|
|
148
|
+
webhookUrl: record.webhook_url,
|
|
149
|
+
webhookSecret: record.webhook_secret,
|
|
150
|
+
status: record.status,
|
|
151
|
+
nextRunAt: parseIso(record.next_run_at),
|
|
152
|
+
estimatedCreditsPerMonth: record.estimated_credits_per_month,
|
|
153
|
+
dashboardUrl: hostedDashboardUrl(endpoint, record.id)
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A hosted monitor as list_scheduled_monitors shows it, beside the local ones. */
|
|
158
|
+
export function listedHostedMonitor(record, endpoint) {
|
|
159
|
+
const active = record.status === 'active';
|
|
160
|
+
return {
|
|
161
|
+
id: record.id,
|
|
162
|
+
hosted: true,
|
|
163
|
+
url: record.targets?.[0]?.url,
|
|
164
|
+
targets: record.targets,
|
|
165
|
+
name: record.name,
|
|
166
|
+
schedule: record.schedule_cron,
|
|
167
|
+
timezone: record.timezone,
|
|
168
|
+
enabled: active,
|
|
169
|
+
nextDueAt: parseIso(record.next_run_at),
|
|
170
|
+
lastCheckAt: parseIso(record.last_check_at),
|
|
171
|
+
lastCheck: record.last_check ?? null,
|
|
172
|
+
estimatedCreditsPerMonth: record.estimated_credits_per_month,
|
|
173
|
+
dashboardUrl: hostedDashboardUrl(endpoint, record.id),
|
|
174
|
+
scheduled: active
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -21,10 +21,18 @@ import SnapshotManager from '../../../core/SnapshotManager.js';
|
|
|
21
21
|
import CacheManager from '../../../core/cache/CacheManager.js';
|
|
22
22
|
import { MonitorStore } from '../../../core/MonitorStore.js';
|
|
23
23
|
import { MonitorScheduler } from '../../../core/MonitorScheduler.js';
|
|
24
|
+
import { setActualCost } from '../../../server/requestContext.js';
|
|
24
25
|
import { TrackChangesSchema } from './schema.js';
|
|
25
26
|
import { fetchContent, mergeHistoryData, matchesSignificanceFilter, calculateAverageInterval, calculateSignificanceDistribution } from './differ.js';
|
|
26
27
|
import { performMonitoringCheck, stopMonitor } from './monitor.js';
|
|
27
28
|
import { sendNotifications } from './notifier.js';
|
|
29
|
+
import {
|
|
30
|
+
HOSTED_FIRING_GUARANTEE_NOTE, createHostedMonitor, createdHostedMonitor, deleteHostedMonitor,
|
|
31
|
+
formatInterval, intervalToCron, listHostedMonitors, listedHostedMonitor, resolveHostedCredentials
|
|
32
|
+
} from './hosted.js';
|
|
33
|
+
|
|
34
|
+
// server.js spreads this into the registered inputSchema (G5: one declaration).
|
|
35
|
+
export { TRACK_CHANGES_INPUT_SHAPE } from './schema.js';
|
|
28
36
|
|
|
29
37
|
export class TrackChangesTool extends EventEmitter {
|
|
30
38
|
constructor(options = {}) {
|
|
@@ -42,6 +50,9 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
42
50
|
enableRealTimeMonitoring: true,
|
|
43
51
|
maxConcurrentMonitors: 50,
|
|
44
52
|
defaultPollingInterval: 300000,
|
|
53
|
+
// The key and endpoint hosted monitors authenticate with; tests inject
|
|
54
|
+
// a stub so nothing reads ~/.crawlforge or reaches the website.
|
|
55
|
+
resolveHostedCredentials,
|
|
45
56
|
...options
|
|
46
57
|
};
|
|
47
58
|
|
|
@@ -404,6 +415,9 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
404
415
|
);
|
|
405
416
|
}
|
|
406
417
|
}
|
|
418
|
+
if (opts.hosted) {
|
|
419
|
+
return this._createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions });
|
|
420
|
+
}
|
|
407
421
|
// Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
|
|
408
422
|
// schema fills monitoringOptions.interval/notificationThreshold with
|
|
409
423
|
// defaults, so they cannot sit above a preset without always winning.
|
|
@@ -423,25 +437,127 @@ export class TrackChangesTool extends EventEmitter {
|
|
|
423
437
|
};
|
|
424
438
|
}
|
|
425
439
|
|
|
440
|
+
/**
|
|
441
|
+
* Hosted (6.1): the website's /api/v1/monitors owns the monitor — its cron
|
|
442
|
+
* fetches, compares, bills and notifies — so nothing is stored or fetched
|
|
443
|
+
* here. The interval precedence matches the local path except that
|
|
444
|
+
* monitoringOptions.interval (schema-defaulted to 5 min) is not consulted:
|
|
445
|
+
* a hosted check is billed, and the website's own default is hourly.
|
|
446
|
+
*/
|
|
447
|
+
async _createHostedMonitor({ url, opts, preset, trackingOptions, notificationOptions }) {
|
|
448
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
449
|
+
const warnings = [];
|
|
450
|
+
let scheduleCron = opts.schedule;
|
|
451
|
+
const interval = opts.interval ?? preset?.frequency;
|
|
452
|
+
if (!scheduleCron && interval) {
|
|
453
|
+
const slot = intervalToCron(interval);
|
|
454
|
+
scheduleCron = slot.cron;
|
|
455
|
+
if (slot.adjusted) {
|
|
456
|
+
warnings.push(
|
|
457
|
+
`interval ${formatInterval(interval)} is not a hosted schedule slot; the monitor runs every ` +
|
|
458
|
+
`${formatInterval(slot.effectiveIntervalMs)} (${slot.cron}). Hosted runs are at least 5 minutes apart ` +
|
|
459
|
+
'and divide the hour or the day evenly.'
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (opts.goal ?? preset?.goal) {
|
|
464
|
+
warnings.push('goal is judged by the local goal judge only and is not applied to a hosted monitor, which notifies on every changed, new, blocked or errored page');
|
|
465
|
+
}
|
|
466
|
+
if (opts.notificationThreshold) {
|
|
467
|
+
warnings.push('notificationThreshold has no effect on a hosted monitor; hosted checks have no significance threshold');
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const tracking = preset ? { ...preset.options, ...(trackingOptions || {}) } : (trackingOptions || {});
|
|
471
|
+
const selectors = tracking.customSelectors || [];
|
|
472
|
+
const email = notificationOptions?.email;
|
|
473
|
+
const webhook = notificationOptions?.webhook;
|
|
474
|
+
const secret = webhook?.signingSecret;
|
|
475
|
+
const record = await createHostedMonitor({
|
|
476
|
+
name: opts.name || new URL(url).host.slice(0, 80),
|
|
477
|
+
targets: selectors.length ? selectors.map((selector) => ({ url, selector })) : [{ url }],
|
|
478
|
+
...(scheduleCron ? { schedule_cron: scheduleCron } : {}),
|
|
479
|
+
timezone: 'UTC',
|
|
480
|
+
...(email?.enabled && email.recipients?.length ? { notify_emails: email.recipients } : {}),
|
|
481
|
+
...(webhook?.enabled && webhook.url ? { webhook_url: webhook.url } : {}),
|
|
482
|
+
// A secret outside 16-128 chars is left out so the website generates one.
|
|
483
|
+
...(webhook?.enabled && webhook.url && secret?.length >= 16 && secret.length <= 128 ? { webhook_secret: secret } : {}),
|
|
484
|
+
status: 'active'
|
|
485
|
+
}, creds);
|
|
486
|
+
// Nothing ran on this machine and the monitors API is free (G4).
|
|
487
|
+
setActualCost(0);
|
|
488
|
+
return {
|
|
489
|
+
success: true, operation: 'create_scheduled_monitor', url, hosted: true,
|
|
490
|
+
...(preset ? { templateId: preset.id } : {}),
|
|
491
|
+
monitor: createdHostedMonitor(record, creds.endpoint),
|
|
492
|
+
firingGuarantee: HOSTED_FIRING_GUARANTEE_NOTE,
|
|
493
|
+
...(warnings.length ? { warnings } : {}),
|
|
494
|
+
timestamp: Date.now()
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
|
|
426
498
|
async stopScheduledMonitor(params) {
|
|
427
499
|
const { url, scheduledMonitorOptions } = params;
|
|
428
500
|
const monitorId = scheduledMonitorOptions?.monitorId;
|
|
429
501
|
if (monitorId) {
|
|
430
|
-
|
|
431
|
-
if (
|
|
432
|
-
|
|
502
|
+
if (!this.monitorStore._loaded) await this.monitorStore.load();
|
|
503
|
+
if (this.monitorStore.get(monitorId)) {
|
|
504
|
+
await this.scheduler.stopMonitor(monitorId);
|
|
505
|
+
return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
|
|
433
506
|
}
|
|
434
|
-
|
|
507
|
+
// Not in the local store: it may be hosted.
|
|
508
|
+
try {
|
|
509
|
+
await deleteHostedMonitor(monitorId, await this.options.resolveHostedCredentials());
|
|
510
|
+
} catch (error) {
|
|
511
|
+
const reason = error.status === 404 ? '' : ` (hosted lookup failed: ${error.message})`;
|
|
512
|
+
return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}${reason}`, timestamp: Date.now() };
|
|
513
|
+
}
|
|
514
|
+
// Nothing ran on this machine and the monitors API is free (G4).
|
|
515
|
+
setActualCost(0);
|
|
516
|
+
return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, hosted: true, timestamp: Date.now() };
|
|
435
517
|
}
|
|
436
518
|
if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
|
|
437
519
|
const result = await this.scheduler.stopByUrl(url);
|
|
438
|
-
|
|
520
|
+
// Only a hosted monitor whose every target is this URL — never a
|
|
521
|
+
// multi-target monitor that merely includes it.
|
|
522
|
+
let stoppedHosted = 0;
|
|
523
|
+
let hostedError = null;
|
|
524
|
+
try {
|
|
525
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
526
|
+
for (const m of await listHostedMonitors(creds)) {
|
|
527
|
+
if (m.targets?.length && m.targets.every((t) => t.url === url)) {
|
|
528
|
+
await deleteHostedMonitor(m.id, creds);
|
|
529
|
+
stoppedHosted++;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
} catch (error) {
|
|
533
|
+
hostedError = error.message;
|
|
534
|
+
}
|
|
535
|
+
return {
|
|
536
|
+
success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors: result.stopped, stoppedHosted,
|
|
537
|
+
...(hostedError ? { hostedError } : {}),
|
|
538
|
+
timestamp: Date.now()
|
|
539
|
+
};
|
|
439
540
|
}
|
|
440
541
|
|
|
441
542
|
async listScheduledMonitors() {
|
|
442
543
|
if (!this.monitorStore._loaded) await this.monitorStore.load();
|
|
443
|
-
const
|
|
444
|
-
|
|
544
|
+
const local = this.scheduler.list().map((m) => ({ ...m, hosted: false }));
|
|
545
|
+
// The local list never fails because the website is unreachable.
|
|
546
|
+
let hosted = [];
|
|
547
|
+
let hostedError = null;
|
|
548
|
+
try {
|
|
549
|
+
const creds = await this.options.resolveHostedCredentials();
|
|
550
|
+
hosted = (await listHostedMonitors(creds)).map((r) => listedHostedMonitor(r, creds.endpoint));
|
|
551
|
+
} catch (error) {
|
|
552
|
+
hostedError = error.message;
|
|
553
|
+
}
|
|
554
|
+
const monitors = [...local, ...hosted];
|
|
555
|
+
return {
|
|
556
|
+
success: true, operation: 'list_scheduled_monitors', monitors,
|
|
557
|
+
count: monitors.length, localCount: local.length, hostedCount: hosted.length,
|
|
558
|
+
...(hostedError ? { hostedError } : {}),
|
|
559
|
+
timestamp: Date.now()
|
|
560
|
+
};
|
|
445
561
|
}
|
|
446
562
|
|
|
447
563
|
async getMonitoringDashboard(params) {
|