amalgm 0.1.86 → 0.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/lib/harnesses.js +15 -2
- package/runtime/scripts/amalgm-mcp/automations/rest.js +41 -3
- package/runtime/scripts/amalgm-mcp/automations/runner.js +111 -15
- package/runtime/scripts/amalgm-mcp/automations/scheduler.js +116 -7
- package/runtime/scripts/amalgm-mcp/automations/store.js +508 -66
- package/runtime/scripts/amalgm-mcp/automations/tools.js +55 -10
- package/runtime/scripts/amalgm-mcp/events/ingress.js +142 -29
- package/runtime/scripts/amalgm-mcp/events/matcher.js +9 -4
- package/runtime/scripts/amalgm-mcp/server/routes/automations.js +4 -0
- package/runtime/scripts/amalgm-mcp/state/db.js +12 -0
- package/runtime/scripts/amalgm-mcp/state/snapshot.js +2 -2
- package/runtime/scripts/amalgm-mcp/tests/automations-reliability.test.js +547 -0
- package/runtime/scripts/amalgm-mcp/tests/automations-store-runner.test.js +3 -3
- package/runtime/scripts/chat-core/contract.js +10 -0
- package/runtime/scripts/chat-server/model-catalog.js +2 -1
package/package.json
CHANGED
package/runtime/lib/harnesses.js
CHANGED
|
@@ -105,7 +105,10 @@ var CLAUDE_LEGACY_ALIASES = {
|
|
|
105
105
|
"opus[1m]": CLAUDE_OPUS_4_8_1M_MODEL_ID,
|
|
106
106
|
sonnet: "anthropic/claude-sonnet-4.6",
|
|
107
107
|
"sonnet[1m]": "anthropic/claude-sonnet-4.6",
|
|
108
|
-
haiku: "anthropic/claude-haiku-4.5"
|
|
108
|
+
haiku: "anthropic/claude-haiku-4.5",
|
|
109
|
+
fable: "anthropic/claude-fable-5",
|
|
110
|
+
"claude-code-fable": "anthropic/claude-fable-5",
|
|
111
|
+
"claude-code-fable-5": "anthropic/claude-fable-5"
|
|
109
112
|
};
|
|
110
113
|
var CLAUDE_HARNESS_MODELS = {
|
|
111
114
|
"anthropic/claude-opus-4.8": "opus",
|
|
@@ -115,7 +118,8 @@ var CLAUDE_HARNESS_MODELS = {
|
|
|
115
118
|
"anthropic/claude-opus-4.5": "opus",
|
|
116
119
|
"anthropic/claude-sonnet-4.6": "sonnet",
|
|
117
120
|
"anthropic/claude-sonnet-4.5": "sonnet",
|
|
118
|
-
"anthropic/claude-haiku-4.5": "haiku"
|
|
121
|
+
"anthropic/claude-haiku-4.5": "haiku",
|
|
122
|
+
"anthropic/claude-fable-5": "claude-fable-5"
|
|
119
123
|
};
|
|
120
124
|
var INTERNAL_TO_GATEWAY_MODEL_IDS = {
|
|
121
125
|
[CLAUDE_OPUS_4_8_1M_MODEL_ID]: "anthropic/claude-opus-4.8"
|
|
@@ -430,6 +434,15 @@ var CLAUDE_MODELS = [
|
|
|
430
434
|
cliModel: modelIdToHarnessModel("anthropic/claude-opus-4.8-1m", "claude_code"),
|
|
431
435
|
provider: "claude",
|
|
432
436
|
providerGroup: "anthropic"
|
|
437
|
+
},
|
|
438
|
+
{
|
|
439
|
+
id: "anthropic/claude-fable-5",
|
|
440
|
+
name: "Fable 5",
|
|
441
|
+
description: "Anthropic's most capable model",
|
|
442
|
+
gatewayModelId: "anthropic/claude-fable-5",
|
|
443
|
+
cliModel: modelIdToHarnessModel("anthropic/claude-fable-5", "claude_code"),
|
|
444
|
+
provider: "claude",
|
|
445
|
+
providerGroup: "anthropic"
|
|
433
446
|
}
|
|
434
447
|
];
|
|
435
448
|
var OPENCODE_MODELS = [
|
|
@@ -10,9 +10,11 @@ const {
|
|
|
10
10
|
deleteAutomationByTriggerId,
|
|
11
11
|
getAutomation,
|
|
12
12
|
getAutomationByTriggerId,
|
|
13
|
+
getAutomationRun,
|
|
13
14
|
listAutomationRuns,
|
|
14
15
|
listAutomations,
|
|
15
16
|
listTriggers,
|
|
17
|
+
listWorkflowCellRuns,
|
|
16
18
|
listWorkflows,
|
|
17
19
|
updateAutomation,
|
|
18
20
|
updateAutomationByTriggerId,
|
|
@@ -43,10 +45,44 @@ async function handleGet(body, sendJson) {
|
|
|
43
45
|
if (!automation) return sendJson(404, { error: 'Automation not found' });
|
|
44
46
|
sendJson(200, {
|
|
45
47
|
automation,
|
|
46
|
-
runs: listAutomationRuns({
|
|
48
|
+
runs: listAutomationRuns({
|
|
49
|
+
automationId: automation.id,
|
|
50
|
+
limit: Number(body?.limit) || 50,
|
|
51
|
+
offset: Number(body?.offset) || 0,
|
|
52
|
+
}),
|
|
47
53
|
});
|
|
48
54
|
}
|
|
49
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Paginated run history, plus run detail (per-cell results) when run_id is
|
|
58
|
+
* given. This is the endpoint the UI run views should use instead of the
|
|
59
|
+
* global 100-row state snapshot.
|
|
60
|
+
*/
|
|
61
|
+
async function handleRuns(body, sendJson) {
|
|
62
|
+
try {
|
|
63
|
+
const runId = body?.run_id || body?.runId;
|
|
64
|
+
if (runId) {
|
|
65
|
+
const run = getAutomationRun(runId);
|
|
66
|
+
if (!run) return sendJson(404, { error: 'Run not found' });
|
|
67
|
+
return sendJson(200, { ok: true, run, cellRuns: listWorkflowCellRuns({ runId: run.id }) });
|
|
68
|
+
}
|
|
69
|
+
const automationId = body?.automation_id || body?.automationId;
|
|
70
|
+
if (!automationId) return sendJson(400, { error: 'automation_id or run_id is required' });
|
|
71
|
+
const limit = Math.max(1, Math.min(Number(body?.limit) || 50, 500));
|
|
72
|
+
const offset = Math.max(0, Number(body?.offset) || 0);
|
|
73
|
+
const page = listAutomationRuns({ automationId, limit: limit + 1, offset });
|
|
74
|
+
sendJson(200, {
|
|
75
|
+
ok: true,
|
|
76
|
+
runs: page.slice(0, limit),
|
|
77
|
+
hasMore: page.length > limit,
|
|
78
|
+
limit,
|
|
79
|
+
offset,
|
|
80
|
+
});
|
|
81
|
+
} catch (error) {
|
|
82
|
+
sendError(sendJson, 400, error);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
50
86
|
async function handleCreate(body, sendJson) {
|
|
51
87
|
try {
|
|
52
88
|
const automation = createAutomation(body || {}, { source: 'automations:rest:create' });
|
|
@@ -99,7 +135,7 @@ async function handleRunNow(body, sendJson) {
|
|
|
99
135
|
try {
|
|
100
136
|
const id = body?.automation_id || body?.automationId;
|
|
101
137
|
if (!id) return sendJson(400, { error: 'automation_id is required' });
|
|
102
|
-
const
|
|
138
|
+
const runs = await executeAutomationById(id, {
|
|
103
139
|
id: `manual:${Date.now()}`,
|
|
104
140
|
source: 'amalgm.manual',
|
|
105
141
|
event: 'run_now',
|
|
@@ -107,7 +143,8 @@ async function handleRunNow(body, sendJson) {
|
|
|
107
143
|
headers: {},
|
|
108
144
|
receivedAt: new Date().toISOString(),
|
|
109
145
|
}, { triggerId: body?.trigger_id || body?.triggerId });
|
|
110
|
-
|
|
146
|
+
// One run per workflow; `run` keeps the legacy single-run shape.
|
|
147
|
+
sendJson(200, { ok: true, runs, run: runs[0] || null });
|
|
111
148
|
} catch (error) {
|
|
112
149
|
sendError(sendJson, 400, error);
|
|
113
150
|
}
|
|
@@ -141,6 +178,7 @@ module.exports = {
|
|
|
141
178
|
handleGet,
|
|
142
179
|
handleList,
|
|
143
180
|
handleRunNow,
|
|
181
|
+
handleRuns,
|
|
144
182
|
handleTriggerDelete,
|
|
145
183
|
handleTriggerUpdate,
|
|
146
184
|
handleUpdate,
|
|
@@ -9,16 +9,23 @@ const { DEFAULT_CWD } = require('../config');
|
|
|
9
9
|
const { compileWorkflowText } = require('../workflows/compiler');
|
|
10
10
|
const { callToolboxMcpTool, toolboxMcpToolName } = require('../toolbox/runner');
|
|
11
11
|
const { buildWorkflowInput } = require('./context');
|
|
12
|
-
const {
|
|
12
|
+
const {
|
|
13
|
+
resolveFirstPartyToolAction,
|
|
14
|
+
resolveToolboxAction,
|
|
15
|
+
suggestWorkflowToolActions,
|
|
16
|
+
workflowActionKeys,
|
|
17
|
+
} = require('./tool-actions');
|
|
13
18
|
const {
|
|
14
19
|
getAutomation,
|
|
15
20
|
getAutomationByTriggerId,
|
|
16
21
|
recordAutomationRun,
|
|
17
22
|
recordWorkflowCellRun,
|
|
23
|
+
touchAutomationRun,
|
|
18
24
|
} = require('./store');
|
|
19
25
|
|
|
20
26
|
const DEFAULT_CELL_TIMEOUT_MS = 120_000;
|
|
21
27
|
const DEFAULT_CODE_TIMEOUT_MS = 15_000;
|
|
28
|
+
const RUN_HEARTBEAT_MS = 60_000;
|
|
22
29
|
const MAX_OUTPUT_BYTES = 512 * 1024;
|
|
23
30
|
const runQueues = new Map();
|
|
24
31
|
|
|
@@ -242,15 +249,17 @@ async function runShellCommand(config, allowlist) {
|
|
|
242
249
|
};
|
|
243
250
|
}
|
|
244
251
|
|
|
245
|
-
async function runHttpTool(args) {
|
|
252
|
+
async function runHttpTool(args, options = {}) {
|
|
246
253
|
const method = String(args.method || 'GET').toUpperCase();
|
|
247
254
|
if (!args.url) throw new Error('http.fetch needs a url.');
|
|
255
|
+
const timeoutMs = Number(options.timeoutMs) || DEFAULT_CELL_TIMEOUT_MS;
|
|
248
256
|
const response = await fetch(args.url, {
|
|
249
257
|
method,
|
|
250
258
|
headers: args.headers || {},
|
|
251
259
|
body: ['GET', 'HEAD'].includes(method) ? undefined : (
|
|
252
260
|
typeof args.body === 'string' ? args.body : JSON.stringify(args.body ?? {})
|
|
253
261
|
),
|
|
262
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
254
263
|
});
|
|
255
264
|
const text = await response.text();
|
|
256
265
|
let body = text;
|
|
@@ -278,16 +287,20 @@ function mcpResultToOutput(result, actionKey) {
|
|
|
278
287
|
};
|
|
279
288
|
}
|
|
280
289
|
|
|
281
|
-
async function runToolCell(cell, args, allowlist) {
|
|
290
|
+
async function runToolCell(cell, args, allowlist, options = {}) {
|
|
282
291
|
const actionKey = `${cell.toolId}.${cell.actionName}`;
|
|
283
292
|
const allowedActions = Array.isArray(allowlist.actions) ? allowlist.actions : [];
|
|
284
|
-
|
|
293
|
+
// Match against every alias of the action (e.g. allowlist entries like
|
|
294
|
+
// "amalgm.talk_to_agent" must authorize "agents.talk_to_agent"), not just
|
|
295
|
+
// the raw key — otherwise alias-based allowlists silently block real calls.
|
|
296
|
+
const candidateKeys = workflowActionKeys(cell.toolId, cell.actionName);
|
|
297
|
+
if (allowedActions.length > 0 && !candidateKeys.some((key) => allowedActions.includes(key))) {
|
|
285
298
|
throw new Error(`Workflow is not allowed to call ${actionKey}.`);
|
|
286
299
|
}
|
|
287
300
|
|
|
288
301
|
if (cell.toolId === 'http' && cell.actionName === 'fetch') {
|
|
289
302
|
if (allowlist.network !== true) throw new Error('http.fetch requires allowlist.network=true.');
|
|
290
|
-
return runHttpTool(args);
|
|
303
|
+
return runHttpTool(args, options);
|
|
291
304
|
}
|
|
292
305
|
|
|
293
306
|
const firstPartyAction = resolveFirstPartyToolAction(cell.toolId, cell.actionName);
|
|
@@ -316,14 +329,27 @@ async function runToolCell(cell, args, allowlist) {
|
|
|
316
329
|
return mcpResultToOutput(result, actionKey);
|
|
317
330
|
}
|
|
318
331
|
|
|
319
|
-
function
|
|
320
|
-
const workflow = automation.workflow || automation.workflows?.[0];
|
|
332
|
+
function irForWorkflow(workflow) {
|
|
321
333
|
if (!workflow) throw new Error('Automation has no workflow.');
|
|
322
334
|
if (workflow.workflowIr && workflow.workflowIr.version === 1) return workflow.workflowIr;
|
|
323
335
|
if (workflow.workflowText || workflow.workflow) return compileWorkflowText(workflow.workflowText || workflow.workflow);
|
|
324
336
|
throw new Error('Automation workflow has no script.');
|
|
325
337
|
}
|
|
326
338
|
|
|
339
|
+
function workflowIrForAutomation(automation) {
|
|
340
|
+
return irForWorkflow(automation.workflow || automation.workflows?.[0]);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function automationWorkflows(automation) {
|
|
344
|
+
const workflows = (Array.isArray(automation.workflows) && automation.workflows.length > 0
|
|
345
|
+
? automation.workflows
|
|
346
|
+
: [automation.workflow]).filter(Boolean);
|
|
347
|
+
return {
|
|
348
|
+
workflows,
|
|
349
|
+
enabled: workflows.filter((workflow) => workflow.enabled !== false),
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
327
353
|
async function executeWorkflowCell(cell, ctx, allowlist, limits) {
|
|
328
354
|
const timeoutMs = Number(cell.timeoutMs || limits.cellTimeoutMs) || DEFAULT_CELL_TIMEOUT_MS;
|
|
329
355
|
const stop = () => {
|
|
@@ -350,21 +376,77 @@ async function executeWorkflowCell(cell, ctx, allowlist, limits) {
|
|
|
350
376
|
}
|
|
351
377
|
|
|
352
378
|
if (cell.kind === 'tool') {
|
|
379
|
+
// cellTimeoutMs is the wall-clock budget for the WHOLE cell: argument
|
|
380
|
+
// resolution and the tool call spend from the same budget, and the call
|
|
381
|
+
// itself is hard-bounded — one hung MCP tool, agent call, or fetch
|
|
382
|
+
// otherwise wedges the run (and its concurrency queue) in "running".
|
|
383
|
+
const cellStartedMs = Date.now();
|
|
353
384
|
const args = await resolveDynamic(cell.args || {}, input, { timeoutMs });
|
|
354
|
-
const
|
|
385
|
+
const remainingMs = Math.max(1000, timeoutMs - (Date.now() - cellStartedMs));
|
|
386
|
+
const output = await withTimeout(
|
|
387
|
+
runToolCell(cell, args, allowlist, { timeoutMs: remainingMs }),
|
|
388
|
+
remainingMs,
|
|
389
|
+
`Tool ${cell.toolId}.${cell.actionName}`,
|
|
390
|
+
);
|
|
355
391
|
return { output };
|
|
356
392
|
}
|
|
357
393
|
|
|
358
394
|
throw new Error(`Unsupported workflow cell kind "${cell.kind}".`);
|
|
359
395
|
}
|
|
360
396
|
|
|
397
|
+
/**
|
|
398
|
+
* Run every enabled workflow of the automation, in listed order, recording
|
|
399
|
+
* one run row per workflow. Workflows are independent: one failing must not
|
|
400
|
+
* skip the rest. Returns the array of run records (failed runs included).
|
|
401
|
+
* Throws only when no workflow produced a successful run, so single-workflow
|
|
402
|
+
* callers keep their old failure semantics.
|
|
403
|
+
*/
|
|
361
404
|
async function runAutomationNow(automation, trigger, eventObj = {}) {
|
|
362
405
|
if (!automation) throw new Error('Automation is required.');
|
|
363
406
|
if (automation.enabled === false) throw new Error(`Automation "${automation.name}" is disabled.`);
|
|
364
|
-
const
|
|
365
|
-
if (
|
|
366
|
-
if (
|
|
407
|
+
const { workflows, enabled } = automationWorkflows(automation);
|
|
408
|
+
if (workflows.length === 0) throw new Error(`Automation "${automation.name}" has no workflow.`);
|
|
409
|
+
if (enabled.length === 0) throw new Error(`Automation "${automation.name}" has no enabled workflow.`);
|
|
410
|
+
|
|
411
|
+
const runs = [];
|
|
412
|
+
let firstError = null;
|
|
413
|
+
for (const workflow of enabled) {
|
|
414
|
+
try {
|
|
415
|
+
runs.push(await runWorkflowNow(automation, workflow, trigger, eventObj));
|
|
416
|
+
} catch (error) {
|
|
417
|
+
if (!firstError) firstError = error;
|
|
418
|
+
if (error?.run) {
|
|
419
|
+
runs.push(error.run);
|
|
420
|
+
} else {
|
|
421
|
+
// The workflow failed before its start row was recorded (IR drift,
|
|
422
|
+
// store error) — leave a failed run row so the loss is visible.
|
|
423
|
+
try {
|
|
424
|
+
const timestamp = new Date().toISOString();
|
|
425
|
+
const failedRun = recordAutomationRun(automation.id, {
|
|
426
|
+
runId: crypto.randomUUID(),
|
|
427
|
+
automationId: automation.id,
|
|
428
|
+
automationName: automation.name,
|
|
429
|
+
triggerId: trigger?.id || null,
|
|
430
|
+
triggerName: trigger?.name || null,
|
|
431
|
+
workflowId: workflow.id || null,
|
|
432
|
+
workflowName: workflow.name || null,
|
|
433
|
+
startedAt: timestamp,
|
|
434
|
+
finishedAt: timestamp,
|
|
435
|
+
status: 'failed',
|
|
436
|
+
error: error.message || String(error),
|
|
437
|
+
}, { source: 'automation_runs:workflow:start-failed' });
|
|
438
|
+
if (failedRun) runs.push(failedRun);
|
|
439
|
+
} catch {
|
|
440
|
+
// Recording must never mask the original failure.
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
if (firstError && !runs.some((run) => run && run.status !== 'failed')) throw firstError;
|
|
446
|
+
return runs;
|
|
447
|
+
}
|
|
367
448
|
|
|
449
|
+
async function runWorkflowNow(automation, workflow, trigger, eventObj = {}) {
|
|
368
450
|
const runId = crypto.randomUUID();
|
|
369
451
|
const startedAt = new Date().toISOString();
|
|
370
452
|
const resolvedTrigger = trigger || automation.triggers?.[0] || {
|
|
@@ -373,7 +455,7 @@ async function runAutomationNow(automation, trigger, eventObj = {}) {
|
|
|
373
455
|
kind: 'manual',
|
|
374
456
|
automationId: automation.id,
|
|
375
457
|
};
|
|
376
|
-
const ir =
|
|
458
|
+
const ir = irForWorkflow(workflow);
|
|
377
459
|
const allowlist = {
|
|
378
460
|
...(ir.allowlist || {}),
|
|
379
461
|
...(workflow.allowlist || {}),
|
|
@@ -412,6 +494,13 @@ async function runAutomationNow(automation, trigger, eventObj = {}) {
|
|
|
412
494
|
},
|
|
413
495
|
}, { source: 'automation_runs:workflow:start' });
|
|
414
496
|
|
|
497
|
+
// Liveness heartbeat: run rows are otherwise only written at cell
|
|
498
|
+
// boundaries, so a single cell longer than the reaper's stale threshold
|
|
499
|
+
// would be falsely marked interrupted. unref'd so it never holds the
|
|
500
|
+
// process open.
|
|
501
|
+
const heartbeat = setInterval(() => touchAutomationRun(runId), RUN_HEARTBEAT_MS);
|
|
502
|
+
heartbeat.unref?.();
|
|
503
|
+
|
|
415
504
|
let status = 'completed';
|
|
416
505
|
try {
|
|
417
506
|
for (const cell of ir.cells) {
|
|
@@ -500,7 +589,7 @@ async function runAutomationNow(automation, trigger, eventObj = {}) {
|
|
|
500
589
|
}, { source: `automation_runs:workflow:${status}` });
|
|
501
590
|
return run;
|
|
502
591
|
} catch (error) {
|
|
503
|
-
recordAutomationRun(automation.id, {
|
|
592
|
+
const failedRun = recordAutomationRun(automation.id, {
|
|
504
593
|
runId,
|
|
505
594
|
automationId: automation.id,
|
|
506
595
|
automationName: automation.name,
|
|
@@ -513,7 +602,10 @@ async function runAutomationNow(automation, trigger, eventObj = {}) {
|
|
|
513
602
|
error: error.message || String(error),
|
|
514
603
|
durationMs: durationSince(startedAt),
|
|
515
604
|
}, { source: 'automation_runs:workflow:failed' });
|
|
605
|
+
if (failedRun) error.run = failedRun;
|
|
516
606
|
throw error;
|
|
607
|
+
} finally {
|
|
608
|
+
clearInterval(heartbeat);
|
|
517
609
|
}
|
|
518
610
|
}
|
|
519
611
|
|
|
@@ -533,8 +625,12 @@ function runQueuedAutomation(key, state, item) {
|
|
|
533
625
|
}
|
|
534
626
|
|
|
535
627
|
function executeAutomation(automation, trigger, eventObj = {}) {
|
|
536
|
-
|
|
537
|
-
|
|
628
|
+
// Concurrency gating reads limits from the first ENABLED workflow — a
|
|
629
|
+
// disabled primary is skipped by the runner, so its limits must not govern
|
|
630
|
+
// the firing.
|
|
631
|
+
const { enabled } = automationWorkflows(automation);
|
|
632
|
+
const workflow = enabled[0] || automation.workflow || automation.workflows?.[0];
|
|
633
|
+
const ir = irForWorkflow(workflow);
|
|
538
634
|
const limits = {
|
|
539
635
|
...(ir.limits || {}),
|
|
540
636
|
...(workflow?.limits || {}),
|
|
@@ -7,57 +7,142 @@
|
|
|
7
7
|
* scheduled triggers from the unified automation store and runs their
|
|
8
8
|
* workflows; it does not wake an agent unless the workflow explicitly calls
|
|
9
9
|
* an agent/tool cell.
|
|
10
|
+
*
|
|
11
|
+
* Reliability invariants:
|
|
12
|
+
* - next_run_at is persisted, so triggers due while the process was down are
|
|
13
|
+
* claimed by the first poll after boot/wake (single-shot catch-up).
|
|
14
|
+
* - Boot runs a repair pass that re-arms enabled triggers whose next_run_at
|
|
15
|
+
* was lost (the claim query skips NULL rows, so without repair those
|
|
16
|
+
* triggers are permanently dormant).
|
|
17
|
+
* - One automation failing to start must never abort the rest of a claim
|
|
18
|
+
* batch: claims have already advanced next_run_at, so a lost claim is a
|
|
19
|
+
* silently missed run.
|
|
20
|
+
* - Each tick expires stale "running" run rows so crashes and hung cells do
|
|
21
|
+
* not leave phantom in-progress runs in history.
|
|
10
22
|
*/
|
|
11
23
|
|
|
24
|
+
const crypto = require('crypto');
|
|
25
|
+
|
|
12
26
|
const { SCHEDULER_INTERVAL_MS } = require('../config');
|
|
13
|
-
const {
|
|
27
|
+
const {
|
|
28
|
+
claimDueScheduledTriggers,
|
|
29
|
+
expireStaleAutomationRuns,
|
|
30
|
+
getSchedulerHeartbeat,
|
|
31
|
+
pruneAutomationRuns,
|
|
32
|
+
recordAutomationRun,
|
|
33
|
+
recordSchedulerHeartbeat,
|
|
34
|
+
repairScheduledTriggers,
|
|
35
|
+
} = require('./store');
|
|
14
36
|
const { executeAutomation } = require('./runner');
|
|
15
37
|
|
|
38
|
+
// Runs claimed more than this far past their slot are stamped as catch-up
|
|
39
|
+
// runs so workflows and the UI can distinguish "on time" from "made up".
|
|
40
|
+
const CATCH_UP_GRACE_MS = 5 * 60_000;
|
|
41
|
+
const MAINTENANCE_INTERVAL_MS = 60 * 60_000;
|
|
42
|
+
|
|
16
43
|
let schedulerTimer = null;
|
|
17
44
|
let isPolling = false;
|
|
45
|
+
let lastMaintenanceAt = 0;
|
|
18
46
|
|
|
19
47
|
const schedulerStatus = {
|
|
20
48
|
startedAt: null,
|
|
21
49
|
lastTickAt: null,
|
|
22
50
|
lastError: null,
|
|
23
51
|
lastClaimed: 0,
|
|
52
|
+
lastRepairedTriggers: 0,
|
|
53
|
+
lastExpiredRuns: 0,
|
|
24
54
|
};
|
|
25
55
|
|
|
26
56
|
function startAutomationScheduler() {
|
|
27
57
|
console.log(`[AmalgmMCP:Automations] Starting scheduler (interval: ${SCHEDULER_INTERVAL_MS}ms)`);
|
|
28
58
|
if (schedulerTimer) clearInterval(schedulerTimer);
|
|
29
59
|
schedulerStatus.startedAt = new Date().toISOString();
|
|
60
|
+
|
|
61
|
+
const previousHeartbeat = getSchedulerHeartbeat();
|
|
62
|
+
if (previousHeartbeat) {
|
|
63
|
+
const gapMs = Date.now() - new Date(previousHeartbeat).getTime();
|
|
64
|
+
if (Number.isFinite(gapMs) && gapMs > SCHEDULER_INTERVAL_MS * 4) {
|
|
65
|
+
console.warn(`[AmalgmMCP:Automations] Scheduler was down for ~${Math.round(gapMs / 60_000)}m (last heartbeat ${previousHeartbeat}); overdue triggers will catch up this tick.`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
try {
|
|
70
|
+
const repaired = repairScheduledTriggers({ source: 'automation-scheduler:boot' });
|
|
71
|
+
schedulerStatus.lastRepairedTriggers = repaired.length;
|
|
72
|
+
if (repaired.length > 0) {
|
|
73
|
+
console.log(`[AmalgmMCP:Automations] Re-armed ${repaired.length} dormant scheduled trigger(s) with missing next_run_at`);
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
console.error('[AmalgmMCP:Automations] Trigger repair pass failed:', error.message || error);
|
|
77
|
+
}
|
|
78
|
+
|
|
30
79
|
pollAutomationScheduler();
|
|
31
80
|
schedulerTimer = setInterval(pollAutomationScheduler, SCHEDULER_INTERVAL_MS);
|
|
32
81
|
}
|
|
33
82
|
|
|
83
|
+
function recordLostClaim(claim, error) {
|
|
84
|
+
try {
|
|
85
|
+
const timestamp = new Date().toISOString();
|
|
86
|
+
recordAutomationRun(claim.automation.id, {
|
|
87
|
+
runId: crypto.randomUUID(),
|
|
88
|
+
automationId: claim.automation.id,
|
|
89
|
+
automationName: claim.automation.name,
|
|
90
|
+
triggerId: claim.trigger?.id || null,
|
|
91
|
+
triggerName: claim.trigger?.name || null,
|
|
92
|
+
workflowId: claim.automation.workflowId || null,
|
|
93
|
+
startedAt: timestamp,
|
|
94
|
+
finishedAt: timestamp,
|
|
95
|
+
status: 'failed',
|
|
96
|
+
error: `Scheduled run could not start: ${error.message || error}`,
|
|
97
|
+
}, { source: 'automation_runs:workflow:start-failed' });
|
|
98
|
+
} catch (recordError) {
|
|
99
|
+
console.error('[AmalgmMCP:Automations] Failed to record lost claim:', recordError.message || recordError);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
34
103
|
function pollAutomationScheduler() {
|
|
35
104
|
if (isPolling) return;
|
|
36
105
|
isPolling = true;
|
|
37
106
|
try {
|
|
38
107
|
const now = new Date();
|
|
39
108
|
schedulerStatus.lastTickAt = now.toISOString();
|
|
109
|
+
recordSchedulerHeartbeat(schedulerStatus.lastTickAt);
|
|
40
110
|
const claims = claimDueScheduledTriggers(now, { source: 'automation-scheduler' });
|
|
41
111
|
schedulerStatus.lastClaimed = claims.length;
|
|
42
112
|
schedulerStatus.lastError = null;
|
|
43
113
|
|
|
44
114
|
for (const claim of claims) {
|
|
45
115
|
if (!claim?.automation || !claim?.trigger) continue;
|
|
46
|
-
|
|
47
|
-
|
|
116
|
+
const scheduledFor = claim.scheduledFor || now.toISOString();
|
|
117
|
+
const lateMs = Math.max(0, now.getTime() - new Date(scheduledFor).getTime());
|
|
118
|
+
const eventObj = {
|
|
119
|
+
id: scheduledFor,
|
|
48
120
|
source: 'amalgm.scheduler',
|
|
49
121
|
event: 'scheduled',
|
|
50
122
|
payload: {
|
|
51
|
-
scheduledFor
|
|
123
|
+
scheduledFor,
|
|
124
|
+
catchUp: lateMs > CATCH_UP_GRACE_MS,
|
|
125
|
+
lateMs,
|
|
52
126
|
triggerId: claim.trigger.id,
|
|
53
127
|
automationId: claim.automation.id,
|
|
54
128
|
},
|
|
55
129
|
headers: {},
|
|
56
130
|
receivedAt: now.toISOString(),
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
131
|
+
};
|
|
132
|
+
// The claim already advanced next_run_at, so this run will not be
|
|
133
|
+
// retried — isolate every failure (sync throws included) per claim and
|
|
134
|
+
// leave a failed run row instead of losing the run invisibly.
|
|
135
|
+
try {
|
|
136
|
+
executeAutomation(claim.automation, claim.trigger, eventObj).catch((error) => {
|
|
137
|
+
console.error(`[AmalgmMCP:Automations] Automation ${claim.automation.id} failed:`, error.message);
|
|
138
|
+
});
|
|
139
|
+
} catch (error) {
|
|
140
|
+
console.error(`[AmalgmMCP:Automations] Automation ${claim.automation.id} could not start:`, error.message || error);
|
|
141
|
+
recordLostClaim(claim, error);
|
|
142
|
+
}
|
|
60
143
|
}
|
|
144
|
+
|
|
145
|
+
runSchedulerMaintenance(now);
|
|
61
146
|
} catch (error) {
|
|
62
147
|
schedulerStatus.lastError = error.message || String(error);
|
|
63
148
|
console.error('[AmalgmMCP:Automations] Scheduler poll error:', schedulerStatus.lastError);
|
|
@@ -66,11 +151,35 @@ function pollAutomationScheduler() {
|
|
|
66
151
|
}
|
|
67
152
|
}
|
|
68
153
|
|
|
154
|
+
function runSchedulerMaintenance(now = new Date()) {
|
|
155
|
+
try {
|
|
156
|
+
const expired = expireStaleAutomationRuns();
|
|
157
|
+
schedulerStatus.lastExpiredRuns = expired.length;
|
|
158
|
+
if (expired.length > 0) {
|
|
159
|
+
console.warn(`[AmalgmMCP:Automations] Marked ${expired.length} stale running run(s) as interrupted`);
|
|
160
|
+
}
|
|
161
|
+
} catch (error) {
|
|
162
|
+
console.error('[AmalgmMCP:Automations] Stale-run sweep failed:', error.message || error);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (now.getTime() - lastMaintenanceAt < MAINTENANCE_INTERVAL_MS) return;
|
|
166
|
+
lastMaintenanceAt = now.getTime();
|
|
167
|
+
try {
|
|
168
|
+
const pruned = pruneAutomationRuns();
|
|
169
|
+
if (pruned > 0) {
|
|
170
|
+
console.log(`[AmalgmMCP:Automations] Pruned ${pruned} old automation run(s)`);
|
|
171
|
+
}
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.error('[AmalgmMCP:Automations] Run pruning failed:', error.message || error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
69
177
|
function getAutomationSchedulerStatus() {
|
|
70
178
|
return {
|
|
71
179
|
intervalMs: SCHEDULER_INTERVAL_MS,
|
|
72
180
|
running: Boolean(schedulerTimer),
|
|
73
181
|
polling: isPolling,
|
|
182
|
+
heartbeatAt: getSchedulerHeartbeat(),
|
|
74
183
|
...schedulerStatus,
|
|
75
184
|
};
|
|
76
185
|
}
|