@amenophis1er/foreman 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -3
- package/package.json +1 -1
- package/src/fleet-planner.test.ts +39 -1
- package/src/fleet-planner.ts +83 -3
- package/src/gitwork.test.ts +51 -2
- package/src/gitwork.ts +68 -0
- package/src/mcp.test.ts +38 -1
- package/src/mcp.ts +75 -3
- package/src/notify/commands.test.ts +2 -0
- package/src/notify/commands.ts +6 -0
- package/src/notify/telegram.ts +1 -0
- package/src/notify.test.ts +61 -0
- package/src/notify.ts +45 -1
- package/src/schedule-guards.test.ts +236 -0
- package/src/schedule-guards.ts +149 -0
- package/src/schedule.test.ts +240 -0
- package/src/schedule.ts +343 -0
- package/src/server.ts +540 -7
- package/src/store.test.ts +81 -1
- package/src/store.ts +117 -3
- package/src/types.ts +63 -0
- package/ui/dist/assets/index-Bcc4KMtO.js +68 -0
- package/ui/dist/index.html +1 -1
- package/ui/dist/assets/index-DOVnExqF.js +0 -68
package/src/notify.test.ts
CHANGED
|
@@ -123,6 +123,67 @@ test('a transport that throws costs a failure count, never an exception on the e
|
|
|
123
123
|
assert.equal(hub.delivered, 0);
|
|
124
124
|
});
|
|
125
125
|
|
|
126
|
+
test('a paused schedule says which one and why, and is gated by its cause', () => {
|
|
127
|
+
const c = ctx()();
|
|
128
|
+
const cap = shape(env('schedule_paused', { scheduleId: 's1', name: 'nightly deps', reason: 'monthly-cap' }), c)!;
|
|
129
|
+
assert.equal(cap.key, 'schedule:s1');
|
|
130
|
+
assert.equal(cap.gate, 'budget', 'the monthly ceiling is a spending guard');
|
|
131
|
+
assert.match(cap.text, /nightly deps/);
|
|
132
|
+
assert.match(cap.text, /this month's scheduled spend would go past the ceiling|this month's scheduled spend would go past the ceiling/);
|
|
133
|
+
|
|
134
|
+
const fail = shape(env('schedule_paused', { scheduleId: 's1', name: 'nightly deps', reason: 'failures' }), c)!;
|
|
135
|
+
assert.equal(fail.key, 'schedule:s1');
|
|
136
|
+
assert.equal(fail.gate, 'needsYou', 'only a human can resume it');
|
|
137
|
+
assert.match(fail.text, /two scheduled runs in a row failed/);
|
|
138
|
+
|
|
139
|
+
// Resuming is deliberately a dashboard act, so there is nothing to tap.
|
|
140
|
+
for (const s of [cap, fail]) {
|
|
141
|
+
assert.match(s.text, /Resume it from the Foreman dashboard/);
|
|
142
|
+
assert.equal(s.buttons, undefined);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test('an unknown pause reason still shapes, and does not throw', () => {
|
|
147
|
+
const c = ctx()();
|
|
148
|
+
assert.doesNotThrow(() => shape(env('schedule_paused', { scheduleId: 's1', name: 'nightly deps', reason: 'kaput' }), c));
|
|
149
|
+
const s = shape(env('schedule_paused', { scheduleId: 's1', name: 'nightly deps', reason: 'kaput' }), c)!;
|
|
150
|
+
assert.equal(s.key, 'schedule:s1');
|
|
151
|
+
assert.equal(s.gate, 'done');
|
|
152
|
+
assert.match(s.text, /nightly deps/);
|
|
153
|
+
assert.doesNotThrow(() => shape(env('schedule_skipped', { scheduleId: 's1', name: 'nightly deps', reason: 'moon phase' }), c));
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test('a skipped scheduled run is informational, and consecutive skips are not deduped', () => {
|
|
157
|
+
const c = ctx()();
|
|
158
|
+
const s = shape(env('schedule_skipped', { scheduleId: 's1', name: 'nightly deps', reason: 'project busy' }), c)!;
|
|
159
|
+
assert.equal(s.gate, 'done');
|
|
160
|
+
assert.equal(s.key, 'skip:s1:1000000');
|
|
161
|
+
assert.match(s.text, /nightly deps/);
|
|
162
|
+
assert.match(s.text, /already had a mission running/);
|
|
163
|
+
assert.match(s.text, /next scheduled run stands/);
|
|
164
|
+
// Like a stall, the timestamp is in the key so tonight's skip is not
|
|
165
|
+
// swallowed by last night's.
|
|
166
|
+
const later = shape(env('schedule_skipped', { scheduleId: 's1', name: 'nightly deps', reason: 'project busy' }, { ts: 1_000_500 }), c)!;
|
|
167
|
+
assert.notEqual(later.key, s.key);
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test('a scheduled run says nobody pressed start; an ordinary one reads exactly as before', () => {
|
|
171
|
+
const c = ctx()();
|
|
172
|
+
const plain = shape(env('run_finished', { status: 'done' }), c)!;
|
|
173
|
+
assert.equal(plain.text,
|
|
174
|
+
'<b>Mission done</b> · shop\n<i>Build the checkout page</i>' +
|
|
175
|
+
'\n<a href="http://box.local:4177/#/p/p1/r/r1">Open in Foreman</a>');
|
|
176
|
+
|
|
177
|
+
const sched = shape(env('run_finished', { status: 'done', scheduled: true, scheduleName: 'nightly deps' }), c)!;
|
|
178
|
+
assert.equal(sched.key, plain.key, 'same key, so it still edits and dedupes as before');
|
|
179
|
+
assert.equal(sched.gate, plain.gate);
|
|
180
|
+
assert.match(sched.text, /Mission done · scheduled · nightly deps/);
|
|
181
|
+
|
|
182
|
+
// Without a name it still marks itself as unattended.
|
|
183
|
+
const anon = shape(env('run_finished', { status: 'error', scheduled: true }), c)!;
|
|
184
|
+
assert.match(anon.text, /Mission failed · scheduled/);
|
|
185
|
+
});
|
|
186
|
+
|
|
126
187
|
// ---------------------------------------------------------------------------
|
|
127
188
|
// Telegram, against a stub of the Bot API
|
|
128
189
|
// ---------------------------------------------------------------------------
|
package/src/notify.ts
CHANGED
|
@@ -260,12 +260,56 @@ export function shape(env: Envelope, ctx: NotifyContext): Shaped | null {
|
|
|
260
260
|
case 'run_finished': {
|
|
261
261
|
const status = String(d.status ?? 'done');
|
|
262
262
|
const title = status === 'done' ? 'Mission done' : status === 'error' ? 'Mission failed' : 'Mission interrupted';
|
|
263
|
+
// A scheduled run says so in its title: nobody pressed start, so the
|
|
264
|
+
// first question a phone raises — "who did this?" — is already answered.
|
|
265
|
+
const sched = d.scheduled
|
|
266
|
+
? ` · scheduled${d.scheduleName ? ` · ${clip(d.scheduleName, 40)}` : ''}`
|
|
267
|
+
: '';
|
|
263
268
|
return { key: `finished:${env.runId}`, gate: 'done',
|
|
264
|
-
text: `${head(title)}${runLine}${foot}` };
|
|
269
|
+
text: `${head(title + sched)}${runLine}${foot}` };
|
|
265
270
|
}
|
|
266
271
|
case 'run_error':
|
|
267
272
|
return { key: `error:${env.runId}:${env.ts ?? ''}`, gate: 'done',
|
|
268
273
|
text: `${head('Mission error')}${runLine}\n${esc(clip(d.error))}${foot}` };
|
|
274
|
+
|
|
275
|
+
// --- scheduled missions -------------------------------------------
|
|
276
|
+
case 'schedule_paused': {
|
|
277
|
+
// A paused schedule is the one thing here that stays broken until
|
|
278
|
+
// someone acts, so it says why in plain words. Which toggle carries it
|
|
279
|
+
// follows the cause: the monthly ceiling is a spending guard, repeated
|
|
280
|
+
// failures are a thing only a human can clear. Anything else — a hand
|
|
281
|
+
// on the switch — is news, not a summons.
|
|
282
|
+
const reason = String(d.reason ?? '');
|
|
283
|
+
const gate: keyof NotifyPrefs = reason === 'monthly-cap' ? 'budget' : reason === 'failures' ? 'needsYou' : 'done';
|
|
284
|
+
const why = reason === 'failures'
|
|
285
|
+
? 'two scheduled runs in a row failed'
|
|
286
|
+
: reason === 'monthly-cap'
|
|
287
|
+
? "this month's scheduled spend would go past the ceiling"
|
|
288
|
+
: reason === 'human'
|
|
289
|
+
? 'you paused it'
|
|
290
|
+
: `paused${reason ? ` — ${esc(clip(reason, 60))}` : ''}`;
|
|
291
|
+
return {
|
|
292
|
+
key: `schedule:${d.scheduleId}`, gate,
|
|
293
|
+
// No buttons on purpose: resuming a schedule is a dashboard act, where
|
|
294
|
+
// the cadence, the caps and what it last did are all in view.
|
|
295
|
+
text: `${head('Schedule paused')}\n<b>${esc(clip(d.name, 60))}</b> — ${why}.` +
|
|
296
|
+
`\n<i>Resume it from the Foreman dashboard; there is no resume from here.</i>${foot}`,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
case 'schedule_skipped': {
|
|
300
|
+
// Nothing is wrong and nothing is owed: the cadence simply stepped over
|
|
301
|
+
// a busy project. Keyed by timestamp like a stall, so a run of skips
|
|
302
|
+
// reads as a run of skips rather than one deduped line.
|
|
303
|
+
const reason = String(d.reason ?? '');
|
|
304
|
+
const why = reason === 'project busy'
|
|
305
|
+
? 'the project already had a mission running, so this turn was not started'
|
|
306
|
+
: `not started${reason ? ` — ${esc(clip(reason, 60))}` : ''}`;
|
|
307
|
+
return {
|
|
308
|
+
key: `skip:${d.scheduleId}:${env.ts ?? ''}`, gate: 'done',
|
|
309
|
+
text: `${head('Scheduled run skipped')}\n<b>${esc(clip(d.name, 60))}</b> — ${why}.` +
|
|
310
|
+
`\n<i>The next scheduled run stands.</i>${foot}`,
|
|
311
|
+
};
|
|
312
|
+
}
|
|
269
313
|
default:
|
|
270
314
|
return null;
|
|
271
315
|
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import type { Schedule } from './types.js';
|
|
4
|
+
import {
|
|
5
|
+
DEFAULT_SCHEDULED_MONTHLY_CAP_USD,
|
|
6
|
+
monthlyScheduledSpend,
|
|
7
|
+
isFailedOutcome,
|
|
8
|
+
afterRunOutcome,
|
|
9
|
+
decideTicks,
|
|
10
|
+
type TickAction,
|
|
11
|
+
} from './schedule-guards.js';
|
|
12
|
+
|
|
13
|
+
/** Local time, because months and cadences are both local here. */
|
|
14
|
+
const at = (y: number, m: number, d: number, h = 0, min = 0) => new Date(y, m, d, h, min, 0, 0);
|
|
15
|
+
|
|
16
|
+
/** A run row as monthlyScheduledSpend sees one. */
|
|
17
|
+
function run(over: Partial<{ projectId: string; scheduleId: string; createdAt: number; costUsd: number }> = {}) {
|
|
18
|
+
return { projectId: 'p1', scheduleId: 's1', createdAt: at(2026, 4, 10, 3).getTime(), costUsd: 1, ...over };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** A schedule, due at `nextRunAt`, with only the fields the ticker reads. */
|
|
22
|
+
function schedule(over: Partial<Schedule> = {}): Schedule {
|
|
23
|
+
return {
|
|
24
|
+
id: 's1', projectId: 'p1', name: 'nightly', brief: 'tidy up',
|
|
25
|
+
cadence: { kind: 'daily', at: '03:00' },
|
|
26
|
+
budgetUsd: 5,
|
|
27
|
+
enabled: true,
|
|
28
|
+
createdAt: at(2026, 0, 1).getTime(),
|
|
29
|
+
nextRunAt: at(2026, 4, 10, 3).getTime(),
|
|
30
|
+
consecutiveFailures: 0,
|
|
31
|
+
pausedReason: null,
|
|
32
|
+
...over,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** decideTicks with the plumbing filled in; NOW is 2026-05-10 09:00 local. */
|
|
37
|
+
const NOW = at(2026, 4, 10, 9);
|
|
38
|
+
function decide(schedules: Schedule[], over: Partial<Parameters<typeof decideTicks>[0]> = {}): TickAction[] {
|
|
39
|
+
return decideTicks({
|
|
40
|
+
schedules,
|
|
41
|
+
now: NOW,
|
|
42
|
+
busyProjectIds: [],
|
|
43
|
+
monthSpend: {},
|
|
44
|
+
capFor: () => DEFAULT_SCHEDULED_MONTHLY_CAP_USD,
|
|
45
|
+
...over,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
test('monthlyScheduledSpend: only this project\'s scheduled runs, only this month', () => {
|
|
50
|
+
const now = at(2026, 4, 20, 12);
|
|
51
|
+
const runs = [
|
|
52
|
+
run({ costUsd: 3 }),
|
|
53
|
+
run({ costUsd: 4, scheduleId: 's2' }),
|
|
54
|
+
// A hand-started mission spends the human's attention, not the allowance.
|
|
55
|
+
run({ costUsd: 100, scheduleId: undefined }),
|
|
56
|
+
// Another project's schedule, and a run with no project at all.
|
|
57
|
+
run({ costUsd: 100, projectId: 'p2' }),
|
|
58
|
+
run({ costUsd: 100, projectId: undefined }),
|
|
59
|
+
// Last month, and next month.
|
|
60
|
+
run({ costUsd: 100, createdAt: at(2026, 3, 30, 23, 59).getTime() }),
|
|
61
|
+
run({ costUsd: 100, createdAt: at(2026, 5, 1, 0, 0).getTime() }),
|
|
62
|
+
];
|
|
63
|
+
assert.equal(monthlyScheduledSpend(runs, 'p1', now), 7);
|
|
64
|
+
assert.equal(monthlyScheduledSpend(runs, 'p2', now), 100);
|
|
65
|
+
assert.equal(monthlyScheduledSpend([], 'p1', now), 0);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test('monthlyScheduledSpend: the month boundary is local midnight, and December rolls to January', () => {
|
|
69
|
+
// The last minute of April and the first of May are different months even
|
|
70
|
+
// though they are a minute apart.
|
|
71
|
+
const april = [run({ costUsd: 2, createdAt: at(2026, 3, 30, 23, 59).getTime() })];
|
|
72
|
+
const may = [run({ costUsd: 2, createdAt: at(2026, 4, 1, 0, 0).getTime() })];
|
|
73
|
+
assert.equal(monthlyScheduledSpend(april, 'p1', at(2026, 3, 15)), 2);
|
|
74
|
+
assert.equal(monthlyScheduledSpend(april, 'p1', at(2026, 4, 15)), 0);
|
|
75
|
+
assert.equal(monthlyScheduledSpend(may, 'p1', at(2026, 4, 15)), 2);
|
|
76
|
+
assert.equal(monthlyScheduledSpend(may, 'p1', at(2026, 3, 15)), 0);
|
|
77
|
+
|
|
78
|
+
// December 2026 and January 2027 are both "month 0-ish" traps: the same
|
|
79
|
+
// month number, or the same year, is not the same month.
|
|
80
|
+
const dec = run({ costUsd: 6, createdAt: at(2026, 11, 31, 23, 30).getTime() });
|
|
81
|
+
const jan = run({ costUsd: 9, createdAt: at(2027, 0, 1, 0, 30).getTime() });
|
|
82
|
+
assert.equal(monthlyScheduledSpend([dec, jan], 'p1', at(2026, 11, 31, 23, 59)), 6);
|
|
83
|
+
assert.equal(monthlyScheduledSpend([dec, jan], 'p1', at(2027, 0, 2)), 9);
|
|
84
|
+
// Same month number, previous year: not this month.
|
|
85
|
+
assert.equal(monthlyScheduledSpend([run({ costUsd: 5, createdAt: at(2025, 4, 10).getTime() })], 'p1', at(2026, 4, 10)), 0);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('isFailedOutcome: errors and capped interruptions, not restarts', () => {
|
|
89
|
+
assert.equal(isFailedOutcome({ status: 'error' }), true);
|
|
90
|
+
assert.equal(isFailedOutcome({ status: 'interrupted', stopReason: 'budget' }), true);
|
|
91
|
+
assert.equal(isFailedOutcome({ status: 'interrupted', stopReason: 'turns' }), true);
|
|
92
|
+
assert.equal(isFailedOutcome({ status: 'interrupted', stopReason: 'time' }), true);
|
|
93
|
+
assert.equal(isFailedOutcome({ status: 'interrupted', stopReason: 'tokens' }), true);
|
|
94
|
+
// A sweep on startup leaves no stopReason: the server restarted, the
|
|
95
|
+
// schedule did nothing wrong.
|
|
96
|
+
assert.equal(isFailedOutcome({ status: 'interrupted' }), false);
|
|
97
|
+
assert.equal(isFailedOutcome({ status: 'interrupted', stopReason: null }), false);
|
|
98
|
+
assert.equal(isFailedOutcome({ status: 'done' }), false);
|
|
99
|
+
assert.equal(isFailedOutcome({ status: 'running' }), false);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('afterRunOutcome: two failures in a row pause, a success resets', () => {
|
|
103
|
+
const fresh = { consecutiveFailures: 0, pausedReason: null } as const;
|
|
104
|
+
const once = afterRunOutcome(fresh, { status: 'error' });
|
|
105
|
+
assert.deepEqual(once, { consecutiveFailures: 1, pausedReason: null, pausedNow: false });
|
|
106
|
+
|
|
107
|
+
const twice = afterRunOutcome({ consecutiveFailures: 1, pausedReason: null }, { status: 'interrupted', stopReason: 'budget' });
|
|
108
|
+
assert.deepEqual(twice, { consecutiveFailures: 2, pausedReason: 'failures', pausedNow: true });
|
|
109
|
+
|
|
110
|
+
// A done run clears the count before it ever reaches two.
|
|
111
|
+
assert.deepEqual(
|
|
112
|
+
afterRunOutcome({ consecutiveFailures: 1, pausedReason: null }, { status: 'done' }),
|
|
113
|
+
{ consecutiveFailures: 0, pausedReason: null, pausedNow: false },
|
|
114
|
+
);
|
|
115
|
+
// A restart is not a failure, so it does not count towards the pause.
|
|
116
|
+
assert.deepEqual(
|
|
117
|
+
afterRunOutcome({ consecutiveFailures: 1, pausedReason: null }, { status: 'interrupted' }),
|
|
118
|
+
{ consecutiveFailures: 0, pausedReason: null, pausedNow: false },
|
|
119
|
+
);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test('afterRunOutcome: does not un-pause, and does not re-report a pause it did not cause', () => {
|
|
123
|
+
// A schedule the human stopped, or one the monthly cap stopped, stays
|
|
124
|
+
// stopped until they resume it — a stray success is not consent.
|
|
125
|
+
for (const reason of ['human', 'monthly-cap', 'failures'] as const) {
|
|
126
|
+
assert.deepEqual(
|
|
127
|
+
afterRunOutcome({ consecutiveFailures: 0, pausedReason: reason }, { status: 'done' }),
|
|
128
|
+
{ consecutiveFailures: 0, pausedReason: reason, pausedNow: false },
|
|
129
|
+
);
|
|
130
|
+
assert.deepEqual(
|
|
131
|
+
afterRunOutcome({ consecutiveFailures: 3, pausedReason: reason }, { status: 'error' }),
|
|
132
|
+
{ consecutiveFailures: 4, pausedReason: reason, pausedNow: false },
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test('decideTicks: nothing to do for schedules that are not due, disabled or paused', () => {
|
|
138
|
+
assert.deepEqual(decide([schedule({ nextRunAt: at(2026, 4, 11, 3).getTime() })]), []);
|
|
139
|
+
assert.deepEqual(decide([schedule({ nextRunAt: null })]), []);
|
|
140
|
+
assert.deepEqual(decide([schedule({ enabled: false })]), []);
|
|
141
|
+
for (const reason of ['failures', 'monthly-cap', 'human'] as const) {
|
|
142
|
+
assert.deepEqual(decide([schedule({ pausedReason: reason })]), []);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test('decideTicks: a due schedule starts, and its next firing is after now', () => {
|
|
147
|
+
const actions = decide([schedule()]);
|
|
148
|
+
assert.equal(actions.length, 1);
|
|
149
|
+
const a = actions[0];
|
|
150
|
+
assert.equal(a.kind, 'start');
|
|
151
|
+
assert.equal(a.scheduleId, 's1');
|
|
152
|
+
assert.equal(a.kind === 'start' && a.at, NOW.getTime());
|
|
153
|
+
// Daily 03:00 from 09:00 today is 03:00 tomorrow.
|
|
154
|
+
assert.equal(a.nextRunAt, at(2026, 4, 11, 3).getTime());
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('decideTicks: a busy project misses the mission rather than queueing it', () => {
|
|
158
|
+
const busy = decide([schedule()], { busyProjectIds: new Set(['p1']) });
|
|
159
|
+
assert.deepEqual(busy, [{ kind: 'skip', scheduleId: 's1', reason: 'project busy', nextRunAt: at(2026, 4, 11, 3).getTime() }]);
|
|
160
|
+
// The array form of busyProjectIds says the same thing.
|
|
161
|
+
assert.deepEqual(decide([schedule()], { busyProjectIds: ['p1'] }), busy);
|
|
162
|
+
// Another project being busy is not this one's problem.
|
|
163
|
+
assert.equal(decide([schedule()], { busyProjectIds: ['p2'] })[0].kind, 'start');
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test('decideTicks: catch-up happens once, however long Foreman was down', () => {
|
|
167
|
+
// A schedule whose slot was three months ago fires once now and then goes
|
|
168
|
+
// back to its cadence — the next firing is computed from now, not from the
|
|
169
|
+
// 90 missed 03:00s.
|
|
170
|
+
const stale = schedule({ nextRunAt: at(2026, 1, 4, 3).getTime() });
|
|
171
|
+
const [start] = decide([stale]);
|
|
172
|
+
assert.equal(start.kind, 'start');
|
|
173
|
+
assert.equal(start.nextRunAt, at(2026, 4, 11, 3).getTime());
|
|
174
|
+
|
|
175
|
+
// Same for a skip: the missed slot does not linger in the past waiting to
|
|
176
|
+
// fire again on the next pass.
|
|
177
|
+
const [skip] = decide([stale], { busyProjectIds: ['p1'] });
|
|
178
|
+
assert.equal(skip.kind, 'skip');
|
|
179
|
+
assert.ok(skip.nextRunAt !== null && skip.nextRunAt > NOW.getTime());
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
test('decideTicks: the monthly ceiling pauses, and equal to the cap is still allowed', () => {
|
|
183
|
+
const s = schedule({ budgetUsd: 5 });
|
|
184
|
+
// 20 spent + 5 budget = 25 = the cap: allowed, the rule is strictly past it.
|
|
185
|
+
assert.equal(decide([s], { monthSpend: { p1: 20 } })[0].kind, 'start');
|
|
186
|
+
// A cent more and the run would cross it.
|
|
187
|
+
assert.deepEqual(decide([s], { monthSpend: { p1: 20.01 } }), [
|
|
188
|
+
{ kind: 'pause', scheduleId: 's1', reason: 'monthly-cap', spent: 20.01, cap: 25 },
|
|
189
|
+
]);
|
|
190
|
+
// The Map form says the same thing, and a pause does not advance the next
|
|
191
|
+
// firing: a human resumes it and gets a fresh next time.
|
|
192
|
+
const paused = decide([s], { monthSpend: new Map([['p1', 30]]) });
|
|
193
|
+
assert.deepEqual(paused, [{ kind: 'pause', scheduleId: 's1', reason: 'monthly-cap', spent: 30, cap: 25 }]);
|
|
194
|
+
assert.ok(!('nextRunAt' in paused[0]));
|
|
195
|
+
// The ceiling is per project, and capFor decides it.
|
|
196
|
+
assert.equal(decide([s], { monthSpend: { p1: 30 }, capFor: () => 100 })[0].kind, 'start');
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
test('decideTicks: one start per project, earliest due first, and the start charges the month', () => {
|
|
200
|
+
const early = schedule({ id: 's-early', nextRunAt: at(2026, 4, 10, 3).getTime(), createdAt: 200 });
|
|
201
|
+
const late = schedule({ id: 's-late', nextRunAt: at(2026, 4, 10, 7).getTime(), createdAt: 100 });
|
|
202
|
+
// Given in the wrong order on purpose: due time decides, not array order.
|
|
203
|
+
const actions = decide([late, early]);
|
|
204
|
+
assert.deepEqual(actions.map((a) => [a.kind, a.scheduleId]), [['start', 's-early'], ['skip', 's-late']]);
|
|
205
|
+
assert.equal(actions[1].kind === 'skip' && actions[1].reason, 'project busy');
|
|
206
|
+
|
|
207
|
+
// Due at the same moment: the older schedule wins, so a later-created
|
|
208
|
+
// neighbour cannot starve it night after night.
|
|
209
|
+
const tie = decide([
|
|
210
|
+
schedule({ id: 's-new', createdAt: 999 }),
|
|
211
|
+
schedule({ id: 's-old', createdAt: 1 }),
|
|
212
|
+
]);
|
|
213
|
+
assert.deepEqual(tie.map((a) => [a.kind, a.scheduleId]), [['start', 's-old'], ['skip', 's-new']]);
|
|
214
|
+
|
|
215
|
+
// Different projects: both start, each on its own allowance.
|
|
216
|
+
const two = decide([schedule({ id: 'a', projectId: 'p1' }), schedule({ id: 'b', projectId: 'p2' })]);
|
|
217
|
+
assert.deepEqual(two.map((a) => a.kind), ['start', 'start']);
|
|
218
|
+
|
|
219
|
+
// A start charges the project's month straight away, so no second schedule
|
|
220
|
+
// can slip under the ceiling by being decided in the same pass. Here 24
|
|
221
|
+
// spent leaves room for exactly one $1 run: the first takes it, and the
|
|
222
|
+
// second is turned away — as busy, because one mission per project is the
|
|
223
|
+
// stricter rule and it is checked first.
|
|
224
|
+
const cheap = [
|
|
225
|
+
schedule({ id: 'a', projectId: 'p1', budgetUsd: 1, createdAt: 1 }),
|
|
226
|
+
schedule({ id: 'b', projectId: 'p1', budgetUsd: 1, createdAt: 2 }),
|
|
227
|
+
];
|
|
228
|
+
const charged = decide(cheap, { monthSpend: { p1: 24 } });
|
|
229
|
+
assert.deepEqual(charged.map((a) => [a.kind, a.scheduleId]), [['start', 'a'], ['skip', 'b']]);
|
|
230
|
+
assert.equal(charged[1].kind === 'skip' && charged[1].reason, 'project busy');
|
|
231
|
+
// Nothing started, so nothing is charged: both are judged against the same
|
|
232
|
+
// spend, and the ceiling is what turns them away once the project is free.
|
|
233
|
+
const none = decide(cheap, { monthSpend: { p1: 24.5 }, busyProjectIds: ['p1'] });
|
|
234
|
+
assert.deepEqual(none.map((a) => a.kind), ['skip', 'skip']);
|
|
235
|
+
assert.deepEqual(decide([cheap[0]], { monthSpend: { p1: 24.5 } }).map((a) => a.kind), ['pause']);
|
|
236
|
+
});
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rules a scheduled mission has to get past before it spends anything.
|
|
3
|
+
*
|
|
4
|
+
* A schedule fires while nobody is watching, which is the whole point and also
|
|
5
|
+
* the whole danger: a mission that fails at 03:00 will fail again at 03:00
|
|
6
|
+
* tomorrow, and a cadence that costs $3 a firing costs $90 a month if nothing
|
|
7
|
+
* counts. So the ticker asks four questions every pass — is it due, is the
|
|
8
|
+
* project free, has it been failing, and can the project still afford it — and
|
|
9
|
+
* every one of those answers lives here rather than in the server.
|
|
10
|
+
*
|
|
11
|
+
* The point of the separation is testability: src/server.ts starts listening
|
|
12
|
+
* on import, so nothing that imports it can be a unit test. These functions
|
|
13
|
+
* take the world as arguments and return decisions; the server is left with
|
|
14
|
+
* the doing.
|
|
15
|
+
*/
|
|
16
|
+
import type { Schedule } from './types.js';
|
|
17
|
+
import { nextRunAt } from './schedule.js';
|
|
18
|
+
|
|
19
|
+
/** Default ceiling on what a project's SCHEDULED runs may cost in one calendar month. */
|
|
20
|
+
export const DEFAULT_SCHEDULED_MONTHLY_CAP_USD = 25;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* What this project's scheduled runs have cost so far this calendar month.
|
|
24
|
+
* Months are local, matching the local-time cadences: an operator who says
|
|
25
|
+
* "every night at 2am" means their nights, and their month rolls over on their
|
|
26
|
+
* midnight. Only runs with a scheduleId count; a hand-started mission spends
|
|
27
|
+
* the human's attention, not the schedule's allowance.
|
|
28
|
+
*/
|
|
29
|
+
export function monthlyScheduledSpend(
|
|
30
|
+
runs: Array<{ projectId?: string; scheduleId?: string; createdAt: number; costUsd: number }>,
|
|
31
|
+
projectId: string,
|
|
32
|
+
now: Date,
|
|
33
|
+
): number {
|
|
34
|
+
const year = now.getFullYear();
|
|
35
|
+
const month = now.getMonth();
|
|
36
|
+
let total = 0;
|
|
37
|
+
for (const run of runs) {
|
|
38
|
+
if (!run.scheduleId || run.projectId !== projectId) continue;
|
|
39
|
+
const at = new Date(run.createdAt);
|
|
40
|
+
if (at.getFullYear() !== year || at.getMonth() !== month) continue;
|
|
41
|
+
total += run.costUsd;
|
|
42
|
+
}
|
|
43
|
+
return total;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Did this scheduled run fail for the purposes of the failure pause? — status
|
|
48
|
+
* 'error', or 'interrupted' with a stopReason (budget/turns/time/tokens). An
|
|
49
|
+
* interruption with no stopReason is a server restart sweeping its orphans,
|
|
50
|
+
* not the schedule's fault, and pausing a schedule for that would mean a
|
|
51
|
+
* reboot silently switches off the operator's nightly missions.
|
|
52
|
+
*/
|
|
53
|
+
export function isFailedOutcome(run: { status: string; stopReason?: string | null }): boolean {
|
|
54
|
+
if (run.status === 'error') return true;
|
|
55
|
+
return run.status === 'interrupted' && !!run.stopReason;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The schedule's failure bookkeeping after one of its runs ended. Two
|
|
60
|
+
* consecutive failures pause it: once is a bad night, twice is a standing
|
|
61
|
+
* instruction that no longer works, and there is nobody awake to notice the
|
|
62
|
+
* third. A 'done' resets the count but does not un-pause — a schedule stopped
|
|
63
|
+
* for the monthly cap or by a human stays stopped until they say otherwise.
|
|
64
|
+
*/
|
|
65
|
+
export function afterRunOutcome(
|
|
66
|
+
schedule: Pick<Schedule, 'consecutiveFailures' | 'pausedReason'>,
|
|
67
|
+
run: { status: string; stopReason?: string | null },
|
|
68
|
+
): { consecutiveFailures: number; pausedReason: Schedule['pausedReason']; pausedNow: boolean } {
|
|
69
|
+
if (!isFailedOutcome(run)) {
|
|
70
|
+
return { consecutiveFailures: 0, pausedReason: schedule.pausedReason, pausedNow: false };
|
|
71
|
+
}
|
|
72
|
+
const consecutiveFailures = schedule.consecutiveFailures + 1;
|
|
73
|
+
const pausedNow = schedule.pausedReason === null && consecutiveFailures >= 2;
|
|
74
|
+
return {
|
|
75
|
+
consecutiveFailures,
|
|
76
|
+
pausedReason: pausedNow ? 'failures' : schedule.pausedReason,
|
|
77
|
+
pausedNow,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export type TickAction =
|
|
82
|
+
| { kind: 'start'; scheduleId: string; at: number; nextRunAt: number | null }
|
|
83
|
+
| { kind: 'skip'; scheduleId: string; reason: string; nextRunAt: number | null }
|
|
84
|
+
| { kind: 'pause'; scheduleId: string; reason: 'monthly-cap'; spent: number; cap: number };
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* What the ticker should do this pass. Pure: give it the world, it returns the
|
|
88
|
+
* decisions; the server performs them.
|
|
89
|
+
*/
|
|
90
|
+
export function decideTicks(input: {
|
|
91
|
+
schedules: Schedule[];
|
|
92
|
+
now: Date;
|
|
93
|
+
/** Projects that already have an active mission (reserveProject would fail). */
|
|
94
|
+
busyProjectIds: Set<string> | string[];
|
|
95
|
+
/** This month's scheduled spend per project id. */
|
|
96
|
+
monthSpend: Map<string, number> | Record<string, number>;
|
|
97
|
+
/** The project's monthly ceiling for scheduled runs. */
|
|
98
|
+
capFor: (projectId: string) => number;
|
|
99
|
+
}): TickAction[] {
|
|
100
|
+
const nowMs = input.now.getTime();
|
|
101
|
+
// Copies, because a start in this pass has to be visible to the schedules
|
|
102
|
+
// decided after it.
|
|
103
|
+
const busy = new Set(input.busyProjectIds);
|
|
104
|
+
const spend = input.monthSpend instanceof Map
|
|
105
|
+
? new Map(input.monthSpend)
|
|
106
|
+
: new Map(Object.entries(input.monthSpend));
|
|
107
|
+
|
|
108
|
+
// Earliest due first, tie-broken on createdAt, so that when two schedules in
|
|
109
|
+
// one project come due together the older one starts and the newer one is
|
|
110
|
+
// the one that waits. Without an order the array's order would decide, and a
|
|
111
|
+
// schedule could be starved every night by a later-created neighbour.
|
|
112
|
+
const due = input.schedules
|
|
113
|
+
.filter((s) => s.enabled && s.pausedReason === null && s.nextRunAt !== null && s.nextRunAt <= nowMs)
|
|
114
|
+
.sort((a, b) => (a.nextRunAt! - b.nextRunAt!) || (a.createdAt - b.createdAt) || a.id.localeCompare(b.id));
|
|
115
|
+
|
|
116
|
+
// The next firing is computed from `now`, never from the slot that was
|
|
117
|
+
// missed. That is why this function takes `now` at all: if the machine was
|
|
118
|
+
// asleep for a week, the schedule fires once on waking and then goes back to
|
|
119
|
+
// its cadence — a missed mission is missed, not queued, and catch-up happens
|
|
120
|
+
// at most once per schedule however long Foreman was down.
|
|
121
|
+
const advance = (s: Schedule): number | null => {
|
|
122
|
+
const next = nextRunAt(s.cadence, input.now);
|
|
123
|
+
return next ? next.getTime() : null;
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const actions: TickAction[] = [];
|
|
127
|
+
for (const s of due) {
|
|
128
|
+
if (busy.has(s.projectId)) {
|
|
129
|
+
actions.push({ kind: 'skip', scheduleId: s.id, reason: 'project busy', nextRunAt: advance(s) });
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const spent = spend.get(s.projectId) ?? 0;
|
|
133
|
+
const cap = input.capFor(s.projectId);
|
|
134
|
+
if (spent + s.budgetUsd > cap) {
|
|
135
|
+
// No advance: a paused schedule is resumed by a human, who then gets a
|
|
136
|
+
// fresh next time. Leaving nextRunAt in the past would make it fire the
|
|
137
|
+
// instant it came back.
|
|
138
|
+
actions.push({ kind: 'pause', scheduleId: s.id, reason: 'monthly-cap', spent, cap });
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
actions.push({ kind: 'start', scheduleId: s.id, at: nowMs, nextRunAt: advance(s) });
|
|
142
|
+
// One active mission per project: this project is now taken for the rest
|
|
143
|
+
// of the pass, and its month spend is charged straight away so a second
|
|
144
|
+
// schedule cannot slip under the ceiling by being decided in the same tick.
|
|
145
|
+
busy.add(s.projectId);
|
|
146
|
+
spend.set(s.projectId, spent + s.budgetUsd);
|
|
147
|
+
}
|
|
148
|
+
return actions;
|
|
149
|
+
}
|