@loicngr/kobo 1.7.4 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -9
- package/dist/mcp-server/kobo-tasks-handlers.js +41 -1
- package/dist/mcp-server/kobo-tasks-server.js +157 -8
- package/dist/server/db/migrations.js +87 -0
- package/dist/server/db/schema.js +27 -0
- package/dist/server/index.js +9 -1
- package/dist/server/routes/health.js +68 -3
- package/dist/server/routes/workspaces.js +183 -4
- package/dist/server/services/agent/engines/claude-code/engine.js +13 -6
- package/dist/server/services/agent/engines/claude-code/event-mapper.js +96 -7
- package/dist/server/services/agent/orchestrator.js +113 -71
- package/dist/server/services/auto-loop-service.js +16 -3
- package/dist/server/services/cron-service.js +279 -0
- package/dist/server/services/quota-backoff-service.js +127 -0
- package/dist/server/services/wakeup-service.js +1 -1
- package/dist/server/services/workspace-service.js +98 -0
- package/dist/server/utils/git-ops.js +8 -1
- package/package.json +2 -1
- package/src/client/dist/spa/assets/{ActivityFeed-ClJLeAXJ.js → ActivityFeed-BboSPm4b.js} +2 -2
- package/src/client/dist/spa/assets/{ActivityFeed-DVBfmJWJ.css → ActivityFeed-tE4LVYck.css} +1 -1
- package/src/client/dist/spa/assets/AutoLoopChip-w8D77bI5.js +1 -0
- package/src/client/dist/spa/assets/{CreatePage-BOkt0Psl.js → CreatePage-BDObLDJc.js} +1 -1
- package/src/client/dist/spa/assets/{DiffViewer-Dls1jFCN.js → DiffViewer-CblFgn8w.js} +3 -3
- package/src/client/dist/spa/assets/{DiffViewer-wFfQ9tcY.css → DiffViewer-DTdDcKZC.css} +1 -1
- package/src/client/dist/spa/assets/HealthPage-CBSw7e5q.js +1 -0
- package/src/client/dist/spa/assets/{MainLayout-DHNIerYJ.js → MainLayout-DhaYycak.js} +17 -17
- package/src/client/dist/spa/assets/MainLayout-drolsINz.css +1 -0
- package/src/client/dist/spa/assets/{SearchPage-BEnZ-CLq.js → SearchPage-cZTwP4Lf.js} +1 -1
- package/src/client/dist/spa/assets/{SettingsPage-DeCbWvPb.js → SettingsPage-C1efO0VM.js} +1 -1
- package/src/client/dist/spa/assets/WorkspacePage-3jcof896.js +4 -0
- package/src/client/dist/spa/assets/{WorkspacePage-eymEd4kx.css → WorkspacePage-CCtIrBiR.css} +1 -1
- package/src/client/dist/spa/assets/{cssMode-AlflsawW.js → cssMode-BFLYiiEw.js} +1 -1
- package/src/client/dist/spa/assets/{editor.api-DtvjQlUm.js → editor.api-2asmmhth.js} +1 -1
- package/src/client/dist/spa/assets/{editor.main-Ccy_gjVD.js → editor.main-ChCYZyez.js} +3 -3
- package/src/client/dist/spa/assets/{expand-template-AQsvbQ8_.js → expand-template-CXQFkQOJ.js} +1 -1
- package/src/client/dist/spa/assets/{freemarker2-DdQktlXK.js → freemarker2-BaBL9E9G.js} +1 -1
- package/src/client/dist/spa/assets/{handlebars-CE3ee2NH.js → handlebars-BxDour4L.js} +1 -1
- package/src/client/dist/spa/assets/{html-CCKX8Xv9.js → html-C6hnkfIL.js} +1 -1
- package/src/client/dist/spa/assets/{htmlMode-Dh8jDJum.js → htmlMode-9zT3-dmz.js} +1 -1
- package/src/client/dist/spa/assets/i18n-CLY0XI9-.js +1 -0
- package/src/client/dist/spa/assets/index-D6wj_wQ9.js +2 -0
- package/src/client/dist/spa/assets/{javascript-DhmZNdUp.js → javascript-C3YjvKbE.js} +1 -1
- package/src/client/dist/spa/assets/{jsonMode-B0xAtnNK.js → jsonMode-DcJDgMzf.js} +1 -1
- package/src/client/dist/spa/assets/{liquid-ByL0HpZ0.js → liquid-CsT8SjJM.js} +1 -1
- package/src/client/dist/spa/assets/{mdx-DX4pehAZ.js → mdx-CT3yVSyc.js} +1 -1
- package/src/client/dist/spa/assets/{models-ClWoqWeC.js → models-BsjWUKqM.js} +1 -1
- package/src/client/dist/spa/assets/{monaco.contribution-Fegh8Y1Y.js → monaco.contribution-DKGNz1oQ.js} +2 -2
- package/src/client/dist/spa/assets/{purify.es-BWZjBa9F.js → purify.es-CPieV82n.js} +1 -1
- package/src/client/dist/spa/assets/{python-COS2MM8n.js → python-Ca5miKgj.js} +1 -1
- package/src/client/dist/spa/assets/{razor-Cc3xCJU7.js → razor-7qzusGRc.js} +1 -1
- package/src/client/dist/spa/assets/{render-chat-markdown-DcGIpMoe.js → render-chat-markdown-Bqq2G-yI.js} +1 -1
- package/src/client/dist/spa/assets/{tsMode-eQIJjERk.js → tsMode-BdvO8jZ2.js} +1 -1
- package/src/client/dist/spa/assets/{typescript-DwIlacVU.js → typescript-BfVNzhgs.js} +1 -1
- package/src/client/dist/spa/assets/{xml-DP-09Aih.js → xml-DGNXGqXL.js} +1 -1
- package/src/client/dist/spa/assets/{yaml-BhrtimeA.js → yaml-CtAtOyt5.js} +1 -1
- package/src/client/dist/spa/index.html +1 -1
- package/src/mcp-server/kobo-tasks-handlers.ts +55 -1
- package/src/mcp-server/kobo-tasks-server.ts +165 -7
- package/src/client/dist/spa/assets/HealthPage-CMxH3SBS.js +0 -1
- package/src/client/dist/spa/assets/MainLayout-DKurmqtk.css +0 -1
- package/src/client/dist/spa/assets/WorkspacePage-DFAFT5OW.js +0 -4
- package/src/client/dist/spa/assets/i18n-BOsrrRj4.js +0 -1
- package/src/client/dist/spa/assets/index-_ZaIBxd6.js +0 -2
- /package/src/client/dist/spa/assets/{QPage-ChUKoaKe.js → QPage-DFi3K093.js} +0 -0
- /package/src/client/dist/spa/assets/{formatters-BD0_hovB.js → formatters-DCAQ6ANJ.js} +0 -0
|
@@ -4,7 +4,10 @@ import { getDb } from '../../db/index.js';
|
|
|
4
4
|
import { ensureKoboHome, getCompiledMcpServerPath, getDbPath, getKoboHome, getMcpServerSourcePath, getSettingsPath, getSkillsPath, } from '../../utils/paths.js';
|
|
5
5
|
import { unregisterProcess } from '../../utils/process-tracker.js';
|
|
6
6
|
import * as autoLoopService from '../auto-loop-service.js';
|
|
7
|
+
import * as cronService from '../cron-service.js';
|
|
8
|
+
import * as quotaBackoffService from '../quota-backoff-service.js';
|
|
7
9
|
import { getEffectiveSettings } from '../settings-service.js';
|
|
10
|
+
import { refreshNow } from '../usage/poller.js';
|
|
8
11
|
import * as wakeupService from '../wakeup-service.js';
|
|
9
12
|
import { emit, emitEphemeral } from '../websocket-service.js';
|
|
10
13
|
import { getWorkspace as getWs, markWorkspaceUnread, updateWorkspaceStatus, } from '../workspace-service.js';
|
|
@@ -137,8 +140,6 @@ let availableSkills = (() => {
|
|
|
137
140
|
const retryCounts = new Map();
|
|
138
141
|
/** Tracks workspaces where the current session failed due to a stale --resume session ID. */
|
|
139
142
|
const resumeFailedSet = new Set();
|
|
140
|
-
/** workspaceId -> backoff timer */
|
|
141
|
-
const backoffTimers = new Map();
|
|
142
143
|
// ── Watchdog ──────────────────────────────────────────────────────────────────
|
|
143
144
|
const WATCHDOG_INTERVAL_MS = 30_000;
|
|
144
145
|
let watchdogTimer = null;
|
|
@@ -447,6 +448,46 @@ function handleEvent(workspaceId, agentSessionId, ev) {
|
|
|
447
448
|
wakeupService.schedule(workspaceId, delay, prompt, reason, agentSessionId);
|
|
448
449
|
}
|
|
449
450
|
}
|
|
451
|
+
// Same legacy bridge for the SDK's native `CronCreate`. The native tool is
|
|
452
|
+
// session-only (the cron dies when the agent session exits and is not
|
|
453
|
+
// persisted to disk), which makes it useless for any real "schedule a
|
|
454
|
+
// recurring trigger" need. We intercept the tool:call and arm an equivalent
|
|
455
|
+
// kobo cron in parallel — persistent across restarts, owned by the backend.
|
|
456
|
+
if (ev.kind === 'tool:call' && ev.name === 'CronCreate') {
|
|
457
|
+
const input = ev.input;
|
|
458
|
+
const prompt = typeof input?.prompt === 'string' ? input.prompt : '';
|
|
459
|
+
// The SDK's exact field name has drifted across versions — try the most
|
|
460
|
+
// likely candidates. If none match we log the input shape so the user
|
|
461
|
+
// can extend this list.
|
|
462
|
+
const expression = (typeof input?.cron === 'string' && input.cron) ||
|
|
463
|
+
(typeof input?.schedule === 'string' && input.schedule) ||
|
|
464
|
+
(typeof input?.expression === 'string' && input.expression) ||
|
|
465
|
+
'';
|
|
466
|
+
if (prompt && expression) {
|
|
467
|
+
console.warn(`[orchestrator] Native CronCreate intercepted for workspace '${workspaceId}' — armed equivalent kobo cron. Prefer kobo__cron_create.`);
|
|
468
|
+
try {
|
|
469
|
+
cronService.arm(workspaceId, {
|
|
470
|
+
expression,
|
|
471
|
+
prompt,
|
|
472
|
+
label: 'from-native-CronCreate',
|
|
473
|
+
agentSessionId,
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
catch (err) {
|
|
477
|
+
console.error('[orchestrator] Failed to mirror native CronCreate as kobo cron:', err);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
else if (prompt || input) {
|
|
481
|
+
console.warn(`[orchestrator] Native CronCreate intercepted but unrecognised input shape (workspace '${workspaceId}'):`, Object.keys(input ?? {}));
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
// Native `CronDelete` and `CronList` are noisy but harmless to ignore.
|
|
485
|
+
// The native cron is session-only so deletion is moot once the session
|
|
486
|
+
// ends; the kobo equivalents (kobo__cron_delete / kobo__cron_list) are
|
|
487
|
+
// the persistent path. Log to track usage and avoid silent confusion.
|
|
488
|
+
if (ev.kind === 'tool:call' && (ev.name === 'CronDelete' || ev.name === 'CronList')) {
|
|
489
|
+
console.warn(`[orchestrator] Native ${ev.name} called on workspace '${workspaceId}' — has no effect on kobo crons. Use kobo__${ev.name === 'CronDelete' ? 'cron_delete' : 'cron_list'} instead.`);
|
|
490
|
+
}
|
|
450
491
|
if (ev.kind === 'skills:discovered') {
|
|
451
492
|
availableSkills = ev.skills;
|
|
452
493
|
try {
|
|
@@ -469,7 +510,7 @@ function handleEvent(workspaceId, agentSessionId, ev) {
|
|
|
469
510
|
}
|
|
470
511
|
}
|
|
471
512
|
if (ev.kind === 'error' && ev.category === 'quota') {
|
|
472
|
-
handleQuota(workspaceId, agentSessionId);
|
|
513
|
+
void handleQuota(workspaceId, agentSessionId);
|
|
473
514
|
}
|
|
474
515
|
if (ev.kind === 'error' && ev.category === 'resume_failed') {
|
|
475
516
|
resumeFailedSet.add(workspaceId);
|
|
@@ -569,14 +610,11 @@ function onSessionEnded(workspaceId, agentSessionId, exitCode, reason, resumeFai
|
|
|
569
610
|
if (wasStopping)
|
|
570
611
|
return;
|
|
571
612
|
// When the session hit quota, handleQuota() already transitioned the
|
|
572
|
-
// workspace to `quota` and armed the retry
|
|
573
|
-
//
|
|
613
|
+
// workspace to `quota` and armed the retry via quotaBackoffService.
|
|
614
|
+
// Preserve that pending backoff in the quota path; otherwise clear any
|
|
615
|
+
// stale entry (defensive — shouldn't normally exist on a non-quota end).
|
|
574
616
|
if (!preserveQuotaBackoff) {
|
|
575
|
-
|
|
576
|
-
if (pendingBackoff) {
|
|
577
|
-
clearTimeout(pendingBackoff);
|
|
578
|
-
backoffTimers.delete(workspaceId);
|
|
579
|
-
}
|
|
617
|
+
quotaBackoffService.cancel(workspaceId, 'completed');
|
|
580
618
|
}
|
|
581
619
|
if (preserveQuotaBackoff) {
|
|
582
620
|
try {
|
|
@@ -774,11 +812,8 @@ export function stopAgent(workspaceId) {
|
|
|
774
812
|
// session:ended handler checks identity before removing, so a new controller
|
|
775
813
|
// started in the meantime is preserved.
|
|
776
814
|
controllers.delete(workspaceId);
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
clearTimeout(timer);
|
|
780
|
-
backoffTimers.delete(workspaceId);
|
|
781
|
-
}
|
|
815
|
+
// Manual stop should also drop any pending quota auto-resume.
|
|
816
|
+
quotaBackoffService.cancel(workspaceId, 'user');
|
|
782
817
|
// Fire-and-forget: controller.stop is async but we don't block callers.
|
|
783
818
|
void ctrl.stop().catch((err) => {
|
|
784
819
|
console.error('[orchestrator] controller.stop failed:', err);
|
|
@@ -1098,12 +1133,14 @@ const QUOTA_FALLBACK_LADDER_MINUTES = [15, 30, 60, 180, 300];
|
|
|
1098
1133
|
/**
|
|
1099
1134
|
* Compute the delay before retrying a workspace hit by quota.
|
|
1100
1135
|
*
|
|
1101
|
-
* Prefers the `resetsAt` of the saturated bucket with the furthest-future
|
|
1102
|
-
* reset
|
|
1103
|
-
*
|
|
1104
|
-
*
|
|
1136
|
+
* Prefers (1) the `resetsAt` of the saturated bucket with the furthest-future
|
|
1137
|
+
* reset reported by the agent's `rate_limit` event, then (1.5) the official
|
|
1138
|
+
* Anthropic usage API (`five_hour` bucket) when it reports saturation, and
|
|
1139
|
+
* finally (2) a fixed ladder (15 → 30 → 60 → 180 → 300 min) whenever neither
|
|
1140
|
+
* source is usable.
|
|
1105
1141
|
*/
|
|
1106
|
-
export function computeQuotaBackoffMs(workspaceId, retryCount) {
|
|
1142
|
+
export async function computeQuotaBackoffMs(workspaceId, retryCount) {
|
|
1143
|
+
// 1. Prefer the rate_limit event from the agent stream — most recent + most precise.
|
|
1107
1144
|
const info = latestRateLimitInfo.get(workspaceId);
|
|
1108
1145
|
if (info?.buckets?.length) {
|
|
1109
1146
|
const candidates = info.buckets
|
|
@@ -1119,15 +1156,38 @@ export function computeQuotaBackoffMs(workspaceId, retryCount) {
|
|
|
1119
1156
|
}
|
|
1120
1157
|
}
|
|
1121
1158
|
}
|
|
1159
|
+
// 1.5. Try the official usage API (Claude subscription). Best-effort; never throws.
|
|
1160
|
+
try {
|
|
1161
|
+
const snap = await refreshNow('claude-code');
|
|
1162
|
+
if (snap) {
|
|
1163
|
+
const fiveHour = snap.buckets.find((b) => b.id === 'five_hour');
|
|
1164
|
+
if (fiveHour &&
|
|
1165
|
+
typeof fiveHour.usedPct === 'number' &&
|
|
1166
|
+
fiveHour.usedPct >= QUOTA_SATURATION_THRESHOLD_PCT &&
|
|
1167
|
+
typeof fiveHour.resetsAt === 'string') {
|
|
1168
|
+
const resetTs = Date.parse(fiveHour.resetsAt);
|
|
1169
|
+
const delta = resetTs - Date.now() + QUOTA_SAFETY_MARGIN_MS;
|
|
1170
|
+
if (delta > 0 && delta <= QUOTA_MAX_BACKOFF_MS) {
|
|
1171
|
+
return { delayMs: delta, resetsAt: fiveHour.resetsAt, source: 'usage_api' };
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
catch (err) {
|
|
1177
|
+
console.warn('[orchestrator] computeQuotaBackoffMs — usage API call failed:', err);
|
|
1178
|
+
}
|
|
1179
|
+
// 2. Hard-coded ladder.
|
|
1122
1180
|
const idx = Math.min(Math.max(0, retryCount), QUOTA_FALLBACK_LADDER_MINUTES.length - 1);
|
|
1123
1181
|
const backoffMinutes = QUOTA_FALLBACK_LADDER_MINUTES[idx];
|
|
1124
|
-
return { delayMs: backoffMinutes * 60 * 1000, source: '
|
|
1182
|
+
return { delayMs: backoffMinutes * 60 * 1000, source: 'fallback_ladder' };
|
|
1125
1183
|
}
|
|
1184
|
+
/** @internal test-only — re-export of `computeQuotaBackoffMs` to anchor a stable seam. */
|
|
1185
|
+
export const _computeQuotaBackoffMs = computeQuotaBackoffMs;
|
|
1126
1186
|
/** @internal Test-only. */
|
|
1127
1187
|
export function _test_setRateLimitInfo(workspaceId, info) {
|
|
1128
1188
|
latestRateLimitInfo.set(workspaceId, info);
|
|
1129
1189
|
}
|
|
1130
|
-
function handleQuota(workspaceId, _agentSessionId) {
|
|
1190
|
+
async function handleQuota(workspaceId, _agentSessionId) {
|
|
1131
1191
|
try {
|
|
1132
1192
|
updateWorkspaceStatus(workspaceId, 'quota');
|
|
1133
1193
|
}
|
|
@@ -1135,53 +1195,39 @@ function handleQuota(workspaceId, _agentSessionId) {
|
|
|
1135
1195
|
// May fail if transition is not valid
|
|
1136
1196
|
}
|
|
1137
1197
|
const retryCount = retryCounts.get(workspaceId) ?? 0;
|
|
1138
|
-
const { delayMs, resetsAt, source } = computeQuotaBackoffMs(workspaceId, retryCount);
|
|
1139
|
-
const backoffMs = delayMs;
|
|
1198
|
+
const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount);
|
|
1140
1199
|
retryCounts.set(workspaceId, retryCount + 1);
|
|
1141
|
-
//
|
|
1142
|
-
//
|
|
1143
|
-
|
|
1144
|
-
retryCount: retryCount + 1,
|
|
1145
|
-
backoffMinutes: Math.round(delayMs / 60_000),
|
|
1146
|
-
resetsAt,
|
|
1147
|
-
source,
|
|
1148
|
-
});
|
|
1149
|
-
const timer = setTimeout(() => {
|
|
1150
|
-
backoffTimers.delete(workspaceId);
|
|
1151
|
-
if (!controllers.has(workspaceId)) {
|
|
1152
|
-
const freshWs = getWs(workspaceId);
|
|
1153
|
-
if (!freshWs || freshWs.archivedAt !== null || freshWs.status !== 'quota') {
|
|
1154
|
-
return;
|
|
1155
|
-
}
|
|
1156
|
-
try {
|
|
1157
|
-
if (freshWs.autoLoop) {
|
|
1158
|
-
autoLoopService.onQuotaBackoffExpired(workspaceId);
|
|
1159
|
-
}
|
|
1160
|
-
else {
|
|
1161
|
-
const freshWorkingDir = freshWs.worktreePath;
|
|
1162
|
-
startAgent(workspaceId, freshWorkingDir, 'Continue the previous task where you left off.', undefined, true);
|
|
1163
|
-
}
|
|
1164
|
-
}
|
|
1165
|
-
catch (err) {
|
|
1166
|
-
console.error(`[orchestrator] Quota retry for workspace '${workspaceId}' failed:`, err);
|
|
1167
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
1168
|
-
try {
|
|
1169
|
-
updateWorkspaceStatus(workspaceId, 'error');
|
|
1170
|
-
}
|
|
1171
|
-
catch {
|
|
1172
|
-
// transition may not be valid
|
|
1173
|
-
}
|
|
1174
|
-
routeEvent(workspaceId, '', {
|
|
1175
|
-
kind: 'error',
|
|
1176
|
-
category: 'other',
|
|
1177
|
-
message: `Quota retry failed: ${msg}`,
|
|
1178
|
-
});
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
}, backoffMs);
|
|
1182
|
-
timer.unref?.();
|
|
1183
|
-
backoffTimers.set(workspaceId, timer);
|
|
1200
|
+
// The quotaBackoffService owns the timer + the persistent row + the
|
|
1201
|
+
// 'agent:quota-backoff' WS emit. Hand off everything to it.
|
|
1202
|
+
quotaBackoffService.arm(workspaceId, delayMs, { resetsAt: resetsAt ?? null, source });
|
|
1184
1203
|
}
|
|
1204
|
+
/** @internal test-only — re-export of `handleQuota` for direct testing. */
|
|
1205
|
+
export const _handleQuota = handleQuota;
|
|
1206
|
+
/**
|
|
1207
|
+
* Rebuild the in-memory `retryCounts` map from the persisted `pending_quota_backoffs`
|
|
1208
|
+
* rows. Called from `index.ts` at boot, before `quotaBackoffService.restoreOnBoot`.
|
|
1209
|
+
* Without this, an arm() after restart would compute the next backoff from
|
|
1210
|
+
* `retryCount=0`, undoing the ladder progression.
|
|
1211
|
+
*/
|
|
1212
|
+
export function restoreRetryCountsFromDb() {
|
|
1213
|
+
for (const pending of quotaBackoffService.listPending()) {
|
|
1214
|
+
retryCounts.set(pending.workspaceId, pending.retryCount);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
// One-time wire: when the persisted backoff timer fires (or a row is
|
|
1218
|
+
// restored at boot), hand the workspace off to auto-loop. The auto-loop
|
|
1219
|
+
// service decides whether to spawn the next iteration or fall back to a
|
|
1220
|
+
// manual resume.
|
|
1221
|
+
//
|
|
1222
|
+
// IMPORTANT — behavioural contract: only auto-loop workspaces auto-resume
|
|
1223
|
+
// after a quota backoff. `onQuotaBackoffExpired` no-ops if `auto_loop !== 1`
|
|
1224
|
+
// (see auto-loop-service). Workspaces hit by quota WITHOUT auto-loop stay
|
|
1225
|
+
// in `quota` status and require manual user action (resume / new message)
|
|
1226
|
+
// to leave that state. This is intentional: without an auto-loop intent,
|
|
1227
|
+
// firing a fresh agent run in the user's absence would surprise them.
|
|
1228
|
+
quotaBackoffService.setOnFireCallback((workspaceId) => {
|
|
1229
|
+
autoLoopService.onQuotaBackoffExpired(workspaceId);
|
|
1230
|
+
});
|
|
1185
1231
|
// ── Testing utilities ─────────────────────────────────────────────────────────
|
|
1186
1232
|
/** @internal test-only */
|
|
1187
1233
|
export function _getControllers() {
|
|
@@ -1192,10 +1238,6 @@ export function _getRetryCounts() {
|
|
|
1192
1238
|
return retryCounts;
|
|
1193
1239
|
}
|
|
1194
1240
|
/** @internal test-only */
|
|
1195
|
-
export function _getBackoffTimers() {
|
|
1196
|
-
return backoffTimers;
|
|
1197
|
-
}
|
|
1198
|
-
/** @internal test-only */
|
|
1199
1241
|
export function _getSessionIds() {
|
|
1200
1242
|
return sessionIds;
|
|
1201
1243
|
}
|
|
@@ -54,12 +54,17 @@ export function enable(workspaceId) {
|
|
|
54
54
|
if (row.auto_loop_ready !== 1) {
|
|
55
55
|
throw new Error(`Workspace '${workspaceId}' is not ready for auto-loop (run grooming first)`);
|
|
56
56
|
}
|
|
57
|
+
// Refuse to enable when there is nothing to spawn — without this, auto_loop
|
|
58
|
+
// would flip to 1 silently with no iteration running, locking the chat input
|
|
59
|
+
// (auto-loop banner) without doing any work. The user must add a task or
|
|
60
|
+
// unmark a done task before re-enabling.
|
|
61
|
+
const pending = countPendingTasks(workspaceId);
|
|
62
|
+
if (pending === 0) {
|
|
63
|
+
throw new Error(`Workspace '${workspaceId}' has no pending tasks; add or unmark a task before enabling auto-loop`);
|
|
64
|
+
}
|
|
57
65
|
const db = getDb();
|
|
58
66
|
db.prepare('UPDATE workspaces SET auto_loop = 1, no_progress_streak = 0 WHERE id = ?').run(workspaceId);
|
|
59
67
|
emitEphemeral(workspaceId, 'autoloop:enabled', {});
|
|
60
|
-
const pending = countPendingTasks(workspaceId);
|
|
61
|
-
if (pending === 0)
|
|
62
|
-
return;
|
|
63
68
|
if (orchestrator.hasController(workspaceId))
|
|
64
69
|
return;
|
|
65
70
|
// spawnNextIteration throws on initial spawn failure (see flag).
|
|
@@ -309,6 +314,14 @@ function spawnNextIteration(workspaceId, opts = {}) {
|
|
|
309
314
|
* Called by orchestrator.handleQuota's backoff timer when auto-loop is enabled.
|
|
310
315
|
* Spawns the next auto-loop iteration if the workspace is still in quota status
|
|
311
316
|
* with auto_loop active; no-ops otherwise (race-safe).
|
|
317
|
+
*
|
|
318
|
+
* No-op cases — these all leave the workspace in `quota` status awaiting
|
|
319
|
+
* manual user action:
|
|
320
|
+
* - workspace was deleted between arm and fire
|
|
321
|
+
* - `auto_loop !== 1` (workspace was never an auto-loop target, OR the user
|
|
322
|
+
* toggled the loop off during the backoff window)
|
|
323
|
+
* - `status !== 'quota'` (user already manually resumed, or another path
|
|
324
|
+
* transitioned the workspace)
|
|
312
325
|
*/
|
|
313
326
|
export function onQuotaBackoffExpired(workspaceId) {
|
|
314
327
|
const row = getRow(workspaceId);
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import { CronExpressionParser } from 'cron-parser';
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
3
|
+
import { getDb } from '../db/index.js';
|
|
4
|
+
import { slugifyProjectName } from '../utils/project-slug.js';
|
|
5
|
+
import { resolveWorkspaceWorktreePath } from '../utils/worktree-paths.js';
|
|
6
|
+
import * as orchestrator from './agent/orchestrator.js';
|
|
7
|
+
import * as settingsService from './settings-service.js';
|
|
8
|
+
import { emitEphemeral } from './websocket-service.js';
|
|
9
|
+
export const MIN_DELAY_BETWEEN_FIRES_SECONDS = 60;
|
|
10
|
+
// Node `setTimeout` stores the delay as a 32-bit signed int. Anything above
|
|
11
|
+
// 2^31-1 ms (~24.8 days) triggers `TimeoutOverflowWarning` and silently
|
|
12
|
+
// truncates to 1ms — which causes the timer to fire instantly and loop
|
|
13
|
+
// when the real target is years away (e.g. `@yearly` or `0 0 1 1 *`).
|
|
14
|
+
// We cap each setTimeout at this max and re-arm in the callback if the
|
|
15
|
+
// real fire time hasn't been reached yet.
|
|
16
|
+
const MAX_SETTIMEOUT_MS = 2_000_000_000; // ~23 days, with margin under 2^31-1
|
|
17
|
+
const timers = new Map();
|
|
18
|
+
/**
|
|
19
|
+
* Arm a setTimeout that fires `fireOrSkip(id)` when `fireAt` is reached.
|
|
20
|
+
* Replaces any existing timer for this id. Handles long horizons by
|
|
21
|
+
* chaining capped timeouts, since Node's setTimeout overflows past ~24.8
|
|
22
|
+
* days.
|
|
23
|
+
*/
|
|
24
|
+
function scheduleAt(id, fireAt) {
|
|
25
|
+
const previous = timers.get(id);
|
|
26
|
+
if (previous)
|
|
27
|
+
clearTimeout(previous);
|
|
28
|
+
const deltaMs = Math.max(0, fireAt.getTime() - Date.now());
|
|
29
|
+
const cappedMs = Math.min(deltaMs, MAX_SETTIMEOUT_MS);
|
|
30
|
+
const timer = setTimeout(() => {
|
|
31
|
+
timers.delete(id);
|
|
32
|
+
if (Date.now() >= fireAt.getTime()) {
|
|
33
|
+
fireOrSkip(id);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
// Long-horizon cron: hop forward another chunk and re-check.
|
|
37
|
+
scheduleAt(id, fireAt);
|
|
38
|
+
}
|
|
39
|
+
}, cappedMs);
|
|
40
|
+
timer.unref?.();
|
|
41
|
+
timers.set(id, timer);
|
|
42
|
+
}
|
|
43
|
+
function rowToCron(row) {
|
|
44
|
+
return {
|
|
45
|
+
id: row.id,
|
|
46
|
+
workspaceId: row.workspace_id,
|
|
47
|
+
expression: row.expression,
|
|
48
|
+
prompt: row.prompt,
|
|
49
|
+
label: row.label,
|
|
50
|
+
agentSessionId: row.agent_session_id,
|
|
51
|
+
nextFireAt: row.next_fire_at,
|
|
52
|
+
lastFiredAt: row.last_fired_at,
|
|
53
|
+
oneShot: row.one_shot === 1,
|
|
54
|
+
createdAt: row.created_at,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Compute the next fire time for an expression strictly after `from`.
|
|
59
|
+
* Uses cron-parser 5.x API. Throws with a descriptive error if the
|
|
60
|
+
* expression is invalid. Helpers `@hourly` / `@daily` / `@weekly` /
|
|
61
|
+
* `@monthly` / `@yearly` are accepted natively.
|
|
62
|
+
*/
|
|
63
|
+
function nextAfter(expression, from) {
|
|
64
|
+
try {
|
|
65
|
+
const it = CronExpressionParser.parse(expression, { currentDate: from });
|
|
66
|
+
return it.next().toDate();
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
70
|
+
throw new Error(`Invalid cron expression: ${expression} — ${msg}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Validate the expression, persist the row, arm a setTimeout for the next
|
|
75
|
+
* fire, emit `cron:created`. Throws on invalid expression OR when the next
|
|
76
|
+
* fire is < MIN_DELAY_BETWEEN_FIRES_SECONDS seconds in the future.
|
|
77
|
+
*/
|
|
78
|
+
export function arm(workspaceId, args) {
|
|
79
|
+
const now = new Date();
|
|
80
|
+
const next = nextAfter(args.expression, now);
|
|
81
|
+
const deltaMs = next.getTime() - now.getTime();
|
|
82
|
+
if (deltaMs < MIN_DELAY_BETWEEN_FIRES_SECONDS * 1000) {
|
|
83
|
+
throw new Error(`Cron expression resolves too close to now (minimum ${MIN_DELAY_BETWEEN_FIRES_SECONDS}s); use a longer interval`);
|
|
84
|
+
}
|
|
85
|
+
const id = nanoid();
|
|
86
|
+
const db = getDb();
|
|
87
|
+
db.prepare(`INSERT INTO pending_crons (id, workspace_id, expression, prompt, label, agent_session_id, next_fire_at, last_fired_at, one_shot, created_at)
|
|
88
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`).run(id, workspaceId, args.expression, args.prompt, args.label ?? null, args.agentSessionId ?? null, next.toISOString(), args.oneShot ? 1 : 0, now.toISOString());
|
|
89
|
+
scheduleAt(id, next);
|
|
90
|
+
const cron = rowToCron(db.prepare('SELECT * FROM pending_crons WHERE id = ?').get(id));
|
|
91
|
+
emitEphemeral(workspaceId, 'cron:created', { cron });
|
|
92
|
+
return cron;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Remove a single cron by id. Idempotent — returns false if no row matched.
|
|
96
|
+
* Emits `cron:cancelled` only when a row was actually deleted.
|
|
97
|
+
*/
|
|
98
|
+
export function cancel(id, reason) {
|
|
99
|
+
const db = getDb();
|
|
100
|
+
const row = db.prepare('SELECT workspace_id FROM pending_crons WHERE id = ?').get(id);
|
|
101
|
+
if (!row)
|
|
102
|
+
return false;
|
|
103
|
+
db.prepare('DELETE FROM pending_crons WHERE id = ?').run(id);
|
|
104
|
+
const previous = timers.get(id);
|
|
105
|
+
if (previous) {
|
|
106
|
+
clearTimeout(previous);
|
|
107
|
+
timers.delete(id);
|
|
108
|
+
}
|
|
109
|
+
emitEphemeral(row.workspace_id, 'cron:cancelled', { id, reason });
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Remove every cron for a workspace. Returns the number of rows deleted.
|
|
114
|
+
* Used by archive + delete cascades.
|
|
115
|
+
*/
|
|
116
|
+
export function cancelAllForWorkspace(workspaceId, reason) {
|
|
117
|
+
const db = getDb();
|
|
118
|
+
const rows = db.prepare('SELECT id FROM pending_crons WHERE workspace_id = ?').all(workspaceId);
|
|
119
|
+
let deleted = 0;
|
|
120
|
+
for (const r of rows) {
|
|
121
|
+
if (cancel(r.id, reason))
|
|
122
|
+
deleted++;
|
|
123
|
+
}
|
|
124
|
+
return deleted;
|
|
125
|
+
}
|
|
126
|
+
export function getCron(id) {
|
|
127
|
+
const db = getDb();
|
|
128
|
+
const row = db.prepare('SELECT * FROM pending_crons WHERE id = ?').get(id);
|
|
129
|
+
return row ? rowToCron(row) : null;
|
|
130
|
+
}
|
|
131
|
+
export function listForWorkspace(workspaceId) {
|
|
132
|
+
const db = getDb();
|
|
133
|
+
const rows = db
|
|
134
|
+
.prepare('SELECT * FROM pending_crons WHERE workspace_id = ? ORDER BY next_fire_at ASC')
|
|
135
|
+
.all(workspaceId);
|
|
136
|
+
return rows.map(rowToCron);
|
|
137
|
+
}
|
|
138
|
+
export function listAll() {
|
|
139
|
+
const db = getDb();
|
|
140
|
+
const rows = db.prepare('SELECT * FROM pending_crons ORDER BY next_fire_at ASC').all();
|
|
141
|
+
return rows.map(rowToCron);
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Internal — invoked by setTimeout when a cron's `next_fire_at` elapses.
|
|
145
|
+
* Either fires (calls orchestrator.startAgent with resume=true) or skips
|
|
146
|
+
* (when a controller is already active for the workspace), then recomputes
|
|
147
|
+
* the next occurrence and re-arms a fresh setTimeout. Best-effort: any
|
|
148
|
+
* unexpected error is logged and the cron is preserved when possible.
|
|
149
|
+
*/
|
|
150
|
+
function fireOrSkip(id) {
|
|
151
|
+
try {
|
|
152
|
+
timers.delete(id);
|
|
153
|
+
const db = getDb();
|
|
154
|
+
const row = db.prepare('SELECT * FROM pending_crons WHERE id = ?').get(id);
|
|
155
|
+
if (!row)
|
|
156
|
+
return; // cancelled in flight
|
|
157
|
+
const wsRow = db
|
|
158
|
+
.prepare(`SELECT project_path, working_branch, worktree_path, model, agent_permission_mode, reasoning_effort, archived_at
|
|
159
|
+
FROM workspaces WHERE id = ?`)
|
|
160
|
+
.get(row.workspace_id);
|
|
161
|
+
if (!wsRow || wsRow.archived_at !== null) {
|
|
162
|
+
cancel(id, wsRow ? 'archive' : 'deleted');
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
let status = 'skipped-active';
|
|
166
|
+
if (!orchestrator.hasController(row.workspace_id)) {
|
|
167
|
+
status = 'fired';
|
|
168
|
+
try {
|
|
169
|
+
const globalSettings = settingsService.getGlobalSettings();
|
|
170
|
+
const projectSettings = settingsService.getProjectSettings(wsRow.project_path);
|
|
171
|
+
const projectSlug = globalSettings.worktreesPrefixByProject
|
|
172
|
+
? slugifyProjectName(projectSettings?.displayName ?? '', wsRow.project_path)
|
|
173
|
+
: undefined;
|
|
174
|
+
const worktreePath = wsRow.worktree_path ??
|
|
175
|
+
resolveWorkspaceWorktreePath(wsRow.project_path, wsRow.working_branch, globalSettings.worktreesPath, projectSlug);
|
|
176
|
+
const stored = wsRow.agent_permission_mode;
|
|
177
|
+
const agentPermissionMode = stored === 'plan' || stored === 'strict' || stored === 'interactive' ? stored : 'bypass';
|
|
178
|
+
// agent_session_id encodes the cron's mode: non-NULL means "resume
|
|
179
|
+
// that session" (pinned at create time); NULL means "fresh session
|
|
180
|
+
// every fire" (clean context, no conversation continuity).
|
|
181
|
+
const resumeMode = row.agent_session_id !== null;
|
|
182
|
+
orchestrator.startAgent(row.workspace_id, worktreePath, row.prompt, wsRow.model, resumeMode, agentPermissionMode, row.agent_session_id ?? undefined, wsRow.reasoning_effort);
|
|
183
|
+
}
|
|
184
|
+
catch (err) {
|
|
185
|
+
console.error(`[cron-service] startAgent at fire time failed for cron '${id}':`, err);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const now = new Date();
|
|
189
|
+
let nextFire;
|
|
190
|
+
try {
|
|
191
|
+
nextFire = nextAfter(row.expression, now);
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
// Defensive — the expression validated at create time, so reaching here
|
|
195
|
+
// means cron-parser changed its acceptance rules between the original
|
|
196
|
+
// arm and this fire. Cancel as 'completed' (the cron self-terminates)
|
|
197
|
+
// rather than 'user' which would imply the user requested it.
|
|
198
|
+
console.error(`[cron-service] failed to recompute next fire for cron '${id}':`, err);
|
|
199
|
+
cancel(id, 'completed');
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
// One-shot crons cancel themselves after a real fire (not on skip-active —
|
|
203
|
+
// the user expects the cron to actually run once, so a skipped tick must
|
|
204
|
+
// be retried at the next occurrence). Recurring crons re-arm normally.
|
|
205
|
+
if (status === 'fired' && row.one_shot === 1) {
|
|
206
|
+
db.prepare(`UPDATE pending_crons SET last_fired_at = ? WHERE id = ?`).run(now.toISOString(), id);
|
|
207
|
+
emitEphemeral(row.workspace_id, 'cron:fired', {
|
|
208
|
+
id,
|
|
209
|
+
status,
|
|
210
|
+
nextFireAt: null,
|
|
211
|
+
lastFiredAt: now.toISOString(),
|
|
212
|
+
oneShotConsumed: true,
|
|
213
|
+
});
|
|
214
|
+
cancel(id, 'completed');
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
db.prepare(`UPDATE pending_crons SET next_fire_at = ?, last_fired_at = ? WHERE id = ?`).run(nextFire.toISOString(), now.toISOString(), id);
|
|
218
|
+
scheduleAt(id, nextFire);
|
|
219
|
+
emitEphemeral(row.workspace_id, 'cron:fired', {
|
|
220
|
+
id,
|
|
221
|
+
status,
|
|
222
|
+
nextFireAt: nextFire.toISOString(),
|
|
223
|
+
lastFiredAt: now.toISOString(),
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
catch (err) {
|
|
227
|
+
console.error(`[cron-service] fireOrSkip uncaught error for cron '${id}':`, err);
|
|
228
|
+
timers.delete(id);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Re-arm timers for rows persisted across restart. Skip-missed semantics:
|
|
233
|
+
* if the stored next_fire_at is in the past, recompute next() based on the
|
|
234
|
+
* current time and update the row before arming (mirror of POSIX crontab —
|
|
235
|
+
* no catchup spam after server downtime).
|
|
236
|
+
*
|
|
237
|
+
* Rows pointing at deleted/archived workspaces are removed without firing.
|
|
238
|
+
*/
|
|
239
|
+
export function restoreOnBoot() {
|
|
240
|
+
try {
|
|
241
|
+
// Boot semantics: clear any existing in-memory timers before rearming.
|
|
242
|
+
for (const t of timers.values())
|
|
243
|
+
clearTimeout(t);
|
|
244
|
+
timers.clear();
|
|
245
|
+
const db = getDb();
|
|
246
|
+
const rows = db.prepare('SELECT * FROM pending_crons').all();
|
|
247
|
+
const now = new Date();
|
|
248
|
+
for (const row of rows) {
|
|
249
|
+
try {
|
|
250
|
+
const wsRow = db.prepare('SELECT archived_at FROM workspaces WHERE id = ?').get(row.workspace_id);
|
|
251
|
+
if (!wsRow || wsRow.archived_at !== null) {
|
|
252
|
+
db.prepare('DELETE FROM pending_crons WHERE id = ?').run(row.id);
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
const storedDate = new Date(row.next_fire_at);
|
|
256
|
+
let nextFireAt;
|
|
257
|
+
if (storedDate.getTime() <= now.getTime()) {
|
|
258
|
+
// Skip-missed semantics — no catchup spam after downtime.
|
|
259
|
+
const next = nextAfter(row.expression, now);
|
|
260
|
+
nextFireAt = next.toISOString();
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
// Future row: normalise the persisted ISO format.
|
|
264
|
+
nextFireAt = storedDate.toISOString();
|
|
265
|
+
}
|
|
266
|
+
db.prepare('UPDATE pending_crons SET next_fire_at = ? WHERE id = ?').run(nextFireAt, row.id);
|
|
267
|
+
scheduleAt(row.id, new Date(nextFireAt));
|
|
268
|
+
}
|
|
269
|
+
catch (err) {
|
|
270
|
+
console.error(`[cron-service] restoreOnBoot row failed for cron '${row.id}':`, err);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
catch (err) {
|
|
275
|
+
console.error('[cron-service] restoreOnBoot failed:', err);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/** @internal test-only */
|
|
279
|
+
export const _timers = timers;
|