@hasna/loops 0.3.60 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +265 -0
- package/LICENSE +197 -13
- package/README.md +95 -85
- package/dist/api/index.d.ts +11 -0
- package/dist/api/index.js +333 -0
- package/dist/cli/index.js +8013 -6659
- package/dist/daemon/control.d.ts +21 -1
- package/dist/daemon/daemon.d.ts +5 -0
- package/dist/daemon/index.js +2836 -1222
- package/dist/daemon/install.d.ts +1 -1
- package/dist/index.d.ts +23 -8
- package/dist/index.js +5715 -4541
- package/dist/lib/accounts.d.ts +6 -1
- package/dist/lib/agent-adapter.d.ts +74 -0
- package/dist/lib/backup.d.ts +25 -0
- package/dist/lib/errors.d.ts +21 -0
- package/dist/lib/executor.d.ts +10 -1
- package/dist/lib/goal/metadata.d.ts +16 -0
- package/dist/lib/goal/model-factory.d.ts +1 -0
- package/dist/lib/goal/prompts.d.ts +3 -1
- package/dist/lib/goal/types.d.ts +3 -0
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/ids.d.ts +8 -1
- package/dist/lib/machines.d.ts +6 -0
- package/dist/lib/mode.d.ts +48 -0
- package/dist/lib/mode.js +260 -0
- package/dist/lib/process-identity.d.ts +16 -0
- package/dist/lib/{schedule.d.ts → recurrence.d.ts} +1 -0
- package/dist/lib/redact.d.ts +21 -0
- package/dist/lib/route/cursors.d.ts +18 -0
- package/dist/lib/route/drain.d.ts +6 -0
- package/dist/lib/route/fields.d.ts +27 -0
- package/dist/lib/route/gates.d.ts +26 -0
- package/dist/lib/route/index.d.ts +18 -0
- package/dist/lib/route/options.d.ts +31 -0
- package/dist/lib/route/parse.d.ts +8 -0
- package/dist/lib/route/pr-review.d.ts +11 -0
- package/dist/lib/route/provider.d.ts +41 -0
- package/dist/lib/route/route-event.d.ts +23 -0
- package/dist/lib/route/route-tasks.d.ts +75 -0
- package/dist/lib/route/throttle.d.ts +47 -0
- package/dist/lib/route/todos-cli.d.ts +21 -0
- package/dist/lib/route/types.d.ts +88 -0
- package/dist/lib/run-artifacts.d.ts +17 -2
- package/dist/lib/run-envelope.d.ts +41 -0
- package/dist/lib/scheduler.d.ts +78 -2
- package/dist/lib/store.d.ts +63 -0
- package/dist/lib/store.js +1007 -287
- package/dist/lib/template-kit.d.ts +70 -0
- package/dist/lib/templates-custom.d.ts +34 -0
- package/dist/lib/templates.d.ts +13 -81
- package/dist/lib/workflow-runner.d.ts +2 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +3231 -1382
- package/dist/runner/index.d.ts +9 -0
- package/dist/runner/index.js +295 -0
- package/dist/sdk/index.d.ts +29 -3
- package/dist/sdk/index.js +2743 -1044
- package/dist/test-helpers.d.ts +37 -0
- package/dist/types.d.ts +3 -1
- package/docs/DEPLOYMENT_MODES.md +140 -0
- package/docs/USAGE.md +102 -46
- package/package.json +23 -5
package/dist/lib/store.js
CHANGED
|
@@ -1,14 +1,50 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync } from "fs";
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
|
|
5
5
|
import { tmpdir } from "os";
|
|
6
|
-
import { dirname, join as join3 } from "path";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join3 } from "path";
|
|
7
|
+
|
|
8
|
+
// src/lib/errors.ts
|
|
9
|
+
class CodedError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code, message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = new.target.name;
|
|
14
|
+
this.code = code;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
class LoopNotFoundError extends CodedError {
|
|
19
|
+
constructor(idOrName) {
|
|
20
|
+
super("LOOP_NOT_FOUND", `loop not found: ${idOrName}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class LoopArchivedError extends CodedError {
|
|
25
|
+
constructor(idOrName) {
|
|
26
|
+
super("LOOP_ARCHIVED", `loop is archived: ${idOrName}; unarchive it before modifying`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
class AmbiguousNameError extends CodedError {
|
|
31
|
+
constructor(name) {
|
|
32
|
+
super("AMBIGUOUS_NAME", `ambiguous loop name: ${name}; use a loop id`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class ValidationError extends CodedError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super("VALIDATION_ERROR", message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
7
41
|
|
|
8
42
|
// src/lib/ids.ts
|
|
9
43
|
import { randomBytes } from "crypto";
|
|
10
|
-
|
|
11
|
-
|
|
44
|
+
var TIME_HEX_LENGTH = 12;
|
|
45
|
+
function genId() {
|
|
46
|
+
const time = Date.now().toString(16).padStart(TIME_HEX_LENGTH, "0");
|
|
47
|
+
return `${time}${randomBytes(10).toString("hex")}`;
|
|
12
48
|
}
|
|
13
49
|
function nowIso() {
|
|
14
50
|
return new Date().toISOString();
|
|
@@ -42,7 +78,166 @@ function launchdPlistPath() {
|
|
|
42
78
|
return join(homedir(), "Library", "LaunchAgents", "com.hasna.loops.daemon.plist");
|
|
43
79
|
}
|
|
44
80
|
|
|
45
|
-
// src/lib/
|
|
81
|
+
// src/lib/process-identity.ts
|
|
82
|
+
import { readFileSync } from "fs";
|
|
83
|
+
import { spawnSync } from "child_process";
|
|
84
|
+
var START_TIME_TOLERANCE_MS = 5000;
|
|
85
|
+
var clockTicksCache;
|
|
86
|
+
function clockTicksPerSecond() {
|
|
87
|
+
if (clockTicksCache !== undefined)
|
|
88
|
+
return clockTicksCache;
|
|
89
|
+
try {
|
|
90
|
+
const run = spawnSync("getconf", ["CLK_TCK"], { encoding: "utf8" });
|
|
91
|
+
const value = Number(run.stdout.trim());
|
|
92
|
+
clockTicksCache = run.status === 0 && Number.isFinite(value) && value > 0 ? value : 100;
|
|
93
|
+
} catch {
|
|
94
|
+
clockTicksCache = 100;
|
|
95
|
+
}
|
|
96
|
+
return clockTicksCache;
|
|
97
|
+
}
|
|
98
|
+
var bootTimeCache;
|
|
99
|
+
var bootTimeResolved = false;
|
|
100
|
+
function bootTimeMs() {
|
|
101
|
+
if (bootTimeResolved)
|
|
102
|
+
return bootTimeCache;
|
|
103
|
+
bootTimeResolved = true;
|
|
104
|
+
try {
|
|
105
|
+
const match = readFileSync("/proc/stat", "utf8").match(/^btime (\d+)$/m);
|
|
106
|
+
bootTimeCache = match ? Number(match[1]) * 1000 : undefined;
|
|
107
|
+
} catch {
|
|
108
|
+
bootTimeCache = undefined;
|
|
109
|
+
}
|
|
110
|
+
return bootTimeCache;
|
|
111
|
+
}
|
|
112
|
+
function procStatFields(path) {
|
|
113
|
+
try {
|
|
114
|
+
const stat = readFileSync(path, "utf8");
|
|
115
|
+
const closeParen = stat.lastIndexOf(")");
|
|
116
|
+
if (closeParen < 0)
|
|
117
|
+
return;
|
|
118
|
+
return stat.slice(closeParen + 2).split(" ");
|
|
119
|
+
} catch {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function processStartTimeMs(pid) {
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
125
|
+
return;
|
|
126
|
+
if (process.platform === "linux") {
|
|
127
|
+
const fields = procStatFields(`/proc/${pid}/stat`);
|
|
128
|
+
const startTicks = fields ? Number(fields[19]) : Number.NaN;
|
|
129
|
+
const boot = bootTimeMs();
|
|
130
|
+
if (Number.isFinite(startTicks) && boot !== undefined) {
|
|
131
|
+
return boot + Math.round(startTicks / clockTicksPerSecond() * 1000);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
const run = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" });
|
|
136
|
+
if (run.status === 0) {
|
|
137
|
+
const parsed = Date.parse(run.stdout.trim());
|
|
138
|
+
if (Number.isFinite(parsed))
|
|
139
|
+
return parsed;
|
|
140
|
+
}
|
|
141
|
+
} catch {}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
function sameProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
145
|
+
if (recorded === undefined || actualMs === undefined)
|
|
146
|
+
return true;
|
|
147
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
148
|
+
if (!Number.isFinite(recordedMs))
|
|
149
|
+
return true;
|
|
150
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
151
|
+
}
|
|
152
|
+
function verifiedProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
153
|
+
if (recorded === undefined || actualMs === undefined)
|
|
154
|
+
return false;
|
|
155
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
156
|
+
if (!Number.isFinite(recordedMs))
|
|
157
|
+
return false;
|
|
158
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/lib/redact.ts
|
|
162
|
+
var SCRUBBED = "[SCRUBBED]";
|
|
163
|
+
var TOKEN_PATTERNS = [
|
|
164
|
+
/\bsk-ant-[A-Za-z0-9_-]{8,}/g,
|
|
165
|
+
/\bsk-proj-[A-Za-z0-9_-]{8,}/g,
|
|
166
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
167
|
+
/\bghp_[A-Za-z0-9]{16,}\b/g,
|
|
168
|
+
/\bgithub_pat_[A-Za-z0-9_]{16,}\b/g,
|
|
169
|
+
/\bxox[a-z]-[A-Za-z0-9-]{8,}/g,
|
|
170
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{10,}\b/g
|
|
171
|
+
];
|
|
172
|
+
var AUTHORIZATION_PATTERN = /(\bAuthorization(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?(?:Bearer|Basic)\s+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
173
|
+
var PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)/g;
|
|
174
|
+
var ASSIGNMENT_PATTERN = /([A-Za-z0-9_.-]{0,64}(?:key|token|secret|passwd|password|passphrase|credentials?)(?:\\?["'])?\s*[:=]\s*)(\\"([^"\\\r\n]{12,})\\"|"((?:[^"\\\r\n]|\\[^\r\n]){12,})"|'([^'\\\r\n]{12,})'|([A-Za-z0-9+/_.=-]{16,}))/gi;
|
|
175
|
+
var BENIGN_KEY_NAME_PATTERN = /(?:^|[._-])(?:idempotency|dedupe|route)[_-]?key$/i;
|
|
176
|
+
var ASSIGNMENT_HINTS = ["key", "token", "secret", "pass", "credential"];
|
|
177
|
+
function mayContainAssignment(text) {
|
|
178
|
+
const lowered = text.toLowerCase();
|
|
179
|
+
return ASSIGNMENT_HINTS.some((hint) => lowered.includes(hint));
|
|
180
|
+
}
|
|
181
|
+
function shannonEntropy(value) {
|
|
182
|
+
const counts = new Map;
|
|
183
|
+
for (const char of value)
|
|
184
|
+
counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
185
|
+
let entropy = 0;
|
|
186
|
+
for (const count of counts.values()) {
|
|
187
|
+
const probability = count / value.length;
|
|
188
|
+
entropy -= probability * Math.log2(probability);
|
|
189
|
+
}
|
|
190
|
+
return entropy;
|
|
191
|
+
}
|
|
192
|
+
function isLikelySecretValue(value) {
|
|
193
|
+
if (value.length < 12)
|
|
194
|
+
return false;
|
|
195
|
+
if (value.includes(SCRUBBED))
|
|
196
|
+
return false;
|
|
197
|
+
if (value.startsWith("/") || value.startsWith("~") || value.startsWith("./"))
|
|
198
|
+
return false;
|
|
199
|
+
return shannonEntropy(value) >= 3.2;
|
|
200
|
+
}
|
|
201
|
+
function scrubSecrets(text) {
|
|
202
|
+
if (!text)
|
|
203
|
+
return text;
|
|
204
|
+
let scrubbed = text;
|
|
205
|
+
for (const pattern of TOKEN_PATTERNS)
|
|
206
|
+
scrubbed = scrubbed.replace(pattern, SCRUBBED);
|
|
207
|
+
scrubbed = scrubbed.replace(AUTHORIZATION_PATTERN, `$1${SCRUBBED}`);
|
|
208
|
+
scrubbed = scrubbed.replace(PEM_PATTERN, SCRUBBED);
|
|
209
|
+
if (!mayContainAssignment(scrubbed))
|
|
210
|
+
return scrubbed;
|
|
211
|
+
scrubbed = scrubbed.replace(ASSIGNMENT_PATTERN, (match, prefix, quotedValue, escapedQuoted, doubleQuoted, singleQuoted, bare) => {
|
|
212
|
+
const value = escapedQuoted ?? doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
213
|
+
const keyName = /^[A-Za-z0-9_.-]*/.exec(prefix)?.[0] ?? "";
|
|
214
|
+
if (BENIGN_KEY_NAME_PATTERN.test(keyName))
|
|
215
|
+
return match;
|
|
216
|
+
if (!isLikelySecretValue(value))
|
|
217
|
+
return match;
|
|
218
|
+
const quote = quotedValue.startsWith("\\\"") ? "\\\"" : quotedValue.startsWith('"') ? '"' : quotedValue.startsWith("'") ? "'" : "";
|
|
219
|
+
return `${prefix}${quote}${SCRUBBED}${quote}`;
|
|
220
|
+
});
|
|
221
|
+
return scrubbed;
|
|
222
|
+
}
|
|
223
|
+
function scrubSecretsDeep(value) {
|
|
224
|
+
if (typeof value === "string")
|
|
225
|
+
return scrubSecrets(value);
|
|
226
|
+
if (Array.isArray(value))
|
|
227
|
+
return value.map((entry) => scrubSecretsDeep(entry));
|
|
228
|
+
if (value !== null && typeof value === "object") {
|
|
229
|
+
const toJSON = value.toJSON;
|
|
230
|
+
if (typeof toJSON === "function")
|
|
231
|
+
return scrubSecretsDeep(toJSON.call(value));
|
|
232
|
+
const scrubbed = {};
|
|
233
|
+
for (const [key, entry] of Object.entries(value))
|
|
234
|
+
scrubbed[key] = scrubSecretsDeep(entry);
|
|
235
|
+
return scrubbed;
|
|
236
|
+
}
|
|
237
|
+
return value;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/lib/recurrence.ts
|
|
46
241
|
function assertDate(value, label) {
|
|
47
242
|
const date = new Date(value);
|
|
48
243
|
if (Number.isNaN(date.getTime()))
|
|
@@ -79,7 +274,20 @@ function parseField(expr, min, max) {
|
|
|
79
274
|
}
|
|
80
275
|
return out;
|
|
81
276
|
}
|
|
277
|
+
var MINUTE_MS = 60000;
|
|
278
|
+
var PARSED_CRON_CACHE_LIMIT = 256;
|
|
279
|
+
var parsedCronCache = new Map;
|
|
280
|
+
var emittedScheduleWarnings = new Set;
|
|
281
|
+
function warnOnce(key, message) {
|
|
282
|
+
if (emittedScheduleWarnings.has(key))
|
|
283
|
+
return;
|
|
284
|
+
emittedScheduleWarnings.add(key);
|
|
285
|
+
console.warn(`[open-loops] WARN ${message}`);
|
|
286
|
+
}
|
|
82
287
|
function parseCron(expr) {
|
|
288
|
+
const cached = parsedCronCache.get(expr);
|
|
289
|
+
if (cached)
|
|
290
|
+
return cached;
|
|
83
291
|
const fields = expr.trim().split(/\s+/);
|
|
84
292
|
if (fields.length !== 5)
|
|
85
293
|
throw new Error(`cron must have 5 fields, got ${fields.length}: "${expr}"`);
|
|
@@ -89,7 +297,7 @@ function parseCron(expr) {
|
|
|
89
297
|
dowSet.delete(7);
|
|
90
298
|
dowSet.add(0);
|
|
91
299
|
}
|
|
92
|
-
|
|
300
|
+
const parsed = {
|
|
93
301
|
minute: parseField(minute, 0, 59),
|
|
94
302
|
hour: parseField(hour, 0, 23),
|
|
95
303
|
dom: parseField(dom, 1, 31),
|
|
@@ -98,6 +306,13 @@ function parseCron(expr) {
|
|
|
98
306
|
domRestricted: dom !== "*",
|
|
99
307
|
dowRestricted: dow !== "*"
|
|
100
308
|
};
|
|
309
|
+
if (parsedCronCache.size >= PARSED_CRON_CACHE_LIMIT) {
|
|
310
|
+
const oldest = parsedCronCache.keys().next().value;
|
|
311
|
+
if (oldest !== undefined)
|
|
312
|
+
parsedCronCache.delete(oldest);
|
|
313
|
+
}
|
|
314
|
+
parsedCronCache.set(expr, parsed);
|
|
315
|
+
return parsed;
|
|
101
316
|
}
|
|
102
317
|
function cronMatches(cron, date) {
|
|
103
318
|
if (!cron.minute.has(date.getMinutes()))
|
|
@@ -171,14 +386,59 @@ function latestIntervalSlot(first, now, everyMs) {
|
|
|
171
386
|
const steps = Math.floor((now.getTime() - first.getTime()) / everyMs);
|
|
172
387
|
return new Date(first.getTime() + steps * everyMs);
|
|
173
388
|
}
|
|
389
|
+
function floorToMinute(date) {
|
|
390
|
+
const floored = new Date(date.getTime());
|
|
391
|
+
floored.setSeconds(0, 0);
|
|
392
|
+
return floored;
|
|
393
|
+
}
|
|
394
|
+
var LATEST_CRON_SCAN_LIMIT_MINUTES = 366 * 24 * 60;
|
|
395
|
+
function latestDailyCronSlot(cron, now) {
|
|
396
|
+
const hoursDesc = [...cron.hour].sort((a, b) => b - a);
|
|
397
|
+
const minutesDesc = [...cron.minute].sort((a, b) => b - a);
|
|
398
|
+
const candidate = floorToMinute(now);
|
|
399
|
+
for (let dayOffset = 0;dayOffset < 2; dayOffset += 1) {
|
|
400
|
+
const isToday = dayOffset === 0;
|
|
401
|
+
for (const hour of hoursDesc) {
|
|
402
|
+
if (isToday && hour > now.getHours())
|
|
403
|
+
continue;
|
|
404
|
+
const minuteCap = isToday && hour === now.getHours() ? now.getMinutes() : 59;
|
|
405
|
+
const minute = minutesDesc.find((value) => value <= minuteCap);
|
|
406
|
+
if (minute === undefined)
|
|
407
|
+
continue;
|
|
408
|
+
candidate.setHours(hour, minute, 0, 0);
|
|
409
|
+
return candidate;
|
|
410
|
+
}
|
|
411
|
+
candidate.setDate(candidate.getDate() - 1);
|
|
412
|
+
}
|
|
413
|
+
throw new Error("unreachable: daily cron pattern must match within two days");
|
|
414
|
+
}
|
|
174
415
|
function latestCronSlot(first, now, expression) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
416
|
+
if (first.getTime() > now.getTime())
|
|
417
|
+
return first;
|
|
418
|
+
const cron = parseCron(expression);
|
|
419
|
+
if (!cron.domRestricted && !cron.dowRestricted && cron.month.size === 12) {
|
|
420
|
+
const latest = latestDailyCronSlot(cron, now);
|
|
421
|
+
return latest.getTime() >= first.getTime() ? latest : first;
|
|
422
|
+
}
|
|
423
|
+
const scanFloorMs = Math.max(first.getTime(), now.getTime() - LATEST_CRON_SCAN_LIMIT_MINUTES * MINUTE_MS);
|
|
424
|
+
const cursor = floorToMinute(now);
|
|
425
|
+
while (cursor.getTime() >= scanFloorMs) {
|
|
426
|
+
if (cronMatches(cron, cursor))
|
|
427
|
+
return cursor;
|
|
428
|
+
cursor.setMinutes(cursor.getMinutes() - 1);
|
|
181
429
|
}
|
|
430
|
+
if (first.getTime() < scanFloorMs) {
|
|
431
|
+
warnOnce(`latest-cron:${expression}`, `latestCronSlot: no match for "${expression}" within the ${LATEST_CRON_SCAN_LIMIT_MINUTES}-minute scan window; using the stored next run as the latest slot`);
|
|
432
|
+
}
|
|
433
|
+
return first;
|
|
434
|
+
}
|
|
435
|
+
var MAX_CATCH_UP_SLOTS = 1000;
|
|
436
|
+
function catchUpSlotLimit(loop) {
|
|
437
|
+
if (loop.catchUpLimit > MAX_CATCH_UP_SLOTS) {
|
|
438
|
+
warnOnce(`catch-up-limit:${loop.id}`, `dueSlots: loop ${loop.id} catchUpLimit ${loop.catchUpLimit} exceeds the per-plan cap; using ${MAX_CATCH_UP_SLOTS}`);
|
|
439
|
+
return MAX_CATCH_UP_SLOTS;
|
|
440
|
+
}
|
|
441
|
+
return loop.catchUpLimit;
|
|
182
442
|
}
|
|
183
443
|
function dueSlots(loop, now) {
|
|
184
444
|
if (!loop.nextRunAt || loop.status !== "active")
|
|
@@ -197,9 +457,10 @@ function dueSlots(loop, now) {
|
|
|
197
457
|
return { slots: [next.toISOString()] };
|
|
198
458
|
case "interval": {
|
|
199
459
|
if (catchUp === "all") {
|
|
460
|
+
const limit = catchUpSlotLimit(loop);
|
|
200
461
|
const slots = [];
|
|
201
462
|
let cursor = next;
|
|
202
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
463
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
203
464
|
slots.push(cursor.toISOString());
|
|
204
465
|
cursor = new Date(cursor.getTime() + loop.schedule.everyMs);
|
|
205
466
|
}
|
|
@@ -211,9 +472,10 @@ function dueSlots(loop, now) {
|
|
|
211
472
|
}
|
|
212
473
|
case "cron": {
|
|
213
474
|
if (catchUp === "all") {
|
|
475
|
+
const limit = catchUpSlotLimit(loop);
|
|
214
476
|
const slots = [];
|
|
215
477
|
let cursor = next;
|
|
216
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
478
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
217
479
|
slots.push(cursor.toISOString());
|
|
218
480
|
cursor = nextCronRun(loop.schedule.expression, cursor);
|
|
219
481
|
}
|
|
@@ -312,8 +574,311 @@ function updateReadyFlags(nodes, planStatus) {
|
|
|
312
574
|
}
|
|
313
575
|
|
|
314
576
|
// src/lib/workflow-spec.ts
|
|
315
|
-
import { readFileSync } from "fs";
|
|
577
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
316
578
|
import { isAbsolute, resolve } from "path";
|
|
579
|
+
|
|
580
|
+
// src/lib/agent-adapter.ts
|
|
581
|
+
import { spawn } from "child_process";
|
|
582
|
+
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
583
|
+
"e",
|
|
584
|
+
"exec",
|
|
585
|
+
"agent",
|
|
586
|
+
"start",
|
|
587
|
+
"--ephemeral",
|
|
588
|
+
"--ignore-rules",
|
|
589
|
+
"--skip-git-repo-check",
|
|
590
|
+
"--json",
|
|
591
|
+
"--output-last-message",
|
|
592
|
+
"-o",
|
|
593
|
+
"--output-schema",
|
|
594
|
+
"--dangerously-bypass-approvals-and-sandbox"
|
|
595
|
+
]);
|
|
596
|
+
var CODEX_LIKE_SANDBOXES = ["read-only", "workspace-write", "danger-full-access"];
|
|
597
|
+
var CURSOR_SANDBOXES = ["enabled", "disabled"];
|
|
598
|
+
var PERMISSION_MODES = ["default", "plan", "auto", "bypass"];
|
|
599
|
+
function assertOptionalNonEmptyString(value, label) {
|
|
600
|
+
if (value === undefined)
|
|
601
|
+
return;
|
|
602
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
603
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
604
|
+
}
|
|
605
|
+
function validateAgentOptions(target, label, capabilities) {
|
|
606
|
+
const provider = target.provider;
|
|
607
|
+
if (typeof target.prompt !== "string" || target.prompt.trim() === "") {
|
|
608
|
+
throw new Error(`${label}.prompt must be a non-empty string`);
|
|
609
|
+
}
|
|
610
|
+
assertOptionalNonEmptyString(target.model, `${label}.model`);
|
|
611
|
+
assertOptionalNonEmptyString(target.variant, `${label}.variant`);
|
|
612
|
+
assertOptionalNonEmptyString(target.agent, `${label}.agent`);
|
|
613
|
+
assertOptionalNonEmptyString(target.authProfile, `${label}.authProfile`);
|
|
614
|
+
assertOptionalNonEmptyString(target.configIsolation, `${label}.configIsolation`);
|
|
615
|
+
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
616
|
+
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
617
|
+
}
|
|
618
|
+
if (target.authProfile !== undefined && provider !== "codewith") {
|
|
619
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
620
|
+
}
|
|
621
|
+
if (provider === "opencode" && (typeof target.model !== "string" || target.model.trim() === "")) {
|
|
622
|
+
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
623
|
+
}
|
|
624
|
+
if (provider === "cursor" && target.variant !== undefined) {
|
|
625
|
+
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
626
|
+
}
|
|
627
|
+
if (provider === "codex" && target.agent !== undefined) {
|
|
628
|
+
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
629
|
+
}
|
|
630
|
+
if (provider === "codewith" && target.agent !== undefined) {
|
|
631
|
+
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
632
|
+
}
|
|
633
|
+
if (provider === "codewith") {
|
|
634
|
+
const unsafe = target.extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
635
|
+
if (unsafe) {
|
|
636
|
+
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (target.addDirs?.length && !["codewith", "codex"].includes(provider)) {
|
|
640
|
+
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
641
|
+
}
|
|
642
|
+
if (target.permissionMode !== undefined) {
|
|
643
|
+
if (!PERMISSION_MODES.includes(target.permissionMode)) {
|
|
644
|
+
throw new Error(`${label}.permissionMode must be one of ${PERMISSION_MODES.join(", ")}`);
|
|
645
|
+
}
|
|
646
|
+
if (target.permissionMode === "plan" && !["claude", "cursor"].includes(provider)) {
|
|
647
|
+
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
648
|
+
}
|
|
649
|
+
if (target.permissionMode === "auto" && provider !== "claude") {
|
|
650
|
+
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (target.sandbox !== undefined) {
|
|
654
|
+
if (!capabilities.sandbox.length) {
|
|
655
|
+
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
656
|
+
}
|
|
657
|
+
if (!capabilities.sandbox.includes(target.sandbox)) {
|
|
658
|
+
throw new Error(`${label}.sandbox must be one of ${capabilities.sandbox.join(", ")}`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function codewithLikeSandbox(target) {
|
|
663
|
+
return target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
664
|
+
}
|
|
665
|
+
function configStringValue(value) {
|
|
666
|
+
return JSON.stringify(value);
|
|
667
|
+
}
|
|
668
|
+
function buildAgentInvocation(target) {
|
|
669
|
+
const isolation = target.configIsolation ?? "safe";
|
|
670
|
+
const permissionMode = target.permissionMode ?? "default";
|
|
671
|
+
const args = [];
|
|
672
|
+
switch (target.provider) {
|
|
673
|
+
case "claude": {
|
|
674
|
+
if (isolation === "safe")
|
|
675
|
+
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
676
|
+
if (permissionMode !== "default") {
|
|
677
|
+
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode;
|
|
678
|
+
args.push("--permission-mode", mode);
|
|
679
|
+
}
|
|
680
|
+
args.push("-p", "--output-format", "json");
|
|
681
|
+
if (target.model)
|
|
682
|
+
args.push("--model", target.model);
|
|
683
|
+
if (target.variant)
|
|
684
|
+
args.push("--effort", target.variant);
|
|
685
|
+
if (target.agent)
|
|
686
|
+
args.push("--agent", target.agent);
|
|
687
|
+
args.push(...target.extraArgs ?? []);
|
|
688
|
+
return { command: "claude", args, stdin: target.prompt };
|
|
689
|
+
}
|
|
690
|
+
case "cursor": {
|
|
691
|
+
args.push("-c", [
|
|
692
|
+
"set -eu",
|
|
693
|
+
"if command -v agent >/dev/null 2>&1; then",
|
|
694
|
+
' exec agent "$@"',
|
|
695
|
+
"else",
|
|
696
|
+
" echo 'Executable not found in PATH: agent' >&2",
|
|
697
|
+
" exit 127",
|
|
698
|
+
"fi"
|
|
699
|
+
].join(`
|
|
700
|
+
`), "openloops-cursor", "-p");
|
|
701
|
+
if (permissionMode === "plan")
|
|
702
|
+
args.push("--mode", "plan");
|
|
703
|
+
if (permissionMode === "bypass")
|
|
704
|
+
args.push("--force");
|
|
705
|
+
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
706
|
+
if (cursorSandbox)
|
|
707
|
+
args.push("--sandbox", cursorSandbox);
|
|
708
|
+
if (target.model)
|
|
709
|
+
args.push("--model", target.model);
|
|
710
|
+
if (target.agent)
|
|
711
|
+
args.push("--agent", target.agent);
|
|
712
|
+
args.push(...target.extraArgs ?? []);
|
|
713
|
+
return { command: "sh", args, stdin: target.prompt, preflightAnyOf: ["agent"] };
|
|
714
|
+
}
|
|
715
|
+
case "codewith": {
|
|
716
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
717
|
+
if (target.variant)
|
|
718
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
719
|
+
args.push("--ask-for-approval", "never", "--sandbox", codewithLikeSandbox(target));
|
|
720
|
+
if (target.cwd)
|
|
721
|
+
args.push("--cd", target.cwd);
|
|
722
|
+
for (const dir of target.addDirs ?? [])
|
|
723
|
+
args.push("--add-dir", dir);
|
|
724
|
+
if (target.model)
|
|
725
|
+
args.push("--model", target.model);
|
|
726
|
+
args.push(...target.extraArgs ?? []);
|
|
727
|
+
args.push("agent", "start");
|
|
728
|
+
if (target.cwd)
|
|
729
|
+
args.push("--cwd", target.cwd);
|
|
730
|
+
args.push(target.prompt);
|
|
731
|
+
return { command: "codewith", args };
|
|
732
|
+
}
|
|
733
|
+
case "codex": {
|
|
734
|
+
if (target.variant)
|
|
735
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
736
|
+
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
737
|
+
if (isolation === "safe")
|
|
738
|
+
args.push("--ignore-rules");
|
|
739
|
+
if (target.cwd)
|
|
740
|
+
args.push("--cd", target.cwd);
|
|
741
|
+
for (const dir of target.addDirs ?? [])
|
|
742
|
+
args.push("--add-dir", dir);
|
|
743
|
+
if (target.model)
|
|
744
|
+
args.push("--model", target.model);
|
|
745
|
+
args.push(...target.extraArgs ?? []);
|
|
746
|
+
return { command: "codex", args, stdin: target.prompt };
|
|
747
|
+
}
|
|
748
|
+
case "aicopilot":
|
|
749
|
+
case "opencode": {
|
|
750
|
+
args.push("run", "--format", "json");
|
|
751
|
+
if (isolation === "safe")
|
|
752
|
+
args.push("--pure");
|
|
753
|
+
if (permissionMode === "bypass")
|
|
754
|
+
args.push("--dangerously-skip-permissions");
|
|
755
|
+
if (target.cwd)
|
|
756
|
+
args.push("--dir", target.cwd);
|
|
757
|
+
if (target.model)
|
|
758
|
+
args.push("--model", target.model);
|
|
759
|
+
if (target.variant)
|
|
760
|
+
args.push("--variant", target.variant);
|
|
761
|
+
if (target.agent)
|
|
762
|
+
args.push("--agent", target.agent);
|
|
763
|
+
args.push(...target.extraArgs ?? []);
|
|
764
|
+
return { command: target.provider, args, stdin: target.prompt };
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
function adapterFor(provider, capabilities) {
|
|
769
|
+
return {
|
|
770
|
+
provider,
|
|
771
|
+
capabilities,
|
|
772
|
+
validate(target, label = provider) {
|
|
773
|
+
validateAgentOptions(target, label, capabilities);
|
|
774
|
+
},
|
|
775
|
+
buildInvocation(target) {
|
|
776
|
+
validateAgentOptions(target, provider, capabilities);
|
|
777
|
+
return buildAgentInvocation(target);
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
var PROVIDER_ADAPTERS = {
|
|
782
|
+
claude: adapterFor("claude", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
783
|
+
cursor: adapterFor("cursor", { sandbox: CURSOR_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
784
|
+
codewith: adapterFor("codewith", { sandbox: CODEX_LIKE_SANDBOXES, durable: true, remote: false, promptChannel: "argv" }),
|
|
785
|
+
codex: adapterFor("codex", { sandbox: CODEX_LIKE_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
786
|
+
aicopilot: adapterFor("aicopilot", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
787
|
+
opencode: adapterFor("opencode", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" })
|
|
788
|
+
};
|
|
789
|
+
var AGENT_PROVIDERS = Object.keys(PROVIDER_ADAPTERS);
|
|
790
|
+
function providerAdapter(provider) {
|
|
791
|
+
const adapter = PROVIDER_ADAPTERS[provider];
|
|
792
|
+
if (!adapter)
|
|
793
|
+
throw new Error(`unsupported agent provider: ${String(provider)}`);
|
|
794
|
+
return adapter;
|
|
795
|
+
}
|
|
796
|
+
var DEFAULT_CAPTURE_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
797
|
+
function killProcessGroup(pgid) {
|
|
798
|
+
try {
|
|
799
|
+
process.kill(-pgid, "SIGTERM");
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
process.kill(pgid, "SIGTERM");
|
|
803
|
+
} catch {}
|
|
804
|
+
}
|
|
805
|
+
setTimeout(() => {
|
|
806
|
+
try {
|
|
807
|
+
process.kill(-pgid, "SIGKILL");
|
|
808
|
+
} catch {
|
|
809
|
+
try {
|
|
810
|
+
process.kill(pgid, "SIGKILL");
|
|
811
|
+
} catch {}
|
|
812
|
+
}
|
|
813
|
+
}, 2000).unref();
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
class BoundedOutputBuffer {
|
|
817
|
+
maxBytes;
|
|
818
|
+
text = "";
|
|
819
|
+
truncatedBytes = 0;
|
|
820
|
+
constructor(maxBytes) {
|
|
821
|
+
this.maxBytes = maxBytes;
|
|
822
|
+
}
|
|
823
|
+
append(chunk) {
|
|
824
|
+
if (!chunk)
|
|
825
|
+
return;
|
|
826
|
+
this.text += chunk;
|
|
827
|
+
if (Buffer.byteLength(this.text, "utf8") <= this.maxBytes)
|
|
828
|
+
return;
|
|
829
|
+
this.text = scrubSecrets(this.text);
|
|
830
|
+
const encoded = Buffer.from(this.text, "utf8");
|
|
831
|
+
if (encoded.length <= this.maxBytes)
|
|
832
|
+
return;
|
|
833
|
+
let start = encoded.length - this.maxBytes;
|
|
834
|
+
while (start < encoded.length && (encoded[start] & 192) === 128)
|
|
835
|
+
start++;
|
|
836
|
+
this.truncatedBytes += start;
|
|
837
|
+
this.text = encoded.subarray(start).toString("utf8");
|
|
838
|
+
}
|
|
839
|
+
value() {
|
|
840
|
+
if (this.truncatedBytes === 0)
|
|
841
|
+
return this.text;
|
|
842
|
+
return `[truncated ${this.truncatedBytes} bytes]
|
|
843
|
+
${this.text}`;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
async function spawnCapture(command, args, opts) {
|
|
847
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_CAPTURE_MAX_OUTPUT_BYTES;
|
|
848
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
849
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
850
|
+
let timedOut = false;
|
|
851
|
+
let error;
|
|
852
|
+
const child = spawn(command, args, {
|
|
853
|
+
cwd: opts.cwd,
|
|
854
|
+
env: opts.env,
|
|
855
|
+
detached: true,
|
|
856
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
857
|
+
});
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
timedOut = true;
|
|
860
|
+
if (child.pid)
|
|
861
|
+
killProcessGroup(child.pid);
|
|
862
|
+
}, opts.timeoutMs);
|
|
863
|
+
timer.unref();
|
|
864
|
+
child.stdout?.setEncoding("utf8");
|
|
865
|
+
child.stderr?.setEncoding("utf8");
|
|
866
|
+
child.stdout?.on("data", (chunk) => stdout.append(chunk));
|
|
867
|
+
child.stderr?.on("data", (chunk) => stderr.append(chunk));
|
|
868
|
+
const [status, signal] = await new Promise((resolve) => {
|
|
869
|
+
child.once("error", (err) => {
|
|
870
|
+
error = err.message;
|
|
871
|
+
resolve([null, null]);
|
|
872
|
+
});
|
|
873
|
+
child.once("close", (code, sig) => resolve([code, sig]));
|
|
874
|
+
});
|
|
875
|
+
clearTimeout(timer);
|
|
876
|
+
if (timedOut && !error)
|
|
877
|
+
error = `timed out after ${opts.timeoutMs}ms`;
|
|
878
|
+
return { status, signal, stdout: stdout.value(), stderr: stderr.value(), error, timedOut };
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/lib/workflow-spec.ts
|
|
317
882
|
function assertObject(value, label) {
|
|
318
883
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
319
884
|
throw new Error(`${label} must be an object`);
|
|
@@ -356,20 +921,6 @@ function optionalStringArray(value, label) {
|
|
|
356
921
|
}).filter(Boolean);
|
|
357
922
|
return values.length ? values : undefined;
|
|
358
923
|
}
|
|
359
|
-
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
360
|
-
"e",
|
|
361
|
-
"exec",
|
|
362
|
-
"agent",
|
|
363
|
-
"start",
|
|
364
|
-
"--ephemeral",
|
|
365
|
-
"--ignore-rules",
|
|
366
|
-
"--skip-git-repo-check",
|
|
367
|
-
"--json",
|
|
368
|
-
"--output-last-message",
|
|
369
|
-
"-o",
|
|
370
|
-
"--output-schema",
|
|
371
|
-
"--dangerously-bypass-approvals-and-sandbox"
|
|
372
|
-
]);
|
|
373
924
|
function optionalAccountRef(value, label) {
|
|
374
925
|
if (value === undefined)
|
|
375
926
|
return;
|
|
@@ -390,7 +941,7 @@ function readPromptFile(rawPath, label, opts) {
|
|
|
390
941
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
391
942
|
let prompt;
|
|
392
943
|
try {
|
|
393
|
-
prompt =
|
|
944
|
+
prompt = readFileSync2(path, "utf8");
|
|
394
945
|
} catch (error) {
|
|
395
946
|
const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined;
|
|
396
947
|
throw new Error(`${label}.promptFile could not be read${code ? `: ${code}` : ""}`);
|
|
@@ -409,7 +960,7 @@ function matchingPromptSource(value, prompt, opts) {
|
|
|
409
960
|
return;
|
|
410
961
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
411
962
|
try {
|
|
412
|
-
return
|
|
963
|
+
return readFileSync2(path, "utf8") === prompt ? { type: "file", path } : undefined;
|
|
413
964
|
} catch {
|
|
414
965
|
return;
|
|
415
966
|
}
|
|
@@ -456,74 +1007,14 @@ function validateTarget(value, label, opts) {
|
|
|
456
1007
|
if (!hasPrompt && !hasPromptFile)
|
|
457
1008
|
throw new Error(`${label}.prompt must be a non-empty string or ${label}.promptFile must be set`);
|
|
458
1009
|
const promptFields = hasPrompt ? { prompt: value.prompt, promptSource: matchingPromptSource(value.promptSource, value.prompt, opts) } : readPromptFile(value.promptFile, label, opts);
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
1010
|
+
if (!AGENT_PROVIDERS.includes(value.provider)) {
|
|
1011
|
+
throw new Error(`${label}.provider must be one of ${AGENT_PROVIDERS.join(", ")}`);
|
|
1012
|
+
}
|
|
462
1013
|
optionalTimeoutMs(value.timeoutMs, `${label}.timeoutMs`);
|
|
463
1014
|
optionalPositiveInteger(value.idleTimeoutMs, `${label}.idleTimeoutMs`);
|
|
464
|
-
if (value.authProfile !== undefined) {
|
|
465
|
-
assertString(value.authProfile, `${label}.authProfile`);
|
|
466
|
-
if (value.provider !== "codewith")
|
|
467
|
-
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
468
|
-
}
|
|
469
|
-
if (value.configIsolation !== undefined) {
|
|
470
|
-
assertString(value.configIsolation, `${label}.configIsolation`);
|
|
471
|
-
if (!["safe", "none"].includes(value.configIsolation))
|
|
472
|
-
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
473
|
-
}
|
|
474
|
-
if (value.model !== undefined)
|
|
475
|
-
assertString(value.model, `${label}.model`);
|
|
476
|
-
if (value.provider === "opencode" && (typeof value.model !== "string" || value.model.trim() === "")) {
|
|
477
|
-
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
478
|
-
}
|
|
479
|
-
if (value.variant !== undefined)
|
|
480
|
-
assertString(value.variant, `${label}.variant`);
|
|
481
|
-
if (value.agent !== undefined)
|
|
482
|
-
assertString(value.agent, `${label}.agent`);
|
|
483
|
-
if (value.provider === "cursor" && value.variant !== undefined)
|
|
484
|
-
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
485
|
-
if (value.provider === "codex" && value.agent !== undefined)
|
|
486
|
-
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
487
|
-
if (value.provider === "codewith" && value.agent !== undefined) {
|
|
488
|
-
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
489
|
-
}
|
|
490
1015
|
const extraArgs = optionalStringArray(value.extraArgs, `${label}.extraArgs`);
|
|
491
|
-
if (value.provider === "codewith") {
|
|
492
|
-
const unsafe = extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
493
|
-
if (unsafe)
|
|
494
|
-
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
495
|
-
}
|
|
496
1016
|
optionalStringArray(value.addDirs, `${label}.addDirs`);
|
|
497
|
-
|
|
498
|
-
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
499
|
-
}
|
|
500
|
-
if (value.permissionMode !== undefined) {
|
|
501
|
-
assertString(value.permissionMode, `${label}.permissionMode`);
|
|
502
|
-
const permissionModes = ["default", "plan", "auto", "bypass"];
|
|
503
|
-
if (!permissionModes.includes(value.permissionMode)) {
|
|
504
|
-
throw new Error(`${label}.permissionMode must be one of ${permissionModes.join(", ")}`);
|
|
505
|
-
}
|
|
506
|
-
if (value.permissionMode === "plan" && !["claude", "cursor"].includes(value.provider)) {
|
|
507
|
-
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
508
|
-
}
|
|
509
|
-
if (value.permissionMode === "auto" && value.provider !== "claude") {
|
|
510
|
-
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
if (value.sandbox !== undefined) {
|
|
514
|
-
assertString(value.sandbox, `${label}.sandbox`);
|
|
515
|
-
const codexLike = ["read-only", "workspace-write", "danger-full-access"];
|
|
516
|
-
const cursorLike = ["enabled", "disabled"];
|
|
517
|
-
if (["codewith", "codex"].includes(value.provider)) {
|
|
518
|
-
if (!codexLike.includes(value.sandbox))
|
|
519
|
-
throw new Error(`${label}.sandbox must be one of ${codexLike.join(", ")}`);
|
|
520
|
-
} else if (value.provider === "cursor") {
|
|
521
|
-
if (!cursorLike.includes(value.sandbox))
|
|
522
|
-
throw new Error(`${label}.sandbox must be one of ${cursorLike.join(", ")}`);
|
|
523
|
-
} else {
|
|
524
|
-
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
525
|
-
}
|
|
526
|
-
}
|
|
1017
|
+
providerAdapter(value.provider).validate({ ...value, extraArgs, ...promptFields }, label);
|
|
527
1018
|
if (value.allowlist !== undefined) {
|
|
528
1019
|
assertObject(value.allowlist, `${label}.allowlist`);
|
|
529
1020
|
optionalStringArray(value.allowlist.tools, `${label}.allowlist.tools`);
|
|
@@ -653,8 +1144,8 @@ function workflowBodyFromJson(value, fallbackName, opts = {}) {
|
|
|
653
1144
|
|
|
654
1145
|
// src/lib/run-artifacts.ts
|
|
655
1146
|
import { createHash } from "crypto";
|
|
656
|
-
import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
|
|
657
|
-
import { basename, join as join2 } from "path";
|
|
1147
|
+
import { mkdirSync as mkdirSync2, renameSync, rmdirSync, rmSync, writeFileSync } from "fs";
|
|
1148
|
+
import { basename, dirname, join as join2 } from "path";
|
|
658
1149
|
function shortHash(value) {
|
|
659
1150
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
660
1151
|
}
|
|
@@ -675,13 +1166,14 @@ function workflowRunProjectSlug(projectKey) {
|
|
|
675
1166
|
return "global";
|
|
676
1167
|
return safeRunPathSlug(projectKey.startsWith("/") ? basename(projectKey) : projectKey, "project");
|
|
677
1168
|
}
|
|
678
|
-
function
|
|
1169
|
+
function stageWorkflowRunManifest(args) {
|
|
679
1170
|
const projectSlug = workflowRunProjectSlug(args.projectKey);
|
|
680
1171
|
const subjectKey = workflowRunSubjectKey(args.subjectKind, args.rawSubjectRef);
|
|
681
1172
|
const dir = join2(args.loopsDataDir, "runs", projectSlug, subjectKey, args.workflowRunId);
|
|
682
1173
|
mkdirSync2(dir, { recursive: true, mode: 448 });
|
|
683
1174
|
const manifestPath = join2(dir, "manifest.json");
|
|
684
|
-
|
|
1175
|
+
const tmpPath = `${manifestPath}.tmp`;
|
|
1176
|
+
writeFileSync(tmpPath, JSON.stringify({
|
|
685
1177
|
version: 1,
|
|
686
1178
|
workflowRunId: args.workflowRunId,
|
|
687
1179
|
workflowId: args.workflowId,
|
|
@@ -694,13 +1186,26 @@ function writeWorkflowRunManifest(args) {
|
|
|
694
1186
|
createdAt: new Date().toISOString(),
|
|
695
1187
|
...args.payload
|
|
696
1188
|
}, null, 2), { mode: 384 });
|
|
697
|
-
return manifestPath;
|
|
1189
|
+
return { manifestPath, tmpPath };
|
|
1190
|
+
}
|
|
1191
|
+
function commitWorkflowRunManifest(staged) {
|
|
1192
|
+
renameSync(staged.tmpPath, staged.manifestPath);
|
|
1193
|
+
return staged.manifestPath;
|
|
1194
|
+
}
|
|
1195
|
+
function discardWorkflowRunManifest(staged) {
|
|
1196
|
+
rmSync(staged.tmpPath, { force: true });
|
|
1197
|
+
try {
|
|
1198
|
+
rmdirSync(dirname(staged.manifestPath));
|
|
1199
|
+
} catch {}
|
|
698
1200
|
}
|
|
699
1201
|
|
|
700
1202
|
// src/lib/store.ts
|
|
701
1203
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
702
1204
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
703
1205
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1206
|
+
var SCHEMA_USER_VERSION = 6;
|
|
1207
|
+
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1208
|
+
var PRUNE_BATCH_SIZE = 400;
|
|
704
1209
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
705
1210
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
706
1211
|
function rowToLoop(row) {
|
|
@@ -741,6 +1246,8 @@ function rowToRun(row) {
|
|
|
741
1246
|
claimedBy: row.claimed_by ?? undefined,
|
|
742
1247
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
743
1248
|
pid: row.pid ?? undefined,
|
|
1249
|
+
pgid: row.pgid ?? undefined,
|
|
1250
|
+
processStartedAt: row.process_started_at ?? undefined,
|
|
744
1251
|
exitCode: row.exit_code ?? undefined,
|
|
745
1252
|
durationMs: row.duration_ms ?? undefined,
|
|
746
1253
|
stdout: row.stdout ?? undefined,
|
|
@@ -926,6 +1433,24 @@ function isProcessAlive(pid) {
|
|
|
926
1433
|
return false;
|
|
927
1434
|
}
|
|
928
1435
|
}
|
|
1436
|
+
function isRecordedProcessAlive(pid, processStartedAt) {
|
|
1437
|
+
if (!pid || !isProcessAlive(pid))
|
|
1438
|
+
return false;
|
|
1439
|
+
return sameProcessStart(processStartedAt ?? undefined, processStartTimeMs(pid));
|
|
1440
|
+
}
|
|
1441
|
+
function isoProcessStart(pid) {
|
|
1442
|
+
const startedMs = processStartTimeMs(pid);
|
|
1443
|
+
return startedMs === undefined ? undefined : new Date(startedMs).toISOString();
|
|
1444
|
+
}
|
|
1445
|
+
function isLiveStepProcess(pid, stepStartedAt) {
|
|
1446
|
+
if (!isProcessAlive(pid))
|
|
1447
|
+
return false;
|
|
1448
|
+
const actualMs = processStartTimeMs(pid);
|
|
1449
|
+
const stepStartMs = stepStartedAt ? Date.parse(stepStartedAt) : Number.NaN;
|
|
1450
|
+
if (actualMs === undefined || !Number.isFinite(stepStartMs))
|
|
1451
|
+
return true;
|
|
1452
|
+
return actualMs >= stepStartMs - START_TIME_TOLERANCE_MS;
|
|
1453
|
+
}
|
|
929
1454
|
function rowToLease(row) {
|
|
930
1455
|
return {
|
|
931
1456
|
id: row.id,
|
|
@@ -945,6 +1470,9 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
945
1470
|
}
|
|
946
1471
|
return;
|
|
947
1472
|
}
|
|
1473
|
+
function scrubbedOrNull(value) {
|
|
1474
|
+
return value == null ? null : scrubSecrets(value);
|
|
1475
|
+
}
|
|
948
1476
|
function chmodIfExists(path, mode) {
|
|
949
1477
|
try {
|
|
950
1478
|
if (existsSync(path))
|
|
@@ -952,7 +1480,7 @@ function chmodIfExists(path, mode) {
|
|
|
952
1480
|
} catch {}
|
|
953
1481
|
}
|
|
954
1482
|
function ensurePrivateStorePath(file) {
|
|
955
|
-
const dir =
|
|
1483
|
+
const dir = dirname2(file);
|
|
956
1484
|
mkdirSync3(dir, { recursive: true, mode: 448 });
|
|
957
1485
|
chmodIfExists(dir, 448);
|
|
958
1486
|
chmodIfExists(file, 384);
|
|
@@ -967,14 +1495,19 @@ class Store {
|
|
|
967
1495
|
const file = path ?? dbPath();
|
|
968
1496
|
if (file !== ":memory:")
|
|
969
1497
|
ensurePrivateStorePath(file);
|
|
970
|
-
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) :
|
|
1498
|
+
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
971
1499
|
this.db = new Database(file);
|
|
972
1500
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
973
1501
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
974
1502
|
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
975
1503
|
if (file !== ":memory:")
|
|
976
1504
|
ensurePrivateStorePath(file);
|
|
977
|
-
|
|
1505
|
+
try {
|
|
1506
|
+
this.migrate();
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
this.db.close();
|
|
1509
|
+
throw error;
|
|
1510
|
+
}
|
|
978
1511
|
}
|
|
979
1512
|
migrate() {
|
|
980
1513
|
this.db.exec(`
|
|
@@ -982,7 +1515,69 @@ class Store {
|
|
|
982
1515
|
id TEXT PRIMARY KEY,
|
|
983
1516
|
applied_at TEXT NOT NULL
|
|
984
1517
|
);
|
|
985
|
-
|
|
1518
|
+
`);
|
|
1519
|
+
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1520
|
+
const userVersion = versionRow?.user_version ?? 0;
|
|
1521
|
+
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1522
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1523
|
+
}
|
|
1524
|
+
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1525
|
+
for (const migration of this.migrations()) {
|
|
1526
|
+
if (!migration.baseline && applied.has(migration.id))
|
|
1527
|
+
continue;
|
|
1528
|
+
migration.apply();
|
|
1529
|
+
if (!applied.has(migration.id)) {
|
|
1530
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (userVersion !== SCHEMA_USER_VERSION)
|
|
1534
|
+
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
1535
|
+
}
|
|
1536
|
+
migrations() {
|
|
1537
|
+
return [
|
|
1538
|
+
{ id: "0001_initial_and_workflows", baseline: true, apply: () => this.createBaseSchema() },
|
|
1539
|
+
{ id: "0002_loop_machines", baseline: true, apply: () => this.addColumnIfMissing("loops", "machine_json", "TEXT") },
|
|
1540
|
+
{
|
|
1541
|
+
id: "0003_goals",
|
|
1542
|
+
baseline: true,
|
|
1543
|
+
apply: () => {
|
|
1544
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1545
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1546
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1547
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1548
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
{
|
|
1552
|
+
id: "0004_loop_archive_metadata",
|
|
1553
|
+
baseline: true,
|
|
1554
|
+
apply: () => {
|
|
1555
|
+
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1556
|
+
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1557
|
+
}
|
|
1558
|
+
},
|
|
1559
|
+
{
|
|
1560
|
+
id: "0005_workflow_invocations_and_admission",
|
|
1561
|
+
baseline: true,
|
|
1562
|
+
apply: () => {
|
|
1563
|
+
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1564
|
+
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1565
|
+
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1566
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1567
|
+
this.createWorkflowRunBackfillIndexes();
|
|
1568
|
+
}
|
|
1569
|
+
},
|
|
1570
|
+
{
|
|
1571
|
+
id: "0006_run_process_tracking",
|
|
1572
|
+
apply: () => {
|
|
1573
|
+
this.addColumnIfMissing("loop_runs", "pgid", "INTEGER");
|
|
1574
|
+
this.addColumnIfMissing("loop_runs", "process_started_at", "TEXT");
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
];
|
|
1578
|
+
}
|
|
1579
|
+
createBaseSchema() {
|
|
1580
|
+
this.db.exec(`
|
|
986
1581
|
CREATE TABLE IF NOT EXISTS loops (
|
|
987
1582
|
id TEXT PRIMARY KEY,
|
|
988
1583
|
name TEXT NOT NULL,
|
|
@@ -1021,6 +1616,8 @@ class Store {
|
|
|
1021
1616
|
claimed_by TEXT,
|
|
1022
1617
|
lease_expires_at TEXT,
|
|
1023
1618
|
pid INTEGER,
|
|
1619
|
+
pgid INTEGER,
|
|
1620
|
+
process_started_at TEXT,
|
|
1024
1621
|
exit_code INTEGER,
|
|
1025
1622
|
duration_ms INTEGER,
|
|
1026
1623
|
stdout TEXT,
|
|
@@ -1245,24 +1842,6 @@ class Store {
|
|
|
1245
1842
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
1246
1843
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1247
1844
|
`);
|
|
1248
|
-
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
1249
|
-
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1250
|
-
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1251
|
-
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1252
|
-
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1253
|
-
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1254
|
-
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1255
|
-
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1256
|
-
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1257
|
-
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1258
|
-
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1259
|
-
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1260
|
-
this.createWorkflowRunBackfillIndexes();
|
|
1261
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
1262
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
1263
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
1264
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0004_loop_archive_metadata", nowIso());
|
|
1265
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0005_workflow_invocations_and_admission", nowIso());
|
|
1266
1845
|
}
|
|
1267
1846
|
addColumnIfMissing(table, column, definition) {
|
|
1268
1847
|
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
@@ -1276,6 +1855,9 @@ class Store {
|
|
|
1276
1855
|
CREATE INDEX IF NOT EXISTS idx_workflow_runs_work_item ON workflow_runs(work_item_id);
|
|
1277
1856
|
`);
|
|
1278
1857
|
}
|
|
1858
|
+
transact(fn) {
|
|
1859
|
+
return this.db.inTransaction ? fn() : this.writeTransaction(fn);
|
|
1860
|
+
}
|
|
1279
1861
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
1280
1862
|
if (!opts.daemonLeaseId)
|
|
1281
1863
|
return;
|
|
@@ -1357,14 +1939,14 @@ class Store {
|
|
|
1357
1939
|
return byId;
|
|
1358
1940
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1359
1941
|
if (rows.length === 0)
|
|
1360
|
-
throw new
|
|
1942
|
+
throw new LoopNotFoundError(idOrName);
|
|
1361
1943
|
if (rows.length > 1)
|
|
1362
|
-
throw new
|
|
1944
|
+
throw new AmbiguousNameError(idOrName);
|
|
1363
1945
|
return rowToLoop(rows[0]);
|
|
1364
1946
|
}
|
|
1365
1947
|
requireLoop(idOrName) {
|
|
1366
1948
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
1367
|
-
throw new
|
|
1949
|
+
throw new LoopNotFoundError(idOrName);
|
|
1368
1950
|
})();
|
|
1369
1951
|
}
|
|
1370
1952
|
listLoops(opts = {}) {
|
|
@@ -1401,7 +1983,9 @@ class Store {
|
|
|
1401
1983
|
try {
|
|
1402
1984
|
const current = this.getLoop(id);
|
|
1403
1985
|
if (!current)
|
|
1404
|
-
throw new
|
|
1986
|
+
throw new LoopNotFoundError(id);
|
|
1987
|
+
if (current.archivedAt)
|
|
1988
|
+
throw new LoopArchivedError(current.name || id);
|
|
1405
1989
|
const merged = { ...current, ...patch, updatedAt: updated };
|
|
1406
1990
|
const res = this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
1407
1991
|
expires_at=$expiresAt, updated_at=$updated
|
|
@@ -1633,10 +2217,10 @@ class Store {
|
|
|
1633
2217
|
renameLoop(id, name, opts = {}) {
|
|
1634
2218
|
const current = this.getLoop(id);
|
|
1635
2219
|
if (!current)
|
|
1636
|
-
throw new
|
|
2220
|
+
throw new LoopNotFoundError(id);
|
|
1637
2221
|
const trimmed = name.trim();
|
|
1638
2222
|
if (!trimmed)
|
|
1639
|
-
throw new
|
|
2223
|
+
throw new ValidationError("loop name must not be empty");
|
|
1640
2224
|
const updated = (opts.now ?? new Date).toISOString();
|
|
1641
2225
|
this.db.query(`UPDATE loops SET name=$name, updated_at=$updated
|
|
1642
2226
|
WHERE id=$id
|
|
@@ -1655,25 +2239,27 @@ class Store {
|
|
|
1655
2239
|
return after;
|
|
1656
2240
|
}
|
|
1657
2241
|
archiveLoop(idOrName) {
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
2242
|
+
return this.transact(() => {
|
|
2243
|
+
const loop = this.requireLoop(idOrName);
|
|
2244
|
+
if (loop.archivedAt)
|
|
2245
|
+
return loop;
|
|
2246
|
+
const updated = nowIso();
|
|
2247
|
+
const archivedStatus = loop.status === "active" ? "paused" : loop.status;
|
|
2248
|
+
this.db.query(`UPDATE loops
|
|
2249
|
+
SET status=$status, archived_at=$archivedAt, archived_from_status=$archivedFromStatus, updated_at=$updated
|
|
2250
|
+
WHERE id=$id`).run({
|
|
2251
|
+
$id: loop.id,
|
|
2252
|
+
$status: archivedStatus,
|
|
2253
|
+
$archivedAt: updated,
|
|
2254
|
+
$archivedFromStatus: loop.status,
|
|
2255
|
+
$updated: updated
|
|
2256
|
+
});
|
|
2257
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
2258
|
+
const archived = this.getLoop(loop.id);
|
|
2259
|
+
if (!archived)
|
|
2260
|
+
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
2261
|
+
return archived;
|
|
1671
2262
|
});
|
|
1672
|
-
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
1673
|
-
const archived = this.getLoop(loop.id);
|
|
1674
|
-
if (!archived)
|
|
1675
|
-
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
1676
|
-
return archived;
|
|
1677
2263
|
}
|
|
1678
2264
|
unarchiveLoop(idOrName) {
|
|
1679
2265
|
const loop = this.requireLoop(idOrName);
|
|
@@ -1694,10 +2280,12 @@ class Store {
|
|
|
1694
2280
|
return unarchived;
|
|
1695
2281
|
}
|
|
1696
2282
|
deleteLoop(idOrName) {
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
2283
|
+
return this.transact(() => {
|
|
2284
|
+
const loop = this.requireLoop(idOrName);
|
|
2285
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2286
|
+
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2287
|
+
return res.changes > 0;
|
|
2288
|
+
});
|
|
1701
2289
|
}
|
|
1702
2290
|
createWorkflow(input) {
|
|
1703
2291
|
const normalized = normalizeCreateWorkflowInput(input);
|
|
@@ -2214,26 +2802,7 @@ class Store {
|
|
|
2214
2802
|
try {
|
|
2215
2803
|
this.assertDaemonLeaseFence(opts, now);
|
|
2216
2804
|
for (const node of withReady) {
|
|
2217
|
-
this.
|
|
2218
|
-
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2219
|
-
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2220
|
-
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2221
|
-
$id: node.nodeId,
|
|
2222
|
-
$goalId: goal.goalId,
|
|
2223
|
-
$planId: goal.planId,
|
|
2224
|
-
$key: node.key,
|
|
2225
|
-
$sequence: node.sequence,
|
|
2226
|
-
$priority: node.priority,
|
|
2227
|
-
$objective: node.objective,
|
|
2228
|
-
$status: node.status,
|
|
2229
|
-
$ready: node.ready ? 1 : 0,
|
|
2230
|
-
$tokenBudget: node.tokenBudget ?? null,
|
|
2231
|
-
$tokensUsed: node.tokensUsed,
|
|
2232
|
-
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2233
|
-
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2234
|
-
$created: node.createdAt,
|
|
2235
|
-
$updated: node.updatedAt
|
|
2236
|
-
});
|
|
2805
|
+
this.insertGoalPlanNode(goal, node);
|
|
2237
2806
|
}
|
|
2238
2807
|
this.db.exec("COMMIT");
|
|
2239
2808
|
} catch (error) {
|
|
@@ -2244,6 +2813,38 @@ class Store {
|
|
|
2244
2813
|
}
|
|
2245
2814
|
return this.listGoalPlanNodes(goalId);
|
|
2246
2815
|
}
|
|
2816
|
+
insertGoalPlanNode(goal, node) {
|
|
2817
|
+
let id = node.nodeId;
|
|
2818
|
+
for (let attempt = 0;attempt < 3; attempt += 1) {
|
|
2819
|
+
const res = this.db.query(`INSERT OR IGNORE INTO goal_plan_nodes (id, goal_id, plan_id, key, sequence, priority, objective, status, ready,
|
|
2820
|
+
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2821
|
+
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2822
|
+
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2823
|
+
$id: id,
|
|
2824
|
+
$goalId: goal.goalId,
|
|
2825
|
+
$planId: goal.planId,
|
|
2826
|
+
$key: node.key,
|
|
2827
|
+
$sequence: node.sequence,
|
|
2828
|
+
$priority: node.priority,
|
|
2829
|
+
$objective: node.objective,
|
|
2830
|
+
$status: node.status,
|
|
2831
|
+
$ready: node.ready ? 1 : 0,
|
|
2832
|
+
$tokenBudget: node.tokenBudget ?? null,
|
|
2833
|
+
$tokensUsed: node.tokensUsed,
|
|
2834
|
+
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2835
|
+
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2836
|
+
$created: node.createdAt,
|
|
2837
|
+
$updated: node.updatedAt
|
|
2838
|
+
});
|
|
2839
|
+
if (res.changes === 1)
|
|
2840
|
+
return;
|
|
2841
|
+
const existingByKey = this.db.query("SELECT id FROM goal_plan_nodes WHERE plan_id = ? AND key = ? LIMIT 1").get(goal.planId, node.key);
|
|
2842
|
+
if (existingByKey)
|
|
2843
|
+
return;
|
|
2844
|
+
id = genId();
|
|
2845
|
+
}
|
|
2846
|
+
throw new Error(`goal plan node was not inserted after id retries: ${goal.planId}/${node.key}`);
|
|
2847
|
+
}
|
|
2247
2848
|
listGoalPlanNodes(goalIdOrPlanId) {
|
|
2248
2849
|
const rows = this.db.query("SELECT * FROM goal_plan_nodes WHERE goal_id = ? OR plan_id = ? ORDER BY sequence ASC").all(goalIdOrPlanId, goalIdOrPlanId);
|
|
2249
2850
|
return rows.map(rowToGoalPlanNode);
|
|
@@ -2335,8 +2936,8 @@ class Store {
|
|
|
2335
2936
|
$status: input.status,
|
|
2336
2937
|
$nodeKey: input.nodeKey ?? null,
|
|
2337
2938
|
$tokensUsed: input.tokensUsed ?? 0,
|
|
2338
|
-
$evidence: input.evidence ? JSON.stringify(input.evidence) : null,
|
|
2339
|
-
$rawResponse: input.rawResponse === undefined ? null : JSON.stringify(input.rawResponse),
|
|
2939
|
+
$evidence: input.evidence ? scrubSecrets(JSON.stringify(scrubSecretsDeep(input.evidence))) : null,
|
|
2940
|
+
$rawResponse: input.rawResponse === undefined ? null : scrubSecrets(JSON.stringify(scrubSecretsDeep(input.rawResponse))),
|
|
2340
2941
|
$created: now,
|
|
2341
2942
|
$updated: now
|
|
2342
2943
|
});
|
|
@@ -2374,7 +2975,6 @@ class Store {
|
|
|
2374
2975
|
const targetInput = input.loop?.target.type === "workflow" ? input.loop.target.input : undefined;
|
|
2375
2976
|
const invocationId = input.invocationId ?? targetInput?.workflowInvocationId ?? targetInput?.invocationId;
|
|
2376
2977
|
const workItemId = input.workItemId ?? targetInput?.workflowWorkItemId ?? targetInput?.workItemId;
|
|
2377
|
-
let manifestPath;
|
|
2378
2978
|
if (input.idempotencyKey) {
|
|
2379
2979
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2380
2980
|
if (existing) {
|
|
@@ -2382,6 +2982,28 @@ class Store {
|
|
|
2382
2982
|
return rowToWorkflowRun(existing);
|
|
2383
2983
|
}
|
|
2384
2984
|
}
|
|
2985
|
+
const runId = genId();
|
|
2986
|
+
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2987
|
+
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2988
|
+
const staged = stageWorkflowRunManifest({
|
|
2989
|
+
loopsDataDir: this.rootDir,
|
|
2990
|
+
workflowRunId: runId,
|
|
2991
|
+
workflowId: input.workflow.id,
|
|
2992
|
+
workflowName: input.workflow.name,
|
|
2993
|
+
invocationId,
|
|
2994
|
+
workItemId,
|
|
2995
|
+
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2996
|
+
subjectKind: invocation?.subjectRef.kind ?? (input.loop ? "loop" : "workflow"),
|
|
2997
|
+
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url ?? input.loop?.name ?? input.workflow.name,
|
|
2998
|
+
payload: {
|
|
2999
|
+
workflowInvocation: invocation,
|
|
3000
|
+
workflowWorkItem: workItem,
|
|
3001
|
+
loopId: input.loop?.id,
|
|
3002
|
+
loopRunId: input.loopRun?.id,
|
|
3003
|
+
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
3006
|
+
const manifestPath = staged.manifestPath;
|
|
2385
3007
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2386
3008
|
try {
|
|
2387
3009
|
this.assertDaemonLeaseFence(input, now);
|
|
@@ -2389,30 +3011,10 @@ class Store {
|
|
|
2389
3011
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2390
3012
|
if (existing) {
|
|
2391
3013
|
this.db.exec("COMMIT");
|
|
3014
|
+
discardWorkflowRunManifest(staged);
|
|
2392
3015
|
return rowToWorkflowRun(existing);
|
|
2393
3016
|
}
|
|
2394
3017
|
}
|
|
2395
|
-
const runId = genId();
|
|
2396
|
-
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2397
|
-
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2398
|
-
manifestPath = invocation || workItem ? writeWorkflowRunManifest({
|
|
2399
|
-
loopsDataDir: this.rootDir,
|
|
2400
|
-
workflowRunId: runId,
|
|
2401
|
-
workflowId: input.workflow.id,
|
|
2402
|
-
workflowName: input.workflow.name,
|
|
2403
|
-
invocationId,
|
|
2404
|
-
workItemId,
|
|
2405
|
-
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2406
|
-
subjectKind: invocation?.subjectRef.kind,
|
|
2407
|
-
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url,
|
|
2408
|
-
payload: {
|
|
2409
|
-
workflowInvocation: invocation,
|
|
2410
|
-
workflowWorkItem: workItem,
|
|
2411
|
-
loopId: input.loop?.id,
|
|
2412
|
-
loopRunId: input.loopRun?.id,
|
|
2413
|
-
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
2414
|
-
}
|
|
2415
|
-
}) : undefined;
|
|
2416
3018
|
this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, invocation_id, work_item_id,
|
|
2417
3019
|
scheduled_for, idempotency_key, manifest_path, status, started_at, finished_at, duration_ms, error,
|
|
2418
3020
|
created_at, updated_at)
|
|
@@ -2479,6 +3081,7 @@ class Store {
|
|
|
2479
3081
|
$created: now
|
|
2480
3082
|
});
|
|
2481
3083
|
this.db.exec("COMMIT");
|
|
3084
|
+
commitWorkflowRunManifest(staged);
|
|
2482
3085
|
const run = this.getWorkflowRun(runId);
|
|
2483
3086
|
if (!run)
|
|
2484
3087
|
throw new Error(`workflow run not found after create: ${runId}`);
|
|
@@ -2487,8 +3090,7 @@ class Store {
|
|
|
2487
3090
|
try {
|
|
2488
3091
|
this.db.exec("ROLLBACK");
|
|
2489
3092
|
} catch {}
|
|
2490
|
-
|
|
2491
|
-
rmSync(manifestPath, { force: true });
|
|
3093
|
+
discardWorkflowRunManifest(staged);
|
|
2492
3094
|
throw error;
|
|
2493
3095
|
}
|
|
2494
3096
|
}
|
|
@@ -2586,29 +3188,32 @@ class Store {
|
|
|
2586
3188
|
return run;
|
|
2587
3189
|
}
|
|
2588
3190
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
3191
|
+
return this.transact(() => {
|
|
3192
|
+
const now = nowIso();
|
|
3193
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
3194
|
+
const live = before.filter((step) => step.pid !== undefined && isLiveStepProcess(step.pid, step.startedAt));
|
|
3195
|
+
if (live.length > 0) {
|
|
3196
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
3197
|
+
}
|
|
3198
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
3199
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
3200
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
3201
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
3202
|
+
if (before.length > 0) {
|
|
3203
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
3204
|
+
reason,
|
|
3205
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
3206
|
+
});
|
|
3207
|
+
}
|
|
3208
|
+
return {
|
|
3209
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
3210
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
3211
|
+
};
|
|
3212
|
+
});
|
|
2609
3213
|
}
|
|
2610
3214
|
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
2611
3215
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3216
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2612
3217
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2613
3218
|
try {
|
|
2614
3219
|
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
@@ -2623,9 +3228,9 @@ class Store {
|
|
|
2623
3228
|
$finished: finishedAt,
|
|
2624
3229
|
$exitCode: patch.exitCode ?? null,
|
|
2625
3230
|
$durationMs: patch.durationMs ?? null,
|
|
2626
|
-
$stdout: patch.stdout
|
|
2627
|
-
$stderr: patch.stderr
|
|
2628
|
-
$error:
|
|
3231
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3232
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3233
|
+
$error: error ?? null,
|
|
2629
3234
|
$updated: finishedAt,
|
|
2630
3235
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2631
3236
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
@@ -2633,15 +3238,15 @@ class Store {
|
|
|
2633
3238
|
if (res.changes === 1) {
|
|
2634
3239
|
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
2635
3240
|
exitCode: patch.exitCode,
|
|
2636
|
-
error
|
|
3241
|
+
error
|
|
2637
3242
|
});
|
|
2638
3243
|
}
|
|
2639
3244
|
this.db.exec("COMMIT");
|
|
2640
|
-
} catch (
|
|
3245
|
+
} catch (error2) {
|
|
2641
3246
|
try {
|
|
2642
3247
|
this.db.exec("ROLLBACK");
|
|
2643
3248
|
} catch {}
|
|
2644
|
-
throw
|
|
3249
|
+
throw error2;
|
|
2645
3250
|
}
|
|
2646
3251
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
2647
3252
|
if (!run)
|
|
@@ -2681,6 +3286,7 @@ class Store {
|
|
|
2681
3286
|
}
|
|
2682
3287
|
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
2683
3288
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3289
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2684
3290
|
let changed = false;
|
|
2685
3291
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2686
3292
|
try {
|
|
@@ -2693,25 +3299,25 @@ class Store {
|
|
|
2693
3299
|
$status: status,
|
|
2694
3300
|
$finished: finishedAt,
|
|
2695
3301
|
$durationMs: patch.durationMs ?? null,
|
|
2696
|
-
$error:
|
|
3302
|
+
$error: error ?? null,
|
|
2697
3303
|
$updated: finishedAt,
|
|
2698
3304
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2699
3305
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
2700
3306
|
});
|
|
2701
3307
|
changed = res.changes === 1;
|
|
2702
3308
|
if (changed)
|
|
2703
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error
|
|
3309
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error });
|
|
2704
3310
|
if (changed) {
|
|
2705
3311
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
2706
|
-
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus,
|
|
3312
|
+
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
2707
3313
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
2708
3314
|
}
|
|
2709
3315
|
this.db.exec("COMMIT");
|
|
2710
|
-
} catch (
|
|
3316
|
+
} catch (error2) {
|
|
2711
3317
|
try {
|
|
2712
3318
|
this.db.exec("ROLLBACK");
|
|
2713
3319
|
} catch {}
|
|
2714
|
-
throw
|
|
3320
|
+
throw error2;
|
|
2715
3321
|
}
|
|
2716
3322
|
const run = this.getWorkflowRun(workflowRunId);
|
|
2717
3323
|
if (!run)
|
|
@@ -2744,24 +3350,26 @@ class Store {
|
|
|
2744
3350
|
}
|
|
2745
3351
|
}
|
|
2746
3352
|
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
3353
|
+
return this.transact(() => {
|
|
3354
|
+
const now = nowIso();
|
|
3355
|
+
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
3356
|
+
const sequence = (current?.sequence ?? 0) + 1;
|
|
3357
|
+
const id = genId();
|
|
3358
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
3359
|
+
VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
|
|
3360
|
+
$id: id,
|
|
3361
|
+
$workflowRunId: workflowRunId,
|
|
3362
|
+
$sequence: sequence,
|
|
3363
|
+
$eventType: eventType,
|
|
3364
|
+
$stepId: stepId ?? null,
|
|
3365
|
+
$payload: payload ? JSON.stringify(payload) : null,
|
|
3366
|
+
$created: now
|
|
3367
|
+
});
|
|
3368
|
+
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
3369
|
+
if (!event)
|
|
3370
|
+
throw new Error(`workflow event not found after append: ${id}`);
|
|
3371
|
+
return rowToWorkflowEvent(event);
|
|
2760
3372
|
});
|
|
2761
|
-
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
2762
|
-
if (!event)
|
|
2763
|
-
throw new Error(`workflow event not found after append: ${id}`);
|
|
2764
|
-
return rowToWorkflowEvent(event);
|
|
2765
3373
|
}
|
|
2766
3374
|
listWorkflowEvents(workflowRunId, limit = 200) {
|
|
2767
3375
|
const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
|
|
@@ -2777,35 +3385,64 @@ class Store {
|
|
|
2777
3385
|
}
|
|
2778
3386
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
2779
3387
|
const now = (opts.now ?? new Date).toISOString();
|
|
2780
|
-
const
|
|
3388
|
+
const startedMs = processStartTimeMs(pid);
|
|
3389
|
+
const processStartedAt = startedMs === undefined ? null : new Date(startedMs).toISOString();
|
|
3390
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2781
3391
|
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
2782
3392
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2783
3393
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2784
3394
|
))`).run({
|
|
2785
3395
|
$id: id,
|
|
2786
3396
|
$pid: pid,
|
|
3397
|
+
$processStartedAt: processStartedAt,
|
|
2787
3398
|
$updated: now,
|
|
2788
3399
|
$claimedBy: claimedBy,
|
|
2789
3400
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2790
3401
|
$now: now
|
|
2791
|
-
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
3402
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2792
3403
|
WHERE id=$id AND status='running'
|
|
2793
3404
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2794
3405
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2795
|
-
))`).run({
|
|
3406
|
+
))`).run({
|
|
3407
|
+
$id: id,
|
|
3408
|
+
$pid: pid,
|
|
3409
|
+
$processStartedAt: processStartedAt,
|
|
3410
|
+
$updated: now,
|
|
3411
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3412
|
+
$now: now
|
|
3413
|
+
});
|
|
2796
3414
|
if (res.changes !== 1)
|
|
2797
3415
|
return;
|
|
2798
3416
|
return this.getRun(id);
|
|
2799
3417
|
}
|
|
3418
|
+
recordRunProcess(runId, info, opts = {}) {
|
|
3419
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3420
|
+
const res = this.db.query(`UPDATE loop_runs SET pid=$pid, pgid=$pgid, process_started_at=$processStartedAt, updated_at=$updated
|
|
3421
|
+
WHERE id=$id AND status='running'
|
|
3422
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3423
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3424
|
+
))`).run({
|
|
3425
|
+
$id: runId,
|
|
3426
|
+
$pid: info.pid,
|
|
3427
|
+
$pgid: info.pgid ?? null,
|
|
3428
|
+
$processStartedAt: info.processStartedAt ?? isoProcessStart(info.pid) ?? now,
|
|
3429
|
+
$updated: now,
|
|
3430
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3431
|
+
$now: now
|
|
3432
|
+
});
|
|
3433
|
+
if (res.changes !== 1)
|
|
3434
|
+
return;
|
|
3435
|
+
return this.getRun(runId);
|
|
3436
|
+
}
|
|
2800
3437
|
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
2801
|
-
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
3438
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid, wsr.started_at AS started_at
|
|
2802
3439
|
FROM workflow_runs wr
|
|
2803
3440
|
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
2804
3441
|
WHERE wr.loop_run_id = ?
|
|
2805
3442
|
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
2806
3443
|
AND wsr.status = 'running'
|
|
2807
3444
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
2808
|
-
return liveWorkflowSteps.some((step) =>
|
|
3445
|
+
return liveWorkflowSteps.some((step) => isLiveStepProcess(step.pid, step.started_at));
|
|
2809
3446
|
}
|
|
2810
3447
|
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
2811
3448
|
const now = nowIso();
|
|
@@ -2879,7 +3516,7 @@ class Store {
|
|
|
2879
3516
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
2880
3517
|
if (existing) {
|
|
2881
3518
|
if (existing.status === "running") {
|
|
2882
|
-
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid
|
|
3519
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
2883
3520
|
this.db.exec("COMMIT");
|
|
2884
3521
|
return;
|
|
2885
3522
|
}
|
|
@@ -2888,7 +3525,7 @@ class Store {
|
|
|
2888
3525
|
return;
|
|
2889
3526
|
}
|
|
2890
3527
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
2891
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3528
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2892
3529
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2893
3530
|
WHERE id=$id AND status='running' AND lease_expires_at <= $now`).run({
|
|
2894
3531
|
$id: existing.id,
|
|
@@ -2910,7 +3547,7 @@ class Store {
|
|
|
2910
3547
|
}
|
|
2911
3548
|
const attempt = existing.attempt + 1;
|
|
2912
3549
|
const res2 = this.db.query(`UPDATE loop_runs SET attempt=$attempt, status='running', started_at=$started, finished_at=NULL,
|
|
2913
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3550
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2914
3551
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2915
3552
|
WHERE id=$id
|
|
2916
3553
|
AND status IN ('failed', 'timed_out', 'abandoned')
|
|
@@ -2958,6 +3595,7 @@ class Store {
|
|
|
2958
3595
|
}
|
|
2959
3596
|
finalizeRun(id, patch, opts = {}) {
|
|
2960
3597
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3598
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2961
3599
|
const params = {
|
|
2962
3600
|
$id: id,
|
|
2963
3601
|
$status: patch.status,
|
|
@@ -2965,41 +3603,43 @@ class Store {
|
|
|
2965
3603
|
$pid: patch.pid ?? null,
|
|
2966
3604
|
$exitCode: patch.exitCode ?? null,
|
|
2967
3605
|
$durationMs: patch.durationMs ?? null,
|
|
2968
|
-
$stdout: patch.stdout
|
|
2969
|
-
$stderr: patch.stderr
|
|
2970
|
-
$error:
|
|
3606
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3607
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3608
|
+
$error: error ?? null,
|
|
2971
3609
|
$updated: finishedAt,
|
|
2972
3610
|
$claimedBy: opts.claimedBy ?? null,
|
|
2973
3611
|
$now: (opts.now ?? new Date).toISOString(),
|
|
2974
3612
|
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
2975
3613
|
};
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
AND
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3614
|
+
return this.transact(() => {
|
|
3615
|
+
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3616
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3617
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
3618
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3619
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3620
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3621
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
3622
|
+
const run = this.getRun(id);
|
|
3623
|
+
if (!run)
|
|
3624
|
+
throw new Error(`run not found after finalize: ${id}`);
|
|
3625
|
+
if (opts.claimedBy && res.changes !== 1)
|
|
3626
|
+
return run;
|
|
3627
|
+
if (res.changes === 1) {
|
|
3628
|
+
this.setWorkflowWorkItemsForLoopRun(run, error, finishedAt);
|
|
3629
|
+
const loop = this.getLoop(run.loopId);
|
|
3630
|
+
const itemStatus = workItemStatusForLoopRun(run.status, run.attempt, loop?.maxAttempts);
|
|
3631
|
+
if (loop?.target.type === "workflow" && itemStatus && itemStatus !== "admitted") {
|
|
3632
|
+
const workItemId = loop.target.input?.workflowWorkItemId ?? loop.target.input?.workItemId;
|
|
3633
|
+
this.maybeArchiveGeneratedRouteWorkflow({
|
|
3634
|
+
workflowId: loop.target.workflowId,
|
|
3635
|
+
loopId: loop.id,
|
|
3636
|
+
workItemId,
|
|
3637
|
+
updated: finishedAt
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
3000
3640
|
}
|
|
3001
|
-
|
|
3002
|
-
|
|
3641
|
+
return run;
|
|
3642
|
+
});
|
|
3003
3643
|
}
|
|
3004
3644
|
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
3005
3645
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
@@ -3049,6 +3689,9 @@ class Store {
|
|
|
3049
3689
|
});
|
|
3050
3690
|
}
|
|
3051
3691
|
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
3692
|
+
return this.recoverExpiredRunLeasesDetailed(now, opts).abandoned;
|
|
3693
|
+
}
|
|
3694
|
+
recoverExpiredRunLeasesDetailed(now = new Date, opts = {}) {
|
|
3052
3695
|
const limit = Math.max(1, Math.min(1000, Math.floor(opts.limit ?? DEFAULT_RECOVERY_BATCH_LIMIT)));
|
|
3053
3696
|
const scanLimit = Math.max(limit, Math.min(5000, Math.floor(opts.scanLimit ?? limit * DEFAULT_RECOVERY_SCAN_MULTIPLIER)));
|
|
3054
3697
|
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
@@ -3056,15 +3699,15 @@ class Store {
|
|
|
3056
3699
|
ORDER BY lease_expires_at ASC
|
|
3057
3700
|
LIMIT ?`).all(now.toISOString(), scanLimit);
|
|
3058
3701
|
const recovered = [];
|
|
3702
|
+
const deferred = [];
|
|
3059
3703
|
for (const row of rows) {
|
|
3060
3704
|
if (recovered.length >= limit)
|
|
3061
3705
|
break;
|
|
3062
|
-
if (row.pid
|
|
3063
|
-
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3064
|
-
continue;
|
|
3065
|
-
}
|
|
3066
|
-
if (this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3706
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at) || this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3067
3707
|
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3708
|
+
const deferredRun = this.getRun(row.id);
|
|
3709
|
+
if (deferredRun)
|
|
3710
|
+
deferred.push(deferredRun);
|
|
3068
3711
|
continue;
|
|
3069
3712
|
}
|
|
3070
3713
|
const finished = now.toISOString();
|
|
@@ -3148,7 +3791,7 @@ class Store {
|
|
|
3148
3791
|
if (run)
|
|
3149
3792
|
recovered.push(run);
|
|
3150
3793
|
}
|
|
3151
|
-
return recovered;
|
|
3794
|
+
return { abandoned: recovered, deferred };
|
|
3152
3795
|
}
|
|
3153
3796
|
expireLoops(now = new Date, opts = {}) {
|
|
3154
3797
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND archived_at IS NULL AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
@@ -3181,6 +3824,83 @@ class Store {
|
|
|
3181
3824
|
const row = status ? this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE status = ?").get(status) : this.db.query("SELECT COUNT(*) AS count FROM loop_runs").get();
|
|
3182
3825
|
return row?.count ?? 0;
|
|
3183
3826
|
}
|
|
3827
|
+
pruneHistory(opts) {
|
|
3828
|
+
const { maxAgeDays, keepPerLoop } = opts;
|
|
3829
|
+
if (maxAgeDays === undefined && keepPerLoop === undefined) {
|
|
3830
|
+
throw new ValidationError("pruneHistory requires maxAgeDays and/or keepPerLoop");
|
|
3831
|
+
}
|
|
3832
|
+
if (maxAgeDays !== undefined && (!Number.isFinite(maxAgeDays) || maxAgeDays < 0)) {
|
|
3833
|
+
throw new ValidationError(`pruneHistory maxAgeDays must be a non-negative number: ${maxAgeDays}`);
|
|
3834
|
+
}
|
|
3835
|
+
if (keepPerLoop !== undefined && (!Number.isInteger(keepPerLoop) || keepPerLoop < 0)) {
|
|
3836
|
+
throw new ValidationError(`pruneHistory keepPerLoop must be a non-negative integer: ${keepPerLoop}`);
|
|
3837
|
+
}
|
|
3838
|
+
const now = opts.now ?? new Date;
|
|
3839
|
+
const dryRun = opts.dryRun ?? false;
|
|
3840
|
+
const cutoff = maxAgeDays === undefined ? undefined : new Date(now.getTime() - maxAgeDays * 86400000).toISOString();
|
|
3841
|
+
const terminal = TERMINAL_RUN_STATUSES.map((status) => `'${status}'`).join(",");
|
|
3842
|
+
const candidateIds = this.db.query(`WITH ranked AS (
|
|
3843
|
+
SELECT id, status, created_at,
|
|
3844
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS recency
|
|
3845
|
+
FROM loop_runs
|
|
3846
|
+
)
|
|
3847
|
+
SELECT id FROM ranked
|
|
3848
|
+
WHERE status IN (${terminal})
|
|
3849
|
+
AND ($cutoff IS NULL OR created_at < $cutoff)
|
|
3850
|
+
AND ($keep IS NULL OR recency > $keep)`).all({ $cutoff: cutoff ?? null, $keep: keepPerLoop ?? null }).map((row) => row.id);
|
|
3851
|
+
const summary = {
|
|
3852
|
+
dryRun,
|
|
3853
|
+
cutoff,
|
|
3854
|
+
keepPerLoop,
|
|
3855
|
+
loopRuns: dryRun ? candidateIds.length : 0,
|
|
3856
|
+
workflowRuns: 0,
|
|
3857
|
+
goalRuns: 0
|
|
3858
|
+
};
|
|
3859
|
+
const manifestPaths = [];
|
|
3860
|
+
for (let offset = 0;offset < candidateIds.length; offset += PRUNE_BATCH_SIZE) {
|
|
3861
|
+
const batch = candidateIds.slice(offset, offset + PRUNE_BATCH_SIZE);
|
|
3862
|
+
const batchPlaceholders = batch.map(() => "?").join(",");
|
|
3863
|
+
if (dryRun) {
|
|
3864
|
+
const workflowRunIds = this.db.query(`SELECT id FROM workflow_runs WHERE loop_run_id IN (${batchPlaceholders})`).all(...batch).map((row) => row.id);
|
|
3865
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3866
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3867
|
+
summary.goalRuns += this.db.query(`SELECT COUNT(*) AS count FROM goal_runs
|
|
3868
|
+
WHERE loop_run_id IN (${batchPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).get(...batch, ...workflowRunIds)?.count ?? 0;
|
|
3869
|
+
continue;
|
|
3870
|
+
}
|
|
3871
|
+
this.transact(() => {
|
|
3872
|
+
const confirmed = this.db.query(`SELECT id FROM loop_runs WHERE id IN (${batchPlaceholders}) AND status IN (${terminal})`).all(...batch).map((row) => row.id);
|
|
3873
|
+
if (confirmed.length === 0)
|
|
3874
|
+
return;
|
|
3875
|
+
const runPlaceholders = confirmed.map(() => "?").join(",");
|
|
3876
|
+
const workflowRuns = this.db.query(`SELECT id, manifest_path FROM workflow_runs WHERE loop_run_id IN (${runPlaceholders})`).all(...confirmed);
|
|
3877
|
+
const workflowRunIds = workflowRuns.map((row) => row.id);
|
|
3878
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3879
|
+
summary.loopRuns += confirmed.length;
|
|
3880
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3881
|
+
summary.goalRuns += this.db.query(`DELETE FROM goal_runs WHERE loop_run_id IN (${runPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).run(...confirmed, ...workflowRunIds).changes;
|
|
3882
|
+
if (workflowRunIds.length > 0) {
|
|
3883
|
+
this.db.query(`DELETE FROM workflow_runs WHERE id IN (${workflowPlaceholders})`).run(...workflowRunIds);
|
|
3884
|
+
}
|
|
3885
|
+
this.db.query(`DELETE FROM loop_runs WHERE id IN (${runPlaceholders}) AND status IN (${terminal})`).run(...confirmed);
|
|
3886
|
+
for (const row of workflowRuns) {
|
|
3887
|
+
if (row.manifest_path)
|
|
3888
|
+
manifestPaths.push(row.manifest_path);
|
|
3889
|
+
}
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
for (const manifestPath of manifestPaths) {
|
|
3893
|
+
const runDir = dirname2(manifestPath);
|
|
3894
|
+
try {
|
|
3895
|
+
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
3896
|
+
rmSync2(runDir, { recursive: true, force: true });
|
|
3897
|
+
} else {
|
|
3898
|
+
rmSync2(manifestPath, { force: true });
|
|
3899
|
+
}
|
|
3900
|
+
} catch {}
|
|
3901
|
+
}
|
|
3902
|
+
return summary;
|
|
3903
|
+
}
|
|
3184
3904
|
acquireDaemonLease(input) {
|
|
3185
3905
|
const now = input.now ?? new Date;
|
|
3186
3906
|
const expiresAt = new Date(now.getTime() + input.ttlMs).toISOString();
|