@awebai/oats 0.39.4 → 0.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +101 -11
- package/docs/capabilities.md +215 -1
- package/docs/desktop-cli-api.md +159 -13
- package/docs/execution-targets.md +4 -0
- package/docs/implementation.md +19 -0
- package/docs/official-catalog.md +2 -2
- package/docs/packages.md +5 -5
- package/docs/release-notes/v0.40.0.md +181 -0
- package/docs/release-notes/v0.40.1.md +20 -0
- package/docs/schedules.md +85 -16
- package/docs/workspaces.md +1 -1
- package/lib/automations.mjs +4 -1
- package/lib/core.mjs +49 -9
- package/lib/instance-events.mjs +178 -23
- package/lib/schedule-command-child.mjs +58 -0
- package/lib/schedule-command.mjs +26 -0
- package/lib/schedule.mjs +142 -37
- package/lib/servers.mjs +14 -2
- package/package-catalog.json +2 -2
- package/package.json +1 -1
- package/skills/oats-getting-started/SKILL.md +1 -1
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/** Private entry point for schedule-command.mjs; no CLI or provider contract.
|
|
2
|
+
* stdin carries launch arguments, stdout carries the observed command result.
|
|
3
|
+
* Command output has separate pipes so it cannot forge that result. */
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { readFileSync } from "node:fs";
|
|
6
|
+
import { signalGroup } from "./process-group.mjs";
|
|
7
|
+
|
|
8
|
+
const { file, args, timeout, graceMs, maxBuffer } = JSON.parse(readFileSync(0, "utf8"));
|
|
9
|
+
const result = await new Promise((resolve) => {
|
|
10
|
+
let child, status = null, signal = null, error;
|
|
11
|
+
let exited = false, closed = false, stopping = false, hardEnd = false, settled = false;
|
|
12
|
+
let deadline, escalation, probe, groupEmpty = false;
|
|
13
|
+
const chunks = { stdout: [], stderr: [] }, sizes = { stdout: 0, stderr: 0 };
|
|
14
|
+
const groupAlive = () => {
|
|
15
|
+
if (groupEmpty || !child?.pid) return false;
|
|
16
|
+
if (process.platform === "win32") return !exited;
|
|
17
|
+
try { process.kill(-child.pid, 0); return true; }
|
|
18
|
+
catch { groupEmpty = true; return false; }
|
|
19
|
+
};
|
|
20
|
+
const send = (sig) => {
|
|
21
|
+
if (process.platform === "win32") { if (!exited) child.kill(sig); }
|
|
22
|
+
else if (groupAlive()) signalGroup(child, sig);
|
|
23
|
+
};
|
|
24
|
+
const finish = () => {
|
|
25
|
+
if (settled || (!closed && !hardEnd) || (!exited && child?.pid)) return;
|
|
26
|
+
// A leader can close while a pipe-free descendant ignores TERM. Keep the
|
|
27
|
+
// supervisor alive until the grace ends or its group is observed empty.
|
|
28
|
+
if (stopping && !hardEnd && groupAlive()) return;
|
|
29
|
+
settled = true;
|
|
30
|
+
clearTimeout(deadline); clearTimeout(escalation); clearInterval(probe);
|
|
31
|
+
resolve({ status, signal, stdout: Buffer.concat(chunks.stdout).toString("utf8"), stderr: Buffer.concat(chunks.stderr).toString("utf8"), ...(error ? { error } : {}) });
|
|
32
|
+
};
|
|
33
|
+
const stop = (cause) => {
|
|
34
|
+
if (stopping || settled) return;
|
|
35
|
+
stopping = true; error = cause;
|
|
36
|
+
send("SIGTERM");
|
|
37
|
+
escalation = setTimeout(() => {
|
|
38
|
+
send("SIGKILL"); hardEnd = true;
|
|
39
|
+
// A detached descendant can escape the group yet retain inherited pipes.
|
|
40
|
+
// Its pipes must not prevent returning after the direct child's exit.
|
|
41
|
+
child.stdout.destroy(); child.stderr.destroy();
|
|
42
|
+
finish();
|
|
43
|
+
}, graceMs);
|
|
44
|
+
probe = setInterval(() => { if (closed) finish(); }, Math.min(50, graceMs));
|
|
45
|
+
};
|
|
46
|
+
try { child = spawn(file, args, { detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] }); }
|
|
47
|
+
catch (e) { error = { code: e.code, message: e.message }; closed = true; finish(); return; }
|
|
48
|
+
for (const stream of ["stdout", "stderr"]) child[stream].on("data", (chunk) => {
|
|
49
|
+
const room = maxBuffer - sizes[stream];
|
|
50
|
+
if (room > 0) { const kept = chunk.subarray(0, room); chunks[stream].push(kept); sizes[stream] += kept.length; }
|
|
51
|
+
if (chunk.length > room) stop({ code: "ENOBUFS", message: `schedule command ${stream} exceeded ${maxBuffer} bytes` });
|
|
52
|
+
});
|
|
53
|
+
child.once("error", (e) => { error ??= { code: e.code, message: e.message }; });
|
|
54
|
+
child.once("exit", (code, sig) => { exited = true; status = code; signal = sig; finish(); });
|
|
55
|
+
child.once("close", () => { closed = true; finish(); });
|
|
56
|
+
deadline = setTimeout(() => stop({ code: "ETIMEDOUT", message: `schedule command exceeded ${timeout} ms` }), timeout);
|
|
57
|
+
});
|
|
58
|
+
process.stdout.write(JSON.stringify(result));
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Synchronous scheduler child execution with an asynchronous timeout supervisor.
|
|
2
|
+
* The tick and its lock callbacks stay synchronous; only the supervisor owns
|
|
3
|
+
* the child, observes its exit and runs the TERM/KILL timers. Its stdout is a
|
|
4
|
+
* private result channel, never the command's stdout. */
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
import { TERM_GRACE_MS } from "./process-group.mjs";
|
|
8
|
+
|
|
9
|
+
const SUPERVISOR = fileURLToPath(new URL("./schedule-command-child.mjs", import.meta.url));
|
|
10
|
+
export function runScheduleCommand(file, args, { cwd, env, timeout = 300_000, graceMs = TERM_GRACE_MS, maxBuffer = 16 * 1024 * 1024 } = {}) {
|
|
11
|
+
for (const [name, value] of Object.entries({ timeout, graceMs, maxBuffer })) {
|
|
12
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > 2 ** 31 - 1) throw new TypeError(`${name} must be a positive 32-bit integer`);
|
|
13
|
+
}
|
|
14
|
+
const r = spawnSync(process.execPath, [SUPERVISOR], {
|
|
15
|
+
cwd, env, encoding: "utf8", input: JSON.stringify({ file, args, timeout, graceMs, maxBuffer }),
|
|
16
|
+
// Each output stream is byte-bounded. JSON can expand a byte to six bytes.
|
|
17
|
+
maxBuffer: maxBuffer * 12 + 65536,
|
|
18
|
+
});
|
|
19
|
+
if (!r.error && r.status === 0) {
|
|
20
|
+
try { return JSON.parse(r.stdout); } catch { /* no trusted supervisor receipt */ }
|
|
21
|
+
}
|
|
22
|
+
// A failed supervisor is not evidence that its command exited. The scheduler
|
|
23
|
+
// conservatively retains the attempt and slot when there is no receipt.
|
|
24
|
+
return { status: null, signal: null, stdout: "", stderr: String(r.stderr || ""),
|
|
25
|
+
error: { code: r.error?.code || "E_SCHEDULE_RUNNER", message: r.error?.message || "schedule child supervisor returned no result" } };
|
|
26
|
+
}
|
package/lib/schedule.mjs
CHANGED
|
@@ -19,8 +19,9 @@
|
|
|
19
19
|
* observed (launched, active, ended, stopped, launch-failed, unknown;
|
|
20
20
|
* delivered, started, skipped for wake) and never claim that a task
|
|
21
21
|
* succeeded. A launch whose side effects cannot be confirmed stays
|
|
22
|
-
* `unknown
|
|
23
|
-
|
|
22
|
+
* `unknown`, its job blocked until `reconcile` proves what happened; it keeps
|
|
23
|
+
* its host slot too, unless it is a command whose process exit was observed. */
|
|
24
|
+
import { runScheduleCommand } from "./schedule-command.mjs";
|
|
24
25
|
import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
25
26
|
import { createHash } from "node:crypto";
|
|
26
27
|
import { homedir } from "node:os";
|
|
@@ -33,12 +34,12 @@ import { herdrSettingRemoved } from "./errors.mjs";
|
|
|
33
34
|
import { withDirLock as withSharedDirLock } from "./dir-lock.mjs";
|
|
34
35
|
import { tickTriggers } from "./triggers.mjs";
|
|
35
36
|
import { resolveMemberClone } from "./instance-resolution.mjs";
|
|
36
|
-
import {
|
|
37
|
-
import { RESERVED_LAUNCH_ENV, MAX_INSTANCE_NAME, findAgent, findInstanceHomes, inspectInstanceSession, inputInstanceSession, startInstanceSession, retirePendingMarkerPath } from "./core.mjs";
|
|
37
|
+
import { SCHEDULE_NAME_RE, readSnapshot, MODEL_RE, automationContext, automationError, baseRow, localEntry, localId, soulOriginOf, splitId } from "./automations.mjs";
|
|
38
|
+
import { RESERVED_LAUNCH_ENV, MAX_INSTANCE_NAME, shq, findAgent, findInstanceHomes, inspectInstanceSession, inputInstanceSession, startInstanceSession, retirePendingMarkerPath } from "./core.mjs";
|
|
38
39
|
|
|
39
40
|
export const SCHEDULE_FILE = "oats-schedules.json";
|
|
40
41
|
export const SCHEDULE_API = 2;
|
|
41
|
-
const ID_RE =
|
|
42
|
+
const ID_RE = SCHEDULE_NAME_RE;
|
|
42
43
|
const HARNESSES = new Set(["pi", "claude", "codex"]);
|
|
43
44
|
const MESSAGE_MAX = 256 * 1024;
|
|
44
45
|
const OATS_BIN = join(dirname(fileURLToPath(import.meta.url)), "..", "bin", "oats.mjs");
|
|
@@ -204,22 +205,51 @@ export function writeState(ws, st) { recordRecentRuns(st); writeJson(statePath(w
|
|
|
204
205
|
|
|
205
206
|
// ------------------------------------------------------- host registry
|
|
206
207
|
|
|
207
|
-
|
|
208
|
-
|
|
208
|
+
const DEFAULT_MAX_CONCURRENT = 5;
|
|
209
|
+
const scheduleCap = (reg) => reg.maxConcurrent ?? DEFAULT_MAX_CONCURRENT;
|
|
210
|
+
|
|
211
|
+
/** Call under registry.lock: persisted caps are explicit operator choices.
|
|
212
|
+
* Pre-v2 registries wrote 1 even when the operator chose nothing. Migrate it
|
|
213
|
+
* once; a subsequent explicit 1 is distinguished by capsVersion: 2. */
|
|
214
|
+
function readRegistryLocked() {
|
|
215
|
+
const path = join(hostScheduleDir(), "registry.json");
|
|
216
|
+
const reg = readJson(path, { version: 1, capsVersion: 2, tickIntervalSec: 60, workspaces: [] });
|
|
209
217
|
if (!Array.isArray(reg.workspaces)) reg.workspaces = [];
|
|
210
|
-
if (!Number.
|
|
218
|
+
if (reg.maxConcurrent !== undefined && (!Number.isSafeInteger(reg.maxConcurrent) || reg.maxConcurrent < 1)) delete reg.maxConcurrent;
|
|
211
219
|
if (!Number.isInteger(reg.tickIntervalSec) || reg.tickIntervalSec < 60) reg.tickIntervalSec = 60;
|
|
212
|
-
//
|
|
213
|
-
// trigger's own concurrency.max still applies). Separate from maxConcurrent: a running scheduled
|
|
214
|
-
// job never holds a trigger, and trigger homes never hold a schedule. Hand-edited, so refused loudly.
|
|
220
|
+
// Trigger and schedule caps are independent; absent trigger cap is unbounded.
|
|
215
221
|
if (reg.triggersMaxConcurrent !== undefined && !(Number.isSafeInteger(reg.triggersMaxConcurrent) && reg.triggersMaxConcurrent >= 1)) {
|
|
216
|
-
throw scheduleError("E_SCHEDULE_INVALID", `${
|
|
222
|
+
throw scheduleError("E_SCHEDULE_INVALID", `${path}: triggersMaxConcurrent must be a positive integer, or absent for no host cap (got ${JSON.stringify(reg.triggersMaxConcurrent)})`, { field: "triggersMaxConcurrent" });
|
|
223
|
+
}
|
|
224
|
+
if ((reg.capsVersion ?? 1) < 2) {
|
|
225
|
+
if (reg.maxConcurrent === 1) delete reg.maxConcurrent;
|
|
226
|
+
reg.capsVersion = 2;
|
|
227
|
+
writeRegistry(reg);
|
|
217
228
|
}
|
|
218
229
|
return reg;
|
|
219
230
|
}
|
|
231
|
+
export function readRegistry() { return withRegistryLock(readRegistryLocked); }
|
|
232
|
+
/** Raw write: callers doing read-modify-write hold registry.lock. */
|
|
220
233
|
export function writeRegistry(reg) { writeJson(join(hostScheduleDir(), "registry.json"), reg); }
|
|
221
|
-
export function registerWorkspace(ws
|
|
222
|
-
|
|
234
|
+
export function registerWorkspace(ws, caps = {}) {
|
|
235
|
+
for (const [field, reset] of [["maxConcurrent", "default"], ["triggersMaxConcurrent", "none"]]) {
|
|
236
|
+
const value = caps[field];
|
|
237
|
+
if (value !== undefined && value !== reset && !(Number.isSafeInteger(value) && value >= 1)) {
|
|
238
|
+
throw scheduleError("E_BAD_ARGS", `${field} must be a positive safe integer or ${reset}`, { field });
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return withRegistryLock(() => {
|
|
242
|
+
const reg = readRegistryLocked(), before = JSON.stringify(reg), abs = resolve(ws);
|
|
243
|
+
for (const [field, reset] of [["maxConcurrent", "default"], ["triggersMaxConcurrent", "none"]]) {
|
|
244
|
+
if (caps[field] === reset) delete reg[field];
|
|
245
|
+
else if (caps[field] !== undefined) reg[field] = caps[field];
|
|
246
|
+
}
|
|
247
|
+
if (!reg.workspaces.includes(abs)) reg.workspaces.push(abs);
|
|
248
|
+
if (JSON.stringify(reg) !== before) writeRegistry(reg);
|
|
249
|
+
return reg;
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
export function unregisterWorkspace(ws) { return withRegistryLock(() => { const reg = readRegistryLocked(); reg.workspaces = reg.workspaces.filter((w) => w !== resolve(ws)); writeRegistry(reg); return reg; }); }
|
|
223
253
|
export function readHostState() { return readJson(join(hostScheduleDir(), "state.json"), {}); }
|
|
224
254
|
export function writeHostState(st) { writeJson(join(hostScheduleDir(), "state.json"), st); }
|
|
225
255
|
|
|
@@ -268,6 +298,11 @@ export function validateCron(cron, tz, field = "cron") {
|
|
|
268
298
|
try { return new Cron(cron.trim(), { timezone: tz, paused: true }); }
|
|
269
299
|
catch (e) { throw scheduleError("E_SCHEDULE_INVALID", `${field}: ${e.message}`, { field }); }
|
|
270
300
|
}
|
|
301
|
+
/** A definition's `description`: a label for people (list, show, the Desktop), never read by a run. */
|
|
302
|
+
const DESCRIPTION_MAX = 200;
|
|
303
|
+
function validateDescription(description) {
|
|
304
|
+
if (typeof description !== "string" || !description.length || [...description].length > DESCRIPTION_MAX || /[\p{Cc}\u2028\u2029]/u.test(description)) throw scheduleError("E_SCHEDULE_INVALID", `description: one line of 1 to ${DESCRIPTION_MAX} characters, without control characters`, { field: "description" });
|
|
305
|
+
}
|
|
271
306
|
function validateMessage(message, field) {
|
|
272
307
|
if (typeof message !== "string" || !message.trim() || message.includes("\0") || Buffer.byteLength(message) > MESSAGE_MAX) throw scheduleError("E_SCHEDULE_INVALID", `${field}: non-empty text without NUL, at most 256 KiB`, { field });
|
|
273
308
|
}
|
|
@@ -286,11 +321,12 @@ export function validateDefinition(ws, def, { checkAgent = true, backendSource }
|
|
|
286
321
|
if (!def || typeof def !== "object" || Array.isArray(def)) throw scheduleError("E_SCHEDULE_INVALID", "definition must be an object", { field: "definition" });
|
|
287
322
|
if (def.kind === "trigger") throw scheduleError("E_SCHEDULE_INVALID", "kind: a trigger is added with `oats trigger add`, not `oats schedule add`", { field: "kind" });
|
|
288
323
|
const id = def.id;
|
|
289
|
-
if (typeof id !== "string" || !ID_RE.test(id)) throw scheduleError("E_SCHEDULE_INVALID", "id: lowercase letters, digits and dashes, 1 to
|
|
324
|
+
if (typeof id !== "string" || !ID_RE.test(id)) throw scheduleError("E_SCHEDULE_INVALID", "id: lowercase letters, digits and dashes, 1 to 100 characters", { field: "id" });
|
|
290
325
|
const enabled = def.enabled === undefined ? true : def.enabled;
|
|
291
326
|
if (typeof enabled !== "boolean") throw scheduleError("E_SCHEDULE_INVALID", "enabled: boolean", { field: "enabled" });
|
|
292
327
|
validateCron(def.cron, def.tz);
|
|
293
328
|
const out = { id, enabled, cron: def.cron.trim(), tz: def.tz.trim(), kind: def.kind };
|
|
329
|
+
if (def.description !== undefined) { validateDescription(def.description); out.description = def.description; }
|
|
294
330
|
if (def.kind === "spawn") {
|
|
295
331
|
if (typeof def.agent !== "string" || !def.agent.trim() || def.agent.trim().startsWith("-")) throw scheduleError("E_SCHEDULE_INVALID", "agent: soul name required (not an option)", { field: "agent" });
|
|
296
332
|
out.agent = def.agent.trim();
|
|
@@ -535,9 +571,9 @@ function spawnViaCli(ws, root, agent, opts, context, io) {
|
|
|
535
571
|
writeFileSync(taskFile, opts.task, { mode: 0o600 });
|
|
536
572
|
argv.push(`--task-file=${taskFile}`);
|
|
537
573
|
let r;
|
|
538
|
-
try { r =
|
|
574
|
+
try { r = runScheduleCommand(process.execPath, [io?.oatsBin || OATS_BIN, ...argv], { cwd: context.dir, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() }); }
|
|
539
575
|
finally { try { rmSync(taskFile, { force: true }); } catch { /* best effort */ } }
|
|
540
|
-
const timedOut = r.error?.code === "ETIMEDOUT" || (r.status === null && r.signal
|
|
576
|
+
const timedOut = r.error?.code === "ETIMEDOUT" || (r.status === null && ["SIGTERM", "SIGKILL"].includes(r.signal));
|
|
541
577
|
const envelope = parseEnvelopeText(r.stdout);
|
|
542
578
|
// The child may have created (and launched) a home before dying: unconfirmed, never a confirmed failure.
|
|
543
579
|
// The thrown message reports RETAINED effects so the tick keeps the attempt and its slot for reconcile.
|
|
@@ -591,25 +627,31 @@ export function parseEnvelopeText(text) {
|
|
|
591
627
|
}
|
|
592
628
|
/** Run one command job: oats-only argv in the job's cwd. A timeout or an
|
|
593
629
|
* unparseable answer means the side effects are unconfirmed: `unknown`,
|
|
594
|
-
* never `launch-failed`.
|
|
630
|
+
* never `launch-failed`. An unconfirmed answer carries `exit` ({status,
|
|
631
|
+
* signal}) when the process's exit was observed. The supervisor sends SIGTERM
|
|
632
|
+
* at the timeout and SIGKILL after the grace, and observes the direct child's
|
|
633
|
+
* exit before returning. `io.command`, `io.oatsBin`, `io.commandTimeoutMs`
|
|
634
|
+
* and `io.commandGraceMs` are test seams; `io.command` counts as exited. */
|
|
595
635
|
function launchCommand(ws, def, io) {
|
|
596
636
|
const argv = def.argv.includes("--json") ? def.argv.slice(1) : [...def.argv.slice(1), "--json"];
|
|
597
|
-
let envelope, timedOut = false, raw = "";
|
|
598
|
-
if (io?.command) envelope = io.command({ cwd: def.cwd, argv });
|
|
637
|
+
let envelope, timedOut = false, raw = "", exit;
|
|
638
|
+
if (io?.command) { envelope = io.command({ cwd: def.cwd, argv }); exit = { status: null, signal: null }; }
|
|
599
639
|
else {
|
|
600
|
-
const r =
|
|
601
|
-
timedOut = r.error?.code === "ETIMEDOUT" || (r.status === null && r.signal
|
|
640
|
+
const r = runScheduleCommand(process.execPath, [io?.oatsBin || OATS_BIN, ...argv], { cwd: def.cwd, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() });
|
|
641
|
+
timedOut = r.error?.code === "ETIMEDOUT" || (r.status === null && ["SIGTERM", "SIGKILL"].includes(r.signal));
|
|
642
|
+
if (r.status !== null || r.signal !== null) exit = { status: r.status, signal: r.signal };
|
|
602
643
|
raw = String(r.stdout || "");
|
|
603
644
|
envelope = parseEnvelopeText(raw);
|
|
604
645
|
}
|
|
646
|
+
const exited = exit ? { exit } : {};
|
|
605
647
|
// No envelope, a malformed one, or a timeout: the process may still have
|
|
606
648
|
// created a home. That is unconfirmed, never a confirmed failure.
|
|
607
|
-
if (timedOut || !envelope || typeof envelope !== "object" || typeof envelope.ok !== "boolean") return { kind: "command", launched: false, unconfirmed: true, error: timedOut ? "command timed out; its side effects are unconfirmed" : "command answered no valid envelope; its side effects are unconfirmed" };
|
|
649
|
+
if (timedOut || !envelope || typeof envelope !== "object" || typeof envelope.ok !== "boolean") return { kind: "command", launched: false, unconfirmed: true, ...exited, error: timedOut ? "command timed out; its side effects are unconfirmed" : "command answered no valid envelope; its side effects are unconfirmed" };
|
|
608
650
|
const res = envelope?.result || {};
|
|
609
651
|
const instance = typeof res.instance === "string" ? res.instance : undefined;
|
|
610
652
|
let home = typeof res.home === "string" ? res.home : undefined;
|
|
611
653
|
if (instance && !home) { const found = findHomesInScope(ws, instance); if (found.length === 1) home = found[0]; }
|
|
612
|
-
if (envelope.ok && instance && !home) return { kind: "command", launched: true, instance, unconfirmed: true, error: `the envelope names instance ${instance} but no home for it is in this scope's roster; run oats schedule reconcile once it appears` };
|
|
654
|
+
if (envelope.ok && instance && !home) return { kind: "command", launched: true, instance, unconfirmed: true, ...exited, error: `the envelope names instance ${instance} but no home for it is in this scope's roster; run oats schedule reconcile once it appears` };
|
|
613
655
|
// A valid ok:false answer that reports an incomplete rollback (a harvest
|
|
614
656
|
// spawn whose compensation could not stop or remove everything), or an
|
|
615
657
|
// operation runner's unconfirmed outcome (timeout, no valid receipt, a
|
|
@@ -619,7 +661,7 @@ function launchCommand(ws, def, io) {
|
|
|
619
661
|
if (!envelope.ok && (UNCONFIRMED_ERROR_CODES.has(envelope.error?.code) || envelope.error?.details?.unconfirmed === true || reportsRetainedEffects(envelope.error?.message))) {
|
|
620
662
|
const partial = envelope.error?.details?.envelope?.result;
|
|
621
663
|
const named = instance || (partial && typeof partial.instance === "string" ? partial.instance : undefined);
|
|
622
|
-
return { kind: "command", launched: false, unconfirmed: true, ...(named ? { instance: named } : {}), error: `command outcome unconfirmed: ${envelope.error?.message}`, errorCode: envelope.error?.code };
|
|
664
|
+
return { kind: "command", launched: false, unconfirmed: true, ...exited, ...(named ? { instance: named } : {}), error: `command outcome unconfirmed: ${envelope.error?.message}`, errorCode: envelope.error?.code };
|
|
623
665
|
}
|
|
624
666
|
// Provenance the recorder has at write time: the answering server (remote
|
|
625
667
|
// spawn) and the launched home's incarnation. Never a transcript.
|
|
@@ -721,7 +763,7 @@ export function tickWorkspace(ws, { now = new Date(), io, reg, wsList, dryRun =
|
|
|
721
763
|
const minute = minuteStart(now);
|
|
722
764
|
const key = minuteKey(minute);
|
|
723
765
|
const registry = reg || readRegistry();
|
|
724
|
-
const bound = registry
|
|
766
|
+
const bound = scheduleCap(registry);
|
|
725
767
|
const wsAll = wsList || [ws];
|
|
726
768
|
const defs = effectiveDefinitions(ws, ctx);
|
|
727
769
|
const st = readState(ws);
|
|
@@ -776,7 +818,11 @@ export function tickWorkspace(ws, { now = new Date(), io, reg, wsList, dryRun =
|
|
|
776
818
|
let invalid;
|
|
777
819
|
try { validateAttemptExecution(js.attempt, jobLockInfo(ws, id)); } catch (error) { invalid = error; }
|
|
778
820
|
record(id, invalid ? "invalid" : "skipped", { reason: invalid ? `admitted attempt is invalid (${invalid.code || "error"}): ${invalid.message}` : "an earlier launch has no recorded result; run oats schedule reconcile", ...(invalid ? { errorCode: invalid.code } : {}) });
|
|
779
|
-
|
|
821
|
+
// Older kernels kept the cause only on lastRun. Adopt it only for this
|
|
822
|
+
// attempt: a crash before recording the result can leave a prior run there.
|
|
823
|
+
const sameRun = js.lastRun?.outcome === "unknown" && js.lastRun.scheduledFor === js.attempt.scheduledFor && js.lastRun.startedAt === js.attempt.startedAt;
|
|
824
|
+
js.attempt.error ??= (sameRun ? js.lastRun.error : null) ?? "launch attempt without a recorded result";
|
|
825
|
+
js.lastRun = { ...(sameRun ? js.lastRun : { kind: def.kind }), outcome: "unknown", error: invalid ? `invalid admitted attempt: ${invalid.message}` : js.attempt.error, ...(invalid ? { errorCode: invalid.code } : {}), scheduledFor: js.attempt.scheduledFor, startedAt: js.attempt.startedAt };
|
|
780
826
|
continue;
|
|
781
827
|
}
|
|
782
828
|
if (def.kind === "wake") {
|
|
@@ -813,8 +859,14 @@ export function tickWorkspace(ws, { now = new Date(), io, reg, wsList, dryRun =
|
|
|
813
859
|
run = { kind: def.kind, launched: false, error: e.message, errorCode: e.code, unconfirmed: def.kind !== "spawn" || reportsRetainedEffects(e.message) };
|
|
814
860
|
}
|
|
815
861
|
if (run.unconfirmed) {
|
|
816
|
-
// Side effects unconfirmed: keep the attempt
|
|
817
|
-
|
|
862
|
+
// Side effects unconfirmed: keep the attempt, with its first error; reconcile decides.
|
|
863
|
+
// A command (or operation) whose process exit was observed runs nothing any more,
|
|
864
|
+
// so it frees its host slot; its attempt still blocks the job itself. Anything else
|
|
865
|
+
// (a spawn, a command that threw) keeps the slot.
|
|
866
|
+
const { exit, ...facts } = run;
|
|
867
|
+
js.attempt = { ...js.attempt, error: run.error, ...(exit ? { exited: true, exitStatus: exit.status, exitSignal: exit.signal } : {}) };
|
|
868
|
+
js.lastRun = { scheduledFor: minute.toISOString(), startedAt: now.toISOString(), ...facts, outcome: "unknown" };
|
|
869
|
+
if (exit) releaseJobLock(ws, id);
|
|
818
870
|
record(id, "unknown", { error: run.error });
|
|
819
871
|
continue;
|
|
820
872
|
}
|
|
@@ -894,7 +946,7 @@ export function runNow(ws, qid, { now = new Date(), io, force = false, ctx = nul
|
|
|
894
946
|
writeState(ws, st);
|
|
895
947
|
if (js.attempt) throw scheduleError("E_SCHEDULE_UNRESOLVED", `schedule ${id} has a launch attempt without a recorded result; run oats schedule reconcile ${id} first`);
|
|
896
948
|
if (def.kind !== "wake" && jobLockInfo(ws, id)) throw scheduleError("E_SCHEDULE_RUNNING", `schedule ${id} is still running`);
|
|
897
|
-
if (def.kind !== "wake" && liveLocks(wsList) >= reg
|
|
949
|
+
if (def.kind !== "wake" && liveLocks(wsList) >= scheduleCap(reg)) throw scheduleError("E_SCHEDULER_BUSY", `host concurrency ${scheduleCap(reg)} reached`);
|
|
898
950
|
const considered = tickWorkspace(ws, { now, io, reg, wsList, only: id, ctx });
|
|
899
951
|
const after = readState(ws);
|
|
900
952
|
if (after.jobs[id]) { after.jobs[id].lastAttemptedMinute = minuteKey(minuteStart(now)); writeState(ws, after); }
|
|
@@ -933,8 +985,7 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
|
|
|
933
985
|
if (homes.length === 1) adopted = { instance: `${def.agent}-${purpose}`, home: homes[0] };
|
|
934
986
|
else if (homes.length > 1) ambiguous = `${homes.length} homes named ${def.agent}-${purpose}: ${homes.join(", ")}`;
|
|
935
987
|
} else if (attemptKind === "command" || attemptKind === "operation") {
|
|
936
|
-
|
|
937
|
-
if (js.lastRun?.instance) named = findHomesInScope(ws, js.lastRun.instance).map((home) => ({ instance: js.lastRun.instance, home }));
|
|
988
|
+
const named = namedHomes(ws, js);
|
|
938
989
|
if (named.length === 1) adopted = named[0];
|
|
939
990
|
else if (named.length > 1) ambiguous = `${named.length} homes named ${js.lastRun.instance}: ${named.map((c) => c.home).join(", ")}`;
|
|
940
991
|
else if (!clear) return { schedule: describeHere(), reconciled: "unknown", remedy: `the command's effects cannot be proven from a matching admitted receipt; check the roster and the host by hand, then run oats schedule reconcile ${id} --clear to record launch-failed and free the slot, or oats schedule remove --force ${id}` };
|
|
@@ -955,6 +1006,60 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
|
|
|
955
1006
|
});
|
|
956
1007
|
}
|
|
957
1008
|
|
|
1009
|
+
/** The homes in the scope of the instance a command attempt's answer named: its only adoptable
|
|
1010
|
+
* receipt. Exactly one is adoptable; none or several leave its effects unproven. */
|
|
1011
|
+
function namedHomes(ws, js) {
|
|
1012
|
+
const instance = js.lastRun?.instance;
|
|
1013
|
+
return instance ? findHomesInScope(ws, instance).map((home) => ({ instance, home })) : [];
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
/** `oats doctor`: every launch attempt without a recorded result (what reconcile acts on: an
|
|
1017
|
+
* attempt, or a lastRun left `unknown`) in the deployment `dep` and in the other scopes this host
|
|
1018
|
+
* ticks, which share its slots. Each is a warning with its age, whether it holds a host slot and
|
|
1019
|
+
* the reconcile command; `--clear` when it is a command whose effects no named home proves.
|
|
1020
|
+
* Reads state only. → { problems, information } */
|
|
1021
|
+
export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
|
|
1022
|
+
const problems = [], information = [];
|
|
1023
|
+
const here = resolve(dep);
|
|
1024
|
+
let scopes = [here];
|
|
1025
|
+
try { scopes = [here, ...readRegistry().workspaces.filter((w) => resolve(w) !== here && existsSync(w))]; }
|
|
1026
|
+
catch (e) { information.push(`the host schedule registry could not be read, so only this deployment's schedules were checked: ${e.message}`); }
|
|
1027
|
+
for (const ws of scopes) {
|
|
1028
|
+
let defs, st;
|
|
1029
|
+
try { defs = readDefinitions(ws); st = readState(ws); }
|
|
1030
|
+
catch (e) { information.push(`the schedules of ${ws} could not be read: ${e.message}`); continue; }
|
|
1031
|
+
// Read the last discovery snapshot directly: placement and refresh would
|
|
1032
|
+
// require credentials/network, and unresolved work matters even when offline.
|
|
1033
|
+
const workspaceKinds = new Map((readSnapshot(ws)?.schedules || []).map((entry) => [entry.id, entry.definition?.kind]));
|
|
1034
|
+
for (const [key, js] of Object.entries(st.jobs)) {
|
|
1035
|
+
if (!js?.attempt && js?.lastRun?.outcome !== "unknown") continue;
|
|
1036
|
+
// A workspace schedule's state key is <member>~<id> (scheduleStateKey); a local one is its id.
|
|
1037
|
+
const workspace = key.includes("~");
|
|
1038
|
+
if (!workspace && !defs.jobs[key]) continue;
|
|
1039
|
+
const id = workspace ? key.replace("~", "/") : key;
|
|
1040
|
+
const kind = (workspace ? workspaceKinds.get(id) : defs.jobs[key]?.kind) ?? js.lastRun?.kind ?? null;
|
|
1041
|
+
const scheduledFor = js.attempt?.scheduledFor ?? js.lastRun?.scheduledFor ?? null;
|
|
1042
|
+
const startedAt = js.attempt?.startedAt ?? js.lastRun?.startedAt ?? null;
|
|
1043
|
+
const started = Date.parse(startedAt);
|
|
1044
|
+
const ageSeconds = Number.isNaN(started) ? null : Math.max(0, Math.floor((now.getTime() - started) / 1000));
|
|
1045
|
+
const holdsSlot = !!jobLockInfo(ws, key);
|
|
1046
|
+
const error = js.attempt?.error ?? js.lastRun?.error ?? null;
|
|
1047
|
+
const unproven = (kind === "command" || kind === "operation") && namedHomes(ws, js).length !== 1;
|
|
1048
|
+
const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
|
|
1049
|
+
const remedy = `oats schedule reconcile ${id}${unproven ? " --clear" : ""}${dir}`;
|
|
1050
|
+
const message = `schedule ${id}${ws === here ? "" : ` (in ${ws})`}: ${kind ? `a ${kind}` : "a"} launch attempt for ${scheduledFor} has no recorded result (started ${startedAt}${ageSeconds === null ? "" : `, ${formatAge(ageSeconds)} ago`})${error ? `: ${error}` : ""}. The job does not run again until it is reconciled, and ${holdsSlot ? "it holds a host slot (counted against maxConcurrent)" : "it holds no host slot"}. ${unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`}`;
|
|
1051
|
+
problems.push({ code: "schedule-unresolved", severity: "warning", scope: ws, id, kind, scheduledFor, startedAt, ageSeconds, holdsSlot, exited: js.attempt?.exited === true, error, remedy, message });
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
return { problems, information };
|
|
1055
|
+
}
|
|
1056
|
+
function formatAge(s) {
|
|
1057
|
+
if (s < 60) return `${s}s`;
|
|
1058
|
+
if (s < 3600) return `${Math.floor(s / 60)}m`;
|
|
1059
|
+
if (s < 86400) return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`;
|
|
1060
|
+
return `${Math.floor(s / 86400)}d ${Math.floor((s % 86400) / 3600)}h`;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
958
1063
|
// -------------------------------------------------------- definitions CRUD
|
|
959
1064
|
|
|
960
1065
|
export const SCHEDULE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
@@ -985,7 +1090,7 @@ export function testSchedule(ws, qid, io, { now = new Date(), ctx = null } = {})
|
|
|
985
1090
|
if (def.launchConfig) argv.push(`--launch-config=${def.launchConfig}`);
|
|
986
1091
|
if (def.harness) argv.push(`--harness=${def.harness}`);
|
|
987
1092
|
if (def.model) argv.push(`--model=${def.model}`);
|
|
988
|
-
const r =
|
|
1093
|
+
const r = runScheduleCommand(process.execPath, [io?.oatsBin || OATS_BIN, ...argv], { cwd: context.dir, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() });
|
|
989
1094
|
const envelope = parseEnvelopeText(r.stdout);
|
|
990
1095
|
if (!envelope || typeof envelope.ok !== "boolean") throw scheduleError("E_SPAWN_FAILED", `the spawn preview answered no valid envelope (exit ${r.status ?? r.signal})`);
|
|
991
1096
|
if (!envelope.ok) throw scheduleError(envelope.error?.code || "E_SPAWN_FAILED", String(envelope.error?.message || "spawn preview failed"));
|
|
@@ -1036,7 +1141,7 @@ function projectHistory(js) {
|
|
|
1036
1141
|
export function describe(ws, qid, io, { defs, st, now = new Date(), ctx = null } = {}) {
|
|
1037
1142
|
if (typeof qid !== "string" || !qid) throw scheduleError("E_BAD_ARGS", `oats schedule show <id> (local/<id> or <member>/<id>)`);
|
|
1038
1143
|
const { scope, name } = splitId(qid);
|
|
1039
|
-
if (scope !== "local" && (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(scope) || !
|
|
1144
|
+
if (scope !== "local" && (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(scope) || !SCHEDULE_NAME_RE.test(name))) throw scheduleError("E_BAD_ARGS", `a workspace schedule id is <member>/<id> (got ${JSON.stringify(qid)})`);
|
|
1040
1145
|
const entry = scope === "local" ? null : (ctx?.schedules || []).find((x) => x.id === `${scope}/${name}`);
|
|
1041
1146
|
if (scope !== "local" && !entry) throw notHere(ws, `${scope}/${name}`, ctx);
|
|
1042
1147
|
if (entry && !entry.placement.runsHere) return scheduleRow(ws, entry, { ...(entry.definition || {}), scope: ws, scheduleApi: 2, scheduleHistoryApi: SCHEDULE_HISTORY_API, nextRun: null, lastRun: null, running: false }, ctx);
|
|
@@ -1056,7 +1161,7 @@ export function describe(ws, qid, io, { defs, st, now = new Date(), ctx = null }
|
|
|
1056
1161
|
const { workspace: _w, ...stored } = def; void _w;
|
|
1057
1162
|
const base = { ...stored, id, scope: ws, scheduleApi: 2, scheduleHistoryApi: SCHEDULE_HISTORY_API, executionStatus: scheduleExecutionStatus(def, intent, lock), nextRun, lastRun: js.lastRun ? { ...js.lastRun, runId: runIdOf(js.lastRun), session: sessionProvenanceOf(js.lastRun) } : null, history, recentRuns, running: !!lock, ...(js.attempt ? { attempt: js.attempt } : {}), ...(js.pendingWake ? { pendingWake: js.pendingWake } : {}) };
|
|
1058
1163
|
const herdr = entry ? null : storedHerdrBackend(ws, id, def);
|
|
1059
|
-
return scheduleRow(ws, entry ?? localEntry("schedule", { ...def, id: name }, { dep: ws, ...(herdr ? { invalid: { code: herdr.code, message: herdr.message, field: herdr.field } } : {}) }), base, ctx);
|
|
1164
|
+
return scheduleRow(ws, entry ?? { ...localEntry("schedule", { ...def, id: name }, { dep: ws, ...(herdr ? { invalid: { code: herdr.code, message: herdr.message, field: herdr.field } } : {}) }), description: def.description ?? null }, base, ctx);
|
|
1060
1165
|
}
|
|
1061
1166
|
/** A schedule's list row: its stored facts, the fields every trigger and schedule row share
|
|
1062
1167
|
* (lib/automations.mjs baseRow) and the schedule's own: `kind` is its run (spawn, command, wake,
|
|
@@ -1157,7 +1262,7 @@ function localScheduleId(qid, verb) {
|
|
|
1157
1262
|
}
|
|
1158
1263
|
/** Save the wake job for a freshly spawned instance (id wake-<instance>). */
|
|
1159
1264
|
export function saveWakeForHome(ws, { instance, home, wake }) {
|
|
1160
|
-
const id = `wake-${instance}`.slice(0,
|
|
1265
|
+
const id = `wake-${instance}`.slice(0, 100).replace(/-+$/, "");
|
|
1161
1266
|
const spec = { id, enabled: wake.enabled === undefined ? true : wake.enabled, cron: wake.cron, tz: wake.tz, kind: "wake", home, message: wake.message };
|
|
1162
1267
|
return addSchedule(ws, spec);
|
|
1163
1268
|
}
|
|
@@ -1192,5 +1297,5 @@ export function schedulerStatus(ws, io) {
|
|
|
1192
1297
|
const host = readHostState();
|
|
1193
1298
|
let unit = { installed: false, active: false };
|
|
1194
1299
|
try { unit = (io?.hostStatus || (() => ({ installed: false, active: false, unavailable: true })))(); } catch (e) { unit = { installed: false, active: false, error: e.message }; }
|
|
1195
|
-
return { installed: !!unit.installed, active: !!unit.active, ...(unit.unit ? { unit: unit.unit } : {}), ...(unit.error ? { error: unit.error } : {}), lastTick: host.lastTick || null, maxConcurrent: reg.
|
|
1300
|
+
return { installed: !!unit.installed, active: !!unit.active, ...(unit.unit ? { unit: unit.unit } : {}), ...(unit.error ? { error: unit.error } : {}), lastTick: host.lastTick || null, maxConcurrent: scheduleCap(reg), triggersMaxConcurrent: reg.triggersMaxConcurrent ?? null, tickIntervalSec: reg.tickIntervalSec, ...(ws ? { workspace: resolve(ws), registered: reg.workspaces.includes(resolve(ws)) } : {}), workspaces: reg.workspaces, live: liveLocks(reg.workspaces.filter((w) => existsSync(w))) };
|
|
1196
1301
|
}
|
package/lib/servers.mjs
CHANGED
|
@@ -1072,8 +1072,20 @@ export function scheduleRemote(serverId, oatsArgs, io = {}) {
|
|
|
1072
1072
|
io = { ...io, serverId };
|
|
1073
1073
|
const server = io.server || getServer(serverId);
|
|
1074
1074
|
const target = targetOf(server);
|
|
1075
|
-
const
|
|
1076
|
-
|
|
1075
|
+
const setsHostCaps = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.some((arg) =>
|
|
1076
|
+
["--max-concurrent", "--triggers-max-concurrent"].some((name) => arg === name || arg.startsWith(`${name}=`)));
|
|
1077
|
+
const capsUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-caps; upgrade oats on that destination before setting or resetting host caps`);
|
|
1078
|
+
let remote;
|
|
1079
|
+
try {
|
|
1080
|
+
remote = checkRemote(target, io);
|
|
1081
|
+
if (!remote.features.includes("schedule")) throw serverError("E_REMOTE_INCOMPATIBLE", `remote oats ${remote.version} at ${target.sshHost} does not advertise schedules; upgrade it there`);
|
|
1082
|
+
} catch (e) {
|
|
1083
|
+
// Unknown probe formats cannot establish cap support. Keep transport
|
|
1084
|
+
// failures distinct, and preserve the existing contract for no-cap calls.
|
|
1085
|
+
if (setsHostCaps && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw capsUnsupported();
|
|
1086
|
+
throw e;
|
|
1087
|
+
}
|
|
1088
|
+
if (setsHostCaps && !remote.features.includes("schedule-host-caps")) throw capsUnsupported();
|
|
1077
1089
|
if (["add", "update"].includes(oatsArgs[0])) {
|
|
1078
1090
|
const indexes = oatsArgs.flatMap((arg, index) => arg === "--spec-json" ? [index] : []);
|
|
1079
1091
|
if (indexes.length > 1) throw serverError("E_BAD_ARGS", "remote schedule spec must be unambiguous");
|
package/package-catalog.json
CHANGED
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
},
|
|
29
29
|
"oats.engineering": {
|
|
30
30
|
"url": "https://github.com/awebai/oats-engineering.git",
|
|
31
|
-
"ref": "v1.8.
|
|
31
|
+
"ref": "v1.8.1",
|
|
32
32
|
"path": "oats-package"
|
|
33
33
|
},
|
|
34
34
|
"oats.cloning": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"oats.framework": {
|
|
40
40
|
"url": "https://github.com/awebai/oats.git",
|
|
41
|
-
"ref": "oats-framework/v1.
|
|
41
|
+
"ref": "oats-framework/v1.6.0",
|
|
42
42
|
"path": "oats-package"
|
|
43
43
|
}
|
|
44
44
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@awebai/oats",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.40.1",
|
|
4
4
|
"description": "OATS (Open Agent Team Specification) — durable souls, disposable instances, targetable capability packages, and the runtime-neutral oats CLI/kernel.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -60,7 +60,7 @@ members:
|
|
|
60
60
|
- git:github.com/acme/agents # the host is a member too
|
|
61
61
|
- git:github.com/acme/platform
|
|
62
62
|
packages:
|
|
63
|
-
oats.framework: v1.
|
|
63
|
+
oats.framework: v1.6.0 # bare versions resolve through the official catalog
|
|
64
64
|
oats.okf: v4.1.1
|
|
65
65
|
defaults:
|
|
66
66
|
capabilities: { oats.core: { from: package } }
|