@hasna/loops 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +365 -35
- package/dist/daemon/index.js +316 -23
- package/dist/index.d.ts +1 -0
- package/dist/index.js +339 -26
- package/dist/lib/executor.d.ts +10 -1
- package/dist/lib/machines.d.ts +16 -0
- package/dist/lib/store.js +10 -2
- package/dist/sdk/index.js +313 -20
- package/dist/types.d.ts +15 -0
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -345,6 +345,7 @@ function rowToLoop(row) {
|
|
|
345
345
|
status: row.status,
|
|
346
346
|
schedule: JSON.parse(row.schedule_json),
|
|
347
347
|
target: JSON.parse(row.target_json),
|
|
348
|
+
machine: row.machine_json ? JSON.parse(row.machine_json) : undefined,
|
|
348
349
|
nextRunAt: row.next_run_at ?? undefined,
|
|
349
350
|
retryScheduledFor: row.retry_scheduled_for ?? undefined,
|
|
350
351
|
catchUp: row.catch_up,
|
|
@@ -487,6 +488,7 @@ class Store {
|
|
|
487
488
|
status TEXT NOT NULL,
|
|
488
489
|
schedule_json TEXT NOT NULL,
|
|
489
490
|
target_json TEXT NOT NULL,
|
|
491
|
+
machine_json TEXT,
|
|
490
492
|
next_run_at TEXT,
|
|
491
493
|
retry_scheduled_for TEXT,
|
|
492
494
|
catch_up TEXT NOT NULL,
|
|
@@ -608,10 +610,14 @@ class Store {
|
|
|
608
610
|
);
|
|
609
611
|
CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
|
|
610
612
|
`);
|
|
613
|
+
try {
|
|
614
|
+
this.db.query("ALTER TABLE loops ADD COLUMN machine_json TEXT").run();
|
|
615
|
+
} catch {}
|
|
611
616
|
try {
|
|
612
617
|
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
613
618
|
} catch {}
|
|
614
619
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
620
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
615
621
|
}
|
|
616
622
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
617
623
|
if (!opts.daemonLeaseId)
|
|
@@ -629,6 +635,7 @@ class Store {
|
|
|
629
635
|
status: "active",
|
|
630
636
|
schedule: input.schedule,
|
|
631
637
|
target: input.target,
|
|
638
|
+
machine: input.machine,
|
|
632
639
|
nextRunAt: initialNextRun(input.schedule, from),
|
|
633
640
|
catchUp: input.catchUp ?? "latest",
|
|
634
641
|
catchUpLimit: input.catchUpLimit ?? 50,
|
|
@@ -640,9 +647,9 @@ class Store {
|
|
|
640
647
|
createdAt: now,
|
|
641
648
|
updatedAt: now
|
|
642
649
|
};
|
|
643
|
-
this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, next_run_at, retry_scheduled_for,
|
|
650
|
+
this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, machine_json, next_run_at, retry_scheduled_for,
|
|
644
651
|
catch_up, catch_up_limit, overlap, max_attempts, retry_delay_ms, lease_ms, expires_at, created_at, updated_at)
|
|
645
|
-
VALUES ($id, $name, $description, $status, $schedule, $target, $nextRun, NULL, $catchUp, $catchUpLimit,
|
|
652
|
+
VALUES ($id, $name, $description, $status, $schedule, $target, $machine, $nextRun, NULL, $catchUp, $catchUpLimit,
|
|
646
653
|
$overlap, $maxAttempts, $retryDelay, $leaseMs, $expiresAt, $created, $updated)`).run({
|
|
647
654
|
$id: loop.id,
|
|
648
655
|
$name: loop.name,
|
|
@@ -650,6 +657,7 @@ class Store {
|
|
|
650
657
|
$status: loop.status,
|
|
651
658
|
$schedule: JSON.stringify(loop.schedule),
|
|
652
659
|
$target: JSON.stringify(loop.target),
|
|
660
|
+
$machine: loop.machine ? JSON.stringify(loop.machine) : null,
|
|
653
661
|
$nextRun: loop.nextRunAt ?? null,
|
|
654
662
|
$catchUp: loop.catchUp,
|
|
655
663
|
$catchUpLimit: loop.catchUpLimit,
|
|
@@ -1500,8 +1508,9 @@ class Store {
|
|
|
1500
1508
|
}
|
|
1501
1509
|
|
|
1502
1510
|
// src/lib/executor.ts
|
|
1503
|
-
import { spawn } from "child_process";
|
|
1511
|
+
import { spawn, spawnSync as spawnSync2 } from "child_process";
|
|
1504
1512
|
import { once } from "events";
|
|
1513
|
+
import { resolveMachineCommand } from "@hasna/machines/consumer";
|
|
1505
1514
|
|
|
1506
1515
|
// src/lib/accounts.ts
|
|
1507
1516
|
import { spawnSync } from "child_process";
|
|
@@ -1658,6 +1667,59 @@ function commandNotFoundMessage(command, env = process.env) {
|
|
|
1658
1667
|
return `Executable not found in PATH: ${command}. Effective PATH=${env.PATH || "(empty)"}`;
|
|
1659
1668
|
}
|
|
1660
1669
|
|
|
1670
|
+
// src/lib/machines.ts
|
|
1671
|
+
import {
|
|
1672
|
+
discoverMachineTopology,
|
|
1673
|
+
resolveMachineRoute
|
|
1674
|
+
} from "@hasna/machines/consumer";
|
|
1675
|
+
function compact(value) {
|
|
1676
|
+
const text = value?.trim();
|
|
1677
|
+
return text ? text : undefined;
|
|
1678
|
+
}
|
|
1679
|
+
function entryToSummary(entry, topology) {
|
|
1680
|
+
return {
|
|
1681
|
+
id: entry.machine_id,
|
|
1682
|
+
hostname: compact(entry.hostname),
|
|
1683
|
+
platform: compact(entry.platform),
|
|
1684
|
+
user: compact(entry.user),
|
|
1685
|
+
workspacePath: compact(entry.workspace_path),
|
|
1686
|
+
route: entry.ssh.route,
|
|
1687
|
+
local: entry.machine_id === topology.local_machine_id || entry.ssh.route === "local",
|
|
1688
|
+
heartbeatStatus: entry.heartbeat_status,
|
|
1689
|
+
tailscaleOnline: entry.tailscale.online,
|
|
1690
|
+
tags: entry.tags
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1693
|
+
function machineFromRoute(route, topology) {
|
|
1694
|
+
if (!route.ok || !route.machine_id) {
|
|
1695
|
+
throw new Error(`OpenMachines route not found for machine: ${route.requested_machine_id}`);
|
|
1696
|
+
}
|
|
1697
|
+
const entry = topology.machines.find((machine) => machine.machine_id === route.machine_id);
|
|
1698
|
+
return {
|
|
1699
|
+
id: route.machine_id,
|
|
1700
|
+
requestedId: route.requested_machine_id !== route.machine_id ? route.requested_machine_id : undefined,
|
|
1701
|
+
route: route.route,
|
|
1702
|
+
local: route.local,
|
|
1703
|
+
confidence: route.confidence,
|
|
1704
|
+
workspacePath: compact(entry?.workspace_path),
|
|
1705
|
+
resolvedAt: route.generated_at,
|
|
1706
|
+
packageVersion: route.package.version,
|
|
1707
|
+
warnings: route.warnings.length ? route.warnings : undefined
|
|
1708
|
+
};
|
|
1709
|
+
}
|
|
1710
|
+
function listOpenMachines() {
|
|
1711
|
+
const topology = discoverMachineTopology();
|
|
1712
|
+
return topology.machines.map((entry) => entryToSummary(entry, topology));
|
|
1713
|
+
}
|
|
1714
|
+
function resolveLoopMachine(machineId) {
|
|
1715
|
+
const topology = discoverMachineTopology();
|
|
1716
|
+
const route = resolveMachineRoute(machineId, { topology });
|
|
1717
|
+
return machineFromRoute(route, topology);
|
|
1718
|
+
}
|
|
1719
|
+
function refreshLoopMachine(machine) {
|
|
1720
|
+
return resolveLoopMachine(machine.id);
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1661
1723
|
// src/lib/executor.ts
|
|
1662
1724
|
var DEFAULT_TIMEOUT_MS = 30 * 60000;
|
|
1663
1725
|
var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
@@ -1678,6 +1740,23 @@ var AUTH_ENV_KEYS = [
|
|
|
1678
1740
|
"XDG_STATE_HOME",
|
|
1679
1741
|
"XDG_CACHE_HOME"
|
|
1680
1742
|
];
|
|
1743
|
+
var TRANSPORT_ENV_KEYS = new Set([
|
|
1744
|
+
"BUN_INSTALL",
|
|
1745
|
+
"HOME",
|
|
1746
|
+
"LANG",
|
|
1747
|
+
"LANGUAGE",
|
|
1748
|
+
"LOGNAME",
|
|
1749
|
+
"PATH",
|
|
1750
|
+
"SHELL",
|
|
1751
|
+
"SSH_AGENT_PID",
|
|
1752
|
+
"SSH_AUTH_SOCK",
|
|
1753
|
+
"TERM",
|
|
1754
|
+
"TMP",
|
|
1755
|
+
"TMPDIR",
|
|
1756
|
+
"TEMP",
|
|
1757
|
+
"USER",
|
|
1758
|
+
"XDG_RUNTIME_DIR"
|
|
1759
|
+
]);
|
|
1681
1760
|
function appendBounded(current, chunk, maxBytes) {
|
|
1682
1761
|
const next = current + chunk.toString("utf8");
|
|
1683
1762
|
if (Buffer.byteLength(next, "utf8") <= maxBytes)
|
|
@@ -1704,6 +1783,29 @@ function killProcessGroup(pid) {
|
|
|
1704
1783
|
}
|
|
1705
1784
|
}, 2000).unref();
|
|
1706
1785
|
}
|
|
1786
|
+
function shellQuote(value) {
|
|
1787
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
1788
|
+
}
|
|
1789
|
+
function metadataEnv(metadata) {
|
|
1790
|
+
const env = {};
|
|
1791
|
+
if (metadata.loopId)
|
|
1792
|
+
env.LOOPS_LOOP_ID = metadata.loopId;
|
|
1793
|
+
if (metadata.loopName)
|
|
1794
|
+
env.LOOPS_LOOP_NAME = metadata.loopName;
|
|
1795
|
+
if (metadata.runId)
|
|
1796
|
+
env.LOOPS_RUN_ID = metadata.runId;
|
|
1797
|
+
if (metadata.scheduledFor)
|
|
1798
|
+
env.LOOPS_SCHEDULED_FOR = metadata.scheduledFor;
|
|
1799
|
+
if (metadata.workflowId)
|
|
1800
|
+
env.LOOPS_WORKFLOW_ID = metadata.workflowId;
|
|
1801
|
+
if (metadata.workflowName)
|
|
1802
|
+
env.LOOPS_WORKFLOW_NAME = metadata.workflowName;
|
|
1803
|
+
if (metadata.workflowRunId)
|
|
1804
|
+
env.LOOPS_WORKFLOW_RUN_ID = metadata.workflowRunId;
|
|
1805
|
+
if (metadata.workflowStepId)
|
|
1806
|
+
env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
|
|
1807
|
+
return env;
|
|
1808
|
+
}
|
|
1707
1809
|
function providerCommand(provider) {
|
|
1708
1810
|
switch (provider) {
|
|
1709
1811
|
case "claude":
|
|
@@ -1825,26 +1927,213 @@ function executionEnv(spec, metadata, opts) {
|
|
|
1825
1927
|
}
|
|
1826
1928
|
Object.assign(env, spec.env ?? {});
|
|
1827
1929
|
env.PATH = normalizeExecutionPath(env);
|
|
1828
|
-
|
|
1829
|
-
env.LOOPS_LOOP_ID = metadata.loopId;
|
|
1830
|
-
if (metadata.loopName)
|
|
1831
|
-
env.LOOPS_LOOP_NAME = metadata.loopName;
|
|
1832
|
-
if (metadata.runId)
|
|
1833
|
-
env.LOOPS_RUN_ID = metadata.runId;
|
|
1834
|
-
if (metadata.scheduledFor)
|
|
1835
|
-
env.LOOPS_SCHEDULED_FOR = metadata.scheduledFor;
|
|
1836
|
-
if (metadata.workflowId)
|
|
1837
|
-
env.LOOPS_WORKFLOW_ID = metadata.workflowId;
|
|
1838
|
-
if (metadata.workflowName)
|
|
1839
|
-
env.LOOPS_WORKFLOW_NAME = metadata.workflowName;
|
|
1840
|
-
if (metadata.workflowRunId)
|
|
1841
|
-
env.LOOPS_WORKFLOW_RUN_ID = metadata.workflowRunId;
|
|
1842
|
-
if (metadata.workflowStepId)
|
|
1843
|
-
env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
|
|
1930
|
+
Object.assign(env, metadataEnv(metadata));
|
|
1844
1931
|
return env;
|
|
1845
1932
|
}
|
|
1933
|
+
function resolvedMachine(opts) {
|
|
1934
|
+
if (!opts.machine)
|
|
1935
|
+
return;
|
|
1936
|
+
return (opts.machineResolver ?? refreshLoopMachine)(opts.machine);
|
|
1937
|
+
}
|
|
1938
|
+
function commandForShell(spec) {
|
|
1939
|
+
if (!spec.args.length)
|
|
1940
|
+
return spec.command;
|
|
1941
|
+
return [spec.command, ...spec.args.map(shellQuote)].join(" ");
|
|
1942
|
+
}
|
|
1943
|
+
function hereDoc(value) {
|
|
1944
|
+
let delimiter2 = `__OPENLOOPS_STDIN_${Math.random().toString(36).slice(2).toUpperCase()}__`;
|
|
1945
|
+
while (value.split(/\r?\n/).includes(delimiter2)) {
|
|
1946
|
+
delimiter2 = `__OPENLOOPS_STDIN_${Math.random().toString(36).slice(2).toUpperCase()}__`;
|
|
1947
|
+
}
|
|
1948
|
+
return [`cat > "$__OPENLOOPS_STDIN" <<'${delimiter2}'`, value, delimiter2];
|
|
1949
|
+
}
|
|
1950
|
+
function remoteBootstrapLines(spec, metadata) {
|
|
1951
|
+
const lines = [
|
|
1952
|
+
"set -e",
|
|
1953
|
+
'export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$HOME/.npm-global/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${PATH:+:$PATH}"'
|
|
1954
|
+
];
|
|
1955
|
+
if (spec.cwd)
|
|
1956
|
+
lines.push(`cd ${shellQuote(spec.cwd)}`);
|
|
1957
|
+
if (spec.account) {
|
|
1958
|
+
if (!spec.accountTool)
|
|
1959
|
+
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
1960
|
+
lines.push("if ! command -v accounts >/dev/null 2>&1; then echo 'accounts CLI is not available on remote machine' >&2; exit 127; fi", `unset ${AUTH_ENV_KEYS.join(" ")}`, `eval "$(accounts env ${shellQuote(spec.account.profile)} --tool ${shellQuote(spec.accountTool)})"`, `export LOOPS_ACCOUNT_PROFILE=${shellQuote(spec.account.profile)}`, `export LOOPS_ACCOUNT_TOOL=${shellQuote(spec.accountTool)}`);
|
|
1961
|
+
}
|
|
1962
|
+
for (const [key, value] of Object.entries({ ...metadataEnv(metadata), ...spec.env ?? {} })) {
|
|
1963
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
|
|
1964
|
+
continue;
|
|
1965
|
+
lines.push(`export ${key}=${shellQuote(value)}`);
|
|
1966
|
+
}
|
|
1967
|
+
return lines;
|
|
1968
|
+
}
|
|
1969
|
+
function remoteScript(spec, metadata) {
|
|
1970
|
+
const lines = remoteBootstrapLines(spec, metadata);
|
|
1971
|
+
let stdinRedirect = "";
|
|
1972
|
+
if (spec.stdin !== undefined) {
|
|
1973
|
+
lines.push('__OPENLOOPS_STDIN="$(mktemp -t openloops-stdin.XXXXXX)"', `trap 'rm -f "$__OPENLOOPS_STDIN"' EXIT`);
|
|
1974
|
+
lines.push(...hereDoc(spec.stdin));
|
|
1975
|
+
stdinRedirect = ' < "$__OPENLOOPS_STDIN"';
|
|
1976
|
+
}
|
|
1977
|
+
const invocation = spec.shell ? `sh -c ${shellQuote(commandForShell(spec))}${stdinRedirect}` : `${[spec.command, ...spec.args].map(shellQuote).join(" ")}${stdinRedirect}`;
|
|
1978
|
+
lines.push(invocation);
|
|
1979
|
+
return `${lines.join(`
|
|
1980
|
+
`)}
|
|
1981
|
+
`;
|
|
1982
|
+
}
|
|
1983
|
+
function remotePreflightScript(spec, metadata) {
|
|
1984
|
+
return [
|
|
1985
|
+
...remoteBootstrapLines(spec, metadata),
|
|
1986
|
+
"command -v bash >/dev/null 2>&1",
|
|
1987
|
+
`command -v ${shellQuote(spec.shell ? "sh" : spec.command)} >/dev/null 2>&1`
|
|
1988
|
+
].join(`
|
|
1989
|
+
`);
|
|
1990
|
+
}
|
|
1991
|
+
function transportEnv(opts) {
|
|
1992
|
+
const source = opts.env ?? process.env;
|
|
1993
|
+
const env = {};
|
|
1994
|
+
for (const [key, value] of Object.entries(source)) {
|
|
1995
|
+
if (value === undefined)
|
|
1996
|
+
continue;
|
|
1997
|
+
if (TRANSPORT_ENV_KEYS.has(key) || key.startsWith("LC_"))
|
|
1998
|
+
env[key] = value;
|
|
1999
|
+
}
|
|
2000
|
+
env.PATH = normalizeExecutionPath(env);
|
|
2001
|
+
return env;
|
|
2002
|
+
}
|
|
2003
|
+
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
2004
|
+
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
2005
|
+
const result = spawnSync2(plan.command, plan.args, {
|
|
2006
|
+
encoding: "utf8",
|
|
2007
|
+
env: transportEnv(opts),
|
|
2008
|
+
input: remotePreflightScript(spec, metadata),
|
|
2009
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
2010
|
+
timeout: 15000
|
|
2011
|
+
});
|
|
2012
|
+
if (result.error)
|
|
2013
|
+
throw new Error(`remote preflight failed on ${machine.id}: ${result.error.message}`);
|
|
2014
|
+
if ((result.status ?? 1) !== 0) {
|
|
2015
|
+
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
2016
|
+
throw new Error(`remote preflight failed on ${machine.id}${detail ? `: ${detail}` : ""}`);
|
|
2017
|
+
}
|
|
2018
|
+
}
|
|
2019
|
+
async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
2020
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
2021
|
+
const startedAt = nowIso();
|
|
2022
|
+
let stdout = "";
|
|
2023
|
+
let stderr = "";
|
|
2024
|
+
let timedOut = false;
|
|
2025
|
+
let exitCode;
|
|
2026
|
+
let error;
|
|
2027
|
+
let plan;
|
|
2028
|
+
let script;
|
|
2029
|
+
try {
|
|
2030
|
+
plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
2031
|
+
script = remoteScript(spec, metadata);
|
|
2032
|
+
} catch (err) {
|
|
2033
|
+
return {
|
|
2034
|
+
status: "failed",
|
|
2035
|
+
stdout: "",
|
|
2036
|
+
stderr: "",
|
|
2037
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2038
|
+
startedAt,
|
|
2039
|
+
finishedAt: nowIso(),
|
|
2040
|
+
durationMs: 0
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
const child = spawn(plan.command, plan.args, {
|
|
2044
|
+
env: transportEnv(opts),
|
|
2045
|
+
detached: true,
|
|
2046
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2047
|
+
});
|
|
2048
|
+
if (child.pid)
|
|
2049
|
+
opts.onSpawn?.(child.pid);
|
|
2050
|
+
child.stdin?.on("error", (err) => {
|
|
2051
|
+
if (err.code !== "EPIPE")
|
|
2052
|
+
error = err.message;
|
|
2053
|
+
});
|
|
2054
|
+
child.stdin?.end(script);
|
|
2055
|
+
const abortHandler = () => {
|
|
2056
|
+
error = "cancelled";
|
|
2057
|
+
if (child.pid)
|
|
2058
|
+
killProcessGroup(child.pid);
|
|
2059
|
+
};
|
|
2060
|
+
if (opts.signal?.aborted)
|
|
2061
|
+
abortHandler();
|
|
2062
|
+
opts.signal?.addEventListener("abort", abortHandler, { once: true });
|
|
2063
|
+
child.stdout?.on("data", (chunk) => {
|
|
2064
|
+
stdout = appendBounded(stdout, chunk, maxOutputBytes);
|
|
2065
|
+
});
|
|
2066
|
+
child.stderr?.on("data", (chunk) => {
|
|
2067
|
+
stderr = appendBounded(stderr, chunk, maxOutputBytes);
|
|
2068
|
+
});
|
|
2069
|
+
const timer = setTimeout(() => {
|
|
2070
|
+
timedOut = true;
|
|
2071
|
+
if (child.pid)
|
|
2072
|
+
killProcessGroup(child.pid);
|
|
2073
|
+
}, spec.timeoutMs);
|
|
2074
|
+
timer.unref();
|
|
2075
|
+
try {
|
|
2076
|
+
const [code, signal] = await once(child, "exit");
|
|
2077
|
+
if (typeof code === "number")
|
|
2078
|
+
exitCode = code;
|
|
2079
|
+
if (signal)
|
|
2080
|
+
error = `terminated by ${signal}`;
|
|
2081
|
+
} catch (err) {
|
|
2082
|
+
error = err instanceof Error ? err.message : String(err);
|
|
2083
|
+
} finally {
|
|
2084
|
+
clearTimeout(timer);
|
|
2085
|
+
opts.signal?.removeEventListener("abort", abortHandler);
|
|
2086
|
+
}
|
|
2087
|
+
const finishedAt = nowIso();
|
|
2088
|
+
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
2089
|
+
if (timedOut) {
|
|
2090
|
+
return {
|
|
2091
|
+
status: "timed_out",
|
|
2092
|
+
exitCode,
|
|
2093
|
+
stdout,
|
|
2094
|
+
stderr,
|
|
2095
|
+
error: `timed out after ${spec.timeoutMs}ms`,
|
|
2096
|
+
pid: child.pid,
|
|
2097
|
+
startedAt,
|
|
2098
|
+
finishedAt,
|
|
2099
|
+
durationMs
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
if (error || exitCode !== 0) {
|
|
2103
|
+
return {
|
|
2104
|
+
status: "failed",
|
|
2105
|
+
exitCode,
|
|
2106
|
+
stdout,
|
|
2107
|
+
stderr,
|
|
2108
|
+
error: error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`,
|
|
2109
|
+
pid: child.pid,
|
|
2110
|
+
startedAt,
|
|
2111
|
+
finishedAt,
|
|
2112
|
+
durationMs
|
|
2113
|
+
};
|
|
2114
|
+
}
|
|
2115
|
+
return {
|
|
2116
|
+
status: "succeeded",
|
|
2117
|
+
exitCode,
|
|
2118
|
+
stdout,
|
|
2119
|
+
stderr,
|
|
2120
|
+
pid: child.pid,
|
|
2121
|
+
startedAt,
|
|
2122
|
+
finishedAt,
|
|
2123
|
+
durationMs
|
|
2124
|
+
};
|
|
2125
|
+
}
|
|
1846
2126
|
function preflightTarget(target, metadata = {}, opts = {}) {
|
|
1847
2127
|
const spec = commandSpec(target);
|
|
2128
|
+
const machine = resolvedMachine(opts);
|
|
2129
|
+
if (machine && !machine.local) {
|
|
2130
|
+
preflightRemoteSpec(spec, machine, metadata, opts);
|
|
2131
|
+
return {
|
|
2132
|
+
command: spec.command,
|
|
2133
|
+
accountProfile: spec.account?.profile,
|
|
2134
|
+
accountTool: spec.accountTool
|
|
2135
|
+
};
|
|
2136
|
+
}
|
|
1848
2137
|
const env = executionEnv(spec, metadata, opts);
|
|
1849
2138
|
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
1850
2139
|
throw new Error(commandNotFoundMessage(spec.command, env));
|
|
@@ -1857,6 +2146,9 @@ function preflightTarget(target, metadata = {}, opts = {}) {
|
|
|
1857
2146
|
}
|
|
1858
2147
|
async function executeTarget(target, metadata = {}, opts = {}) {
|
|
1859
2148
|
const spec = commandSpec(target);
|
|
2149
|
+
const machine = resolvedMachine(opts);
|
|
2150
|
+
if (machine && !machine.local)
|
|
2151
|
+
return executeRemoteSpec(spec, machine, metadata, opts);
|
|
1860
2152
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
1861
2153
|
const startedAt = nowIso();
|
|
1862
2154
|
let stdout = "";
|
|
@@ -1972,7 +2264,7 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
1972
2264
|
loopName: loop.name,
|
|
1973
2265
|
runId: run.id,
|
|
1974
2266
|
scheduledFor: run.scheduledFor
|
|
1975
|
-
}, opts);
|
|
2267
|
+
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
1976
2268
|
}
|
|
1977
2269
|
|
|
1978
2270
|
// src/lib/workflow-runner.ts
|
|
@@ -2074,6 +2366,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
2074
2366
|
try {
|
|
2075
2367
|
result = await executeTarget(targetWithStepAccount(step), metadata, {
|
|
2076
2368
|
...opts,
|
|
2369
|
+
machine: opts.machine ?? opts.loop?.machine,
|
|
2077
2370
|
signal: controller.signal,
|
|
2078
2371
|
onSpawn: (pid) => {
|
|
2079
2372
|
opts.beforePersist?.();
|
|
@@ -2473,7 +2766,7 @@ function loops(opts = {}) {
|
|
|
2473
2766
|
return new LoopsClient(opts);
|
|
2474
2767
|
}
|
|
2475
2768
|
// src/lib/doctor.ts
|
|
2476
|
-
import { spawnSync as
|
|
2769
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
2477
2770
|
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
2478
2771
|
|
|
2479
2772
|
// src/daemon/control.ts
|
|
@@ -2611,11 +2904,11 @@ var PROVIDER_COMMANDS = [
|
|
|
2611
2904
|
"codex"
|
|
2612
2905
|
];
|
|
2613
2906
|
function hasCommand(command) {
|
|
2614
|
-
const result =
|
|
2907
|
+
const result = spawnSync3("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
2615
2908
|
return (result.status ?? 1) === 0;
|
|
2616
2909
|
}
|
|
2617
2910
|
function commandVersion(command) {
|
|
2618
|
-
const result =
|
|
2911
|
+
const result = spawnSync3(command, ["--version"], {
|
|
2619
2912
|
encoding: "utf8",
|
|
2620
2913
|
stdio: ["ignore", "pipe", "pipe"]
|
|
2621
2914
|
});
|
|
@@ -2641,6 +2934,23 @@ function runDoctor(store) {
|
|
|
2641
2934
|
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
2642
2935
|
const accountsVersion = commandVersion("accounts");
|
|
2643
2936
|
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
2937
|
+
try {
|
|
2938
|
+
const machines = listOpenMachines();
|
|
2939
|
+
const local = machines.find((machine) => machine.local);
|
|
2940
|
+
checks.push({
|
|
2941
|
+
id: "machines",
|
|
2942
|
+
status: "ok",
|
|
2943
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
2944
|
+
detail: local ? `local=${local.id}` : undefined
|
|
2945
|
+
});
|
|
2946
|
+
} catch (error) {
|
|
2947
|
+
checks.push({
|
|
2948
|
+
id: "machines",
|
|
2949
|
+
status: "warn",
|
|
2950
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
2951
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
2952
|
+
});
|
|
2953
|
+
}
|
|
2644
2954
|
for (const command of PROVIDER_COMMANDS) {
|
|
2645
2955
|
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
2646
2956
|
}
|
|
@@ -2653,16 +2963,16 @@ function runDoctor(store) {
|
|
|
2653
2963
|
if (loop.target.type === "workflow") {
|
|
2654
2964
|
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
2655
2965
|
for (const step of workflowExecutionOrder(workflow)) {
|
|
2656
|
-
preflightTarget({ ...step.target, account: step.account ?? step.target.account, timeoutMs: step.timeoutMs ?? step.target.timeoutMs }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id });
|
|
2966
|
+
preflightTarget({ ...step.target, account: step.account ?? step.target.account, timeoutMs: step.timeoutMs ?? step.target.timeoutMs }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
2657
2967
|
}
|
|
2658
2968
|
} else {
|
|
2659
|
-
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name });
|
|
2969
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
2660
2970
|
}
|
|
2661
2971
|
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
2662
2972
|
} catch (error) {
|
|
2663
2973
|
checks.push({
|
|
2664
2974
|
id: `loop:${loop.id}:preflight`,
|
|
2665
|
-
status: "
|
|
2975
|
+
status: "fail",
|
|
2666
2976
|
message: `active loop target preflight failed: ${loop.name}`,
|
|
2667
2977
|
detail: error instanceof Error ? error.message : String(error)
|
|
2668
2978
|
});
|
|
@@ -2678,12 +2988,15 @@ export {
|
|
|
2678
2988
|
workflowBodyFromJson,
|
|
2679
2989
|
tick,
|
|
2680
2990
|
runDoctor,
|
|
2991
|
+
resolveLoopMachine,
|
|
2992
|
+
refreshLoopMachine,
|
|
2681
2993
|
preflightWorkflow,
|
|
2682
2994
|
preflightTarget,
|
|
2683
2995
|
parseDuration,
|
|
2684
2996
|
parseCron,
|
|
2685
2997
|
nextCronRun,
|
|
2686
2998
|
loops,
|
|
2999
|
+
listOpenMachines,
|
|
2687
3000
|
initialNextRun,
|
|
2688
3001
|
executeWorkflow,
|
|
2689
3002
|
executeTarget,
|
package/dist/lib/executor.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
import type { ExecutableTarget, ExecutorResult, Loop, LoopRun, PersistGuardOptions } from "../types.js";
|
|
1
|
+
import type { ExecutableTarget, ExecutorResult, Loop, LoopMachineRef, LoopRun, PersistGuardOptions } from "../types.js";
|
|
2
2
|
export interface ExecuteOptions extends PersistGuardOptions {
|
|
3
3
|
maxOutputBytes?: number;
|
|
4
4
|
env?: NodeJS.ProcessEnv;
|
|
5
5
|
log?: (message: string) => void;
|
|
6
6
|
signal?: AbortSignal;
|
|
7
7
|
onSpawn?: (pid: number) => void;
|
|
8
|
+
machine?: LoopMachineRef;
|
|
9
|
+
machineResolver?: (machine: LoopMachineRef) => LoopMachineRef;
|
|
10
|
+
machineCommandResolver?: (machineId: string, command: string) => MachineCommandPlan;
|
|
8
11
|
}
|
|
9
12
|
export interface ExecutionMetadata {
|
|
10
13
|
loopId?: string;
|
|
@@ -21,6 +24,12 @@ export interface PreflightResult {
|
|
|
21
24
|
accountProfile?: string;
|
|
22
25
|
accountTool?: string;
|
|
23
26
|
}
|
|
27
|
+
interface MachineCommandPlan {
|
|
28
|
+
command: string;
|
|
29
|
+
args: string[];
|
|
30
|
+
source: string;
|
|
31
|
+
}
|
|
24
32
|
export declare function preflightTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): PreflightResult;
|
|
25
33
|
export declare function executeTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): Promise<ExecutorResult>;
|
|
26
34
|
export declare function executeLoop(loop: Loop, run: LoopRun, opts?: ExecuteOptions): Promise<ExecutorResult>;
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { LoopMachineRef } from "../types.js";
|
|
2
|
+
export interface OpenMachineSummary {
|
|
3
|
+
id: string;
|
|
4
|
+
hostname?: string;
|
|
5
|
+
platform?: string;
|
|
6
|
+
user?: string;
|
|
7
|
+
workspacePath?: string;
|
|
8
|
+
route?: string;
|
|
9
|
+
local: boolean;
|
|
10
|
+
heartbeatStatus?: string;
|
|
11
|
+
tailscaleOnline?: boolean | null;
|
|
12
|
+
tags: string[];
|
|
13
|
+
}
|
|
14
|
+
export declare function listOpenMachines(): OpenMachineSummary[];
|
|
15
|
+
export declare function resolveLoopMachine(machineId: string): LoopMachineRef;
|
|
16
|
+
export declare function refreshLoopMachine(machine: LoopMachineRef): LoopMachineRef;
|
package/dist/lib/store.js
CHANGED
|
@@ -345,6 +345,7 @@ function rowToLoop(row) {
|
|
|
345
345
|
status: row.status,
|
|
346
346
|
schedule: JSON.parse(row.schedule_json),
|
|
347
347
|
target: JSON.parse(row.target_json),
|
|
348
|
+
machine: row.machine_json ? JSON.parse(row.machine_json) : undefined,
|
|
348
349
|
nextRunAt: row.next_run_at ?? undefined,
|
|
349
350
|
retryScheduledFor: row.retry_scheduled_for ?? undefined,
|
|
350
351
|
catchUp: row.catch_up,
|
|
@@ -487,6 +488,7 @@ class Store {
|
|
|
487
488
|
status TEXT NOT NULL,
|
|
488
489
|
schedule_json TEXT NOT NULL,
|
|
489
490
|
target_json TEXT NOT NULL,
|
|
491
|
+
machine_json TEXT,
|
|
490
492
|
next_run_at TEXT,
|
|
491
493
|
retry_scheduled_for TEXT,
|
|
492
494
|
catch_up TEXT NOT NULL,
|
|
@@ -608,10 +610,14 @@ class Store {
|
|
|
608
610
|
);
|
|
609
611
|
CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
|
|
610
612
|
`);
|
|
613
|
+
try {
|
|
614
|
+
this.db.query("ALTER TABLE loops ADD COLUMN machine_json TEXT").run();
|
|
615
|
+
} catch {}
|
|
611
616
|
try {
|
|
612
617
|
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
613
618
|
} catch {}
|
|
614
619
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
620
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
615
621
|
}
|
|
616
622
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
617
623
|
if (!opts.daemonLeaseId)
|
|
@@ -629,6 +635,7 @@ class Store {
|
|
|
629
635
|
status: "active",
|
|
630
636
|
schedule: input.schedule,
|
|
631
637
|
target: input.target,
|
|
638
|
+
machine: input.machine,
|
|
632
639
|
nextRunAt: initialNextRun(input.schedule, from),
|
|
633
640
|
catchUp: input.catchUp ?? "latest",
|
|
634
641
|
catchUpLimit: input.catchUpLimit ?? 50,
|
|
@@ -640,9 +647,9 @@ class Store {
|
|
|
640
647
|
createdAt: now,
|
|
641
648
|
updatedAt: now
|
|
642
649
|
};
|
|
643
|
-
this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, next_run_at, retry_scheduled_for,
|
|
650
|
+
this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, machine_json, next_run_at, retry_scheduled_for,
|
|
644
651
|
catch_up, catch_up_limit, overlap, max_attempts, retry_delay_ms, lease_ms, expires_at, created_at, updated_at)
|
|
645
|
-
VALUES ($id, $name, $description, $status, $schedule, $target, $nextRun, NULL, $catchUp, $catchUpLimit,
|
|
652
|
+
VALUES ($id, $name, $description, $status, $schedule, $target, $machine, $nextRun, NULL, $catchUp, $catchUpLimit,
|
|
646
653
|
$overlap, $maxAttempts, $retryDelay, $leaseMs, $expiresAt, $created, $updated)`).run({
|
|
647
654
|
$id: loop.id,
|
|
648
655
|
$name: loop.name,
|
|
@@ -650,6 +657,7 @@ class Store {
|
|
|
650
657
|
$status: loop.status,
|
|
651
658
|
$schedule: JSON.stringify(loop.schedule),
|
|
652
659
|
$target: JSON.stringify(loop.target),
|
|
660
|
+
$machine: loop.machine ? JSON.stringify(loop.machine) : null,
|
|
653
661
|
$nextRun: loop.nextRunAt ?? null,
|
|
654
662
|
$catchUp: loop.catchUp,
|
|
655
663
|
$catchUpLimit: loop.catchUpLimit,
|