@hasna/loops 0.3.9 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +165 -55
- package/dist/daemon/daemon.d.ts +3 -0
- package/dist/daemon/index.js +166 -56
- package/dist/index.js +103 -21
- package/dist/lib/scheduler.d.ts +10 -0
- package/dist/lib/store.d.ts +9 -0
- package/dist/lib/store.js +17 -21
- package/dist/sdk/index.js +103 -21
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -860,31 +860,23 @@ class Store {
|
|
|
860
860
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
861
861
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
862
862
|
`);
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
this.db.query("ALTER TABLE loop_runs ADD COLUMN goal_run_id TEXT").run();
|
|
871
|
-
} catch {}
|
|
872
|
-
try {
|
|
873
|
-
this.db.query("ALTER TABLE workflow_specs ADD COLUMN goal_json TEXT").run();
|
|
874
|
-
} catch {}
|
|
875
|
-
try {
|
|
876
|
-
this.db.query("ALTER TABLE workflow_runs ADD COLUMN goal_run_id TEXT").run();
|
|
877
|
-
} catch {}
|
|
878
|
-
try {
|
|
879
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
880
|
-
} catch {}
|
|
881
|
-
try {
|
|
882
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN goal_run_id TEXT").run();
|
|
883
|
-
} catch {}
|
|
863
|
+
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
864
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
865
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
866
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
867
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
868
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
869
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
884
870
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
885
871
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
886
872
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
887
873
|
}
|
|
874
|
+
addColumnIfMissing(table, column, definition) {
|
|
875
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
876
|
+
if (columns.some((c) => c.name === column))
|
|
877
|
+
return;
|
|
878
|
+
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
879
|
+
}
|
|
888
880
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
889
881
|
if (!opts.daemonLeaseId)
|
|
890
882
|
return;
|
|
@@ -1670,6 +1662,10 @@ class Store {
|
|
|
1670
1662
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1671
1663
|
return (row?.count ?? 0) > 0;
|
|
1672
1664
|
}
|
|
1665
|
+
hasRunningRunForSlot(loopId, scheduledFor) {
|
|
1666
|
+
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
1667
|
+
return (row?.count ?? 0) > 0;
|
|
1668
|
+
}
|
|
1673
1669
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1674
1670
|
const now = (opts.now ?? new Date).toISOString();
|
|
1675
1671
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
@@ -3719,6 +3715,92 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
3719
3715
|
deps.onRun?.(finalRun);
|
|
3720
3716
|
return finalRun;
|
|
3721
3717
|
}
|
|
3718
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
3719
|
+
const now = deps.now?.() ?? new Date;
|
|
3720
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
3721
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
3722
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
3723
|
+
return;
|
|
3724
|
+
let skipped;
|
|
3725
|
+
try {
|
|
3726
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
3727
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3728
|
+
});
|
|
3729
|
+
} catch (error) {
|
|
3730
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3731
|
+
return;
|
|
3732
|
+
throw error;
|
|
3733
|
+
}
|
|
3734
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
3735
|
+
deps.onRun?.(skipped);
|
|
3736
|
+
return skipped;
|
|
3737
|
+
}
|
|
3738
|
+
let claim;
|
|
3739
|
+
try {
|
|
3740
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3741
|
+
} catch (error) {
|
|
3742
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3743
|
+
return;
|
|
3744
|
+
throw error;
|
|
3745
|
+
}
|
|
3746
|
+
if (!claim)
|
|
3747
|
+
return;
|
|
3748
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
3749
|
+
deps.onRun?.(claim.run);
|
|
3750
|
+
return claim;
|
|
3751
|
+
}
|
|
3752
|
+
function claimDueRuns(deps) {
|
|
3753
|
+
const now = deps.now?.() ?? new Date;
|
|
3754
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3755
|
+
const recoveredByLoop = new Map;
|
|
3756
|
+
for (const run of recovered) {
|
|
3757
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
3758
|
+
}
|
|
3759
|
+
for (const runs of recoveredByLoop.values()) {
|
|
3760
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
3761
|
+
if (!loop)
|
|
3762
|
+
continue;
|
|
3763
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
3764
|
+
if (retryable) {
|
|
3765
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
3766
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3767
|
+
});
|
|
3768
|
+
continue;
|
|
3769
|
+
}
|
|
3770
|
+
for (const run of runs) {
|
|
3771
|
+
const current = deps.store.getLoop(run.loopId);
|
|
3772
|
+
if (current) {
|
|
3773
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
3774
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3775
|
+
});
|
|
3776
|
+
}
|
|
3777
|
+
}
|
|
3778
|
+
}
|
|
3779
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3780
|
+
const claims = [];
|
|
3781
|
+
const claimed = [];
|
|
3782
|
+
const skipped = [];
|
|
3783
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
3784
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
3785
|
+
if (claims.length >= maxClaims)
|
|
3786
|
+
break;
|
|
3787
|
+
const plan = dueSlots(loop, now);
|
|
3788
|
+
for (const slot of plan.slots) {
|
|
3789
|
+
if (claims.length >= maxClaims)
|
|
3790
|
+
break;
|
|
3791
|
+
const run = claimSlot(deps, loop, slot);
|
|
3792
|
+
if (!run)
|
|
3793
|
+
continue;
|
|
3794
|
+
if ("loop" in run) {
|
|
3795
|
+
claims.push(run);
|
|
3796
|
+
claimed.push(run.run);
|
|
3797
|
+
} else if (run.status === "skipped") {
|
|
3798
|
+
skipped.push(run);
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
}
|
|
3802
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
3803
|
+
}
|
|
3722
3804
|
async function tick(deps) {
|
|
3723
3805
|
const now = deps.now?.() ?? new Date;
|
|
3724
3806
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
@@ -3905,6 +3987,13 @@ function intervalFromEnv() {
|
|
|
3905
3987
|
const value = Number(raw);
|
|
3906
3988
|
return Number.isFinite(value) && value > 0 ? value : undefined;
|
|
3907
3989
|
}
|
|
3990
|
+
function concurrencyFromEnv() {
|
|
3991
|
+
const raw = process.env.LOOPS_DAEMON_CONCURRENCY;
|
|
3992
|
+
if (!raw)
|
|
3993
|
+
return;
|
|
3994
|
+
const value = Number(raw);
|
|
3995
|
+
return Number.isInteger(value) && value > 0 ? value : undefined;
|
|
3996
|
+
}
|
|
3908
3997
|
async function runDaemon(opts = {}) {
|
|
3909
3998
|
ensureDataDir();
|
|
3910
3999
|
const pidPath = opts.pidPath ?? pidFilePath();
|
|
@@ -3919,6 +4008,7 @@ async function runDaemon(opts = {}) {
|
|
|
3919
4008
|
const runnerId = `${hostname2()}:${process.pid}:${leaseId}`;
|
|
3920
4009
|
const intervalMs = opts.intervalMs ?? intervalFromEnv() ?? 1000;
|
|
3921
4010
|
const leaseTtlMs = opts.leaseTtlMs ?? Math.max(60000, intervalMs * 10);
|
|
4011
|
+
const concurrency = Math.max(1, opts.concurrency ?? concurrencyFromEnv() ?? 4);
|
|
3922
4012
|
const log = opts.log ?? ((message) => console.error(`[loops-daemon] ${message}`));
|
|
3923
4013
|
const lease = store.acquireDaemonLease({
|
|
3924
4014
|
id: leaseId,
|
|
@@ -3933,6 +4023,7 @@ async function runDaemon(opts = {}) {
|
|
|
3933
4023
|
let stopFlag = false;
|
|
3934
4024
|
let leaseLost = false;
|
|
3935
4025
|
const runAbort = new AbortController;
|
|
4026
|
+
const activeRuns = new Map;
|
|
3936
4027
|
const requestStop = (message) => {
|
|
3937
4028
|
stopFlag = true;
|
|
3938
4029
|
if (!runAbort.signal.aborted)
|
|
@@ -3956,6 +4047,48 @@ async function runDaemon(opts = {}) {
|
|
|
3956
4047
|
opts.signal?.addEventListener("abort", onSignal, { once: true });
|
|
3957
4048
|
process.on("SIGINT", onSignal);
|
|
3958
4049
|
process.on("SIGTERM", onSignal);
|
|
4050
|
+
const executeDaemonRun = async (claim) => {
|
|
4051
|
+
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
4052
|
+
const timer = setInterval(() => {
|
|
4053
|
+
try {
|
|
4054
|
+
ensureLease();
|
|
4055
|
+
} catch (err) {
|
|
4056
|
+
log(err instanceof Error ? err.message : String(err));
|
|
4057
|
+
}
|
|
4058
|
+
}, heartbeatMs);
|
|
4059
|
+
timer.unref();
|
|
4060
|
+
try {
|
|
4061
|
+
const finalRun = await executeClaimedRun({
|
|
4062
|
+
store,
|
|
4063
|
+
runnerId,
|
|
4064
|
+
loop: claim.loop,
|
|
4065
|
+
run: claim.run,
|
|
4066
|
+
daemonLeaseId: leaseId,
|
|
4067
|
+
beforeFinalize: () => ensureLease(),
|
|
4068
|
+
execute: opts.execute ?? ((loop, run) => executeLoopTarget(store, loop, run, {
|
|
4069
|
+
signal: runAbort.signal,
|
|
4070
|
+
beforePersist: () => ensureLease(),
|
|
4071
|
+
daemonLeaseId: leaseId,
|
|
4072
|
+
onSpawn: (pid) => {
|
|
4073
|
+
ensureLease();
|
|
4074
|
+
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
4075
|
+
}
|
|
4076
|
+
})),
|
|
4077
|
+
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
4078
|
+
});
|
|
4079
|
+
ensureLease();
|
|
4080
|
+
if (leaseLost)
|
|
4081
|
+
throw new Error("daemon lease lost during run");
|
|
4082
|
+
advanceLoop(store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: leaseId });
|
|
4083
|
+
log(`run ${finalRun.id} ${finalRun.status} loop=${claim.loop.id}`);
|
|
4084
|
+
} finally {
|
|
4085
|
+
clearInterval(timer);
|
|
4086
|
+
}
|
|
4087
|
+
};
|
|
4088
|
+
const startClaim = (claim) => {
|
|
4089
|
+
const task = executeDaemonRun(claim).catch((err) => log(`run ${claim.run.id} error: ${err instanceof Error ? err.message : String(err)}`)).finally(() => activeRuns.delete(claim.run.id));
|
|
4090
|
+
activeRuns.set(claim.run.id, task);
|
|
4091
|
+
};
|
|
3959
4092
|
try {
|
|
3960
4093
|
await runLoop({
|
|
3961
4094
|
intervalMs,
|
|
@@ -3964,49 +4097,26 @@ async function runDaemon(opts = {}) {
|
|
|
3964
4097
|
onTickError: (err) => log(`tick error: ${err instanceof Error ? err.message : String(err)}`),
|
|
3965
4098
|
tickFn: async () => {
|
|
3966
4099
|
ensureLease();
|
|
3967
|
-
const
|
|
4100
|
+
const available = Math.max(0, concurrency - activeRuns.size);
|
|
4101
|
+
const result = claimDueRuns({
|
|
3968
4102
|
store,
|
|
3969
4103
|
runnerId,
|
|
3970
4104
|
daemonLeaseId: leaseId,
|
|
3971
4105
|
beforeRun: () => ensureLease(),
|
|
3972
|
-
|
|
3973
|
-
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
3974
|
-
const timer = setInterval(() => {
|
|
3975
|
-
try {
|
|
3976
|
-
ensureLease();
|
|
3977
|
-
} catch (err) {
|
|
3978
|
-
log(err instanceof Error ? err.message : String(err));
|
|
3979
|
-
}
|
|
3980
|
-
}, heartbeatMs);
|
|
3981
|
-
timer.unref();
|
|
3982
|
-
try {
|
|
3983
|
-
const result2 = await executeLoopTarget(store, loop, run, {
|
|
3984
|
-
signal: runAbort.signal,
|
|
3985
|
-
beforePersist: () => ensureLease(),
|
|
3986
|
-
daemonLeaseId: leaseId,
|
|
3987
|
-
onSpawn: (pid) => {
|
|
3988
|
-
ensureLease();
|
|
3989
|
-
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
3990
|
-
}
|
|
3991
|
-
});
|
|
3992
|
-
ensureLease();
|
|
3993
|
-
if (leaseLost)
|
|
3994
|
-
throw new Error("daemon lease lost during run");
|
|
3995
|
-
return result2;
|
|
3996
|
-
} finally {
|
|
3997
|
-
clearInterval(timer);
|
|
3998
|
-
}
|
|
3999
|
-
},
|
|
4000
|
-
beforeFinalize: () => ensureLease(),
|
|
4001
|
-
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
4106
|
+
maxClaims: available
|
|
4002
4107
|
});
|
|
4003
|
-
const
|
|
4108
|
+
for (const claim of result.claims)
|
|
4109
|
+
startClaim(claim);
|
|
4110
|
+
const changed = result.claims.length + result.skipped.length + result.recovered.length + result.expired.length;
|
|
4004
4111
|
if (changed > 0) {
|
|
4005
|
-
log(`tick
|
|
4112
|
+
log(`tick claimed=${result.claims.length} active=${activeRuns.size} skipped=${result.skipped.length} recovered=${result.recovered.length} expired=${result.expired.length}`);
|
|
4006
4113
|
}
|
|
4007
4114
|
}
|
|
4008
4115
|
});
|
|
4009
4116
|
} finally {
|
|
4117
|
+
if (activeRuns.size > 0 && !runAbort.signal.aborted)
|
|
4118
|
+
runAbort.abort();
|
|
4119
|
+
await Promise.allSettled([...activeRuns.values()]);
|
|
4010
4120
|
opts.signal?.removeEventListener("abort", onSignal);
|
|
4011
4121
|
process.off("SIGINT", onSignal);
|
|
4012
4122
|
process.off("SIGTERM", onSignal);
|
|
@@ -4220,7 +4330,7 @@ function runDoctor(store) {
|
|
|
4220
4330
|
// package.json
|
|
4221
4331
|
var package_default = {
|
|
4222
4332
|
name: "@hasna/loops",
|
|
4223
|
-
version: "0.3.
|
|
4333
|
+
version: "0.3.11",
|
|
4224
4334
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4225
4335
|
type: "module",
|
|
4226
4336
|
main: "dist/index.js",
|
package/dist/daemon/daemon.d.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { Store } from "../lib/store.js";
|
|
2
|
+
import type { ExecutorResult, Loop, LoopRun } from "../types.js";
|
|
2
3
|
export interface RunDaemonOptions {
|
|
3
4
|
intervalMs?: number;
|
|
4
5
|
leaseTtlMs?: number;
|
|
6
|
+
concurrency?: number;
|
|
5
7
|
store?: Store;
|
|
6
8
|
pidPath?: string;
|
|
9
|
+
execute?: (loop: Loop, run: LoopRun) => Promise<ExecutorResult>;
|
|
7
10
|
shouldStop?: () => boolean;
|
|
8
11
|
sleep?: (ms: number) => Promise<void>;
|
|
9
12
|
log?: (message: string) => void;
|
package/dist/daemon/index.js
CHANGED
|
@@ -860,31 +860,23 @@ class Store {
|
|
|
860
860
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
861
861
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
862
862
|
`);
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
this.db.query("ALTER TABLE loop_runs ADD COLUMN goal_run_id TEXT").run();
|
|
871
|
-
} catch {}
|
|
872
|
-
try {
|
|
873
|
-
this.db.query("ALTER TABLE workflow_specs ADD COLUMN goal_json TEXT").run();
|
|
874
|
-
} catch {}
|
|
875
|
-
try {
|
|
876
|
-
this.db.query("ALTER TABLE workflow_runs ADD COLUMN goal_run_id TEXT").run();
|
|
877
|
-
} catch {}
|
|
878
|
-
try {
|
|
879
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
880
|
-
} catch {}
|
|
881
|
-
try {
|
|
882
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN goal_run_id TEXT").run();
|
|
883
|
-
} catch {}
|
|
863
|
+
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
864
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
865
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
866
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
867
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
868
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
869
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
884
870
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
885
871
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
886
872
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
887
873
|
}
|
|
874
|
+
addColumnIfMissing(table, column, definition) {
|
|
875
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
876
|
+
if (columns.some((c) => c.name === column))
|
|
877
|
+
return;
|
|
878
|
+
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
879
|
+
}
|
|
888
880
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
889
881
|
if (!opts.daemonLeaseId)
|
|
890
882
|
return;
|
|
@@ -1670,6 +1662,10 @@ class Store {
|
|
|
1670
1662
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1671
1663
|
return (row?.count ?? 0) > 0;
|
|
1672
1664
|
}
|
|
1665
|
+
hasRunningRunForSlot(loopId, scheduledFor) {
|
|
1666
|
+
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
1667
|
+
return (row?.count ?? 0) > 0;
|
|
1668
|
+
}
|
|
1673
1669
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1674
1670
|
const now = (opts.now ?? new Date).toISOString();
|
|
1675
1671
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
@@ -3614,6 +3610,92 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
3614
3610
|
deps.onRun?.(finalRun);
|
|
3615
3611
|
return finalRun;
|
|
3616
3612
|
}
|
|
3613
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
3614
|
+
const now = deps.now?.() ?? new Date;
|
|
3615
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
3616
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
3617
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
3618
|
+
return;
|
|
3619
|
+
let skipped;
|
|
3620
|
+
try {
|
|
3621
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
3622
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3623
|
+
});
|
|
3624
|
+
} catch (error) {
|
|
3625
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3626
|
+
return;
|
|
3627
|
+
throw error;
|
|
3628
|
+
}
|
|
3629
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
3630
|
+
deps.onRun?.(skipped);
|
|
3631
|
+
return skipped;
|
|
3632
|
+
}
|
|
3633
|
+
let claim;
|
|
3634
|
+
try {
|
|
3635
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3636
|
+
} catch (error) {
|
|
3637
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3638
|
+
return;
|
|
3639
|
+
throw error;
|
|
3640
|
+
}
|
|
3641
|
+
if (!claim)
|
|
3642
|
+
return;
|
|
3643
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
3644
|
+
deps.onRun?.(claim.run);
|
|
3645
|
+
return claim;
|
|
3646
|
+
}
|
|
3647
|
+
function claimDueRuns(deps) {
|
|
3648
|
+
const now = deps.now?.() ?? new Date;
|
|
3649
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3650
|
+
const recoveredByLoop = new Map;
|
|
3651
|
+
for (const run of recovered) {
|
|
3652
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
3653
|
+
}
|
|
3654
|
+
for (const runs of recoveredByLoop.values()) {
|
|
3655
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
3656
|
+
if (!loop)
|
|
3657
|
+
continue;
|
|
3658
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
3659
|
+
if (retryable) {
|
|
3660
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
3661
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3662
|
+
});
|
|
3663
|
+
continue;
|
|
3664
|
+
}
|
|
3665
|
+
for (const run of runs) {
|
|
3666
|
+
const current = deps.store.getLoop(run.loopId);
|
|
3667
|
+
if (current) {
|
|
3668
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
3669
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3670
|
+
});
|
|
3671
|
+
}
|
|
3672
|
+
}
|
|
3673
|
+
}
|
|
3674
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3675
|
+
const claims = [];
|
|
3676
|
+
const claimed = [];
|
|
3677
|
+
const skipped = [];
|
|
3678
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
3679
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
3680
|
+
if (claims.length >= maxClaims)
|
|
3681
|
+
break;
|
|
3682
|
+
const plan = dueSlots(loop, now);
|
|
3683
|
+
for (const slot of plan.slots) {
|
|
3684
|
+
if (claims.length >= maxClaims)
|
|
3685
|
+
break;
|
|
3686
|
+
const run = claimSlot(deps, loop, slot);
|
|
3687
|
+
if (!run)
|
|
3688
|
+
continue;
|
|
3689
|
+
if ("loop" in run) {
|
|
3690
|
+
claims.push(run);
|
|
3691
|
+
claimed.push(run.run);
|
|
3692
|
+
} else if (run.status === "skipped") {
|
|
3693
|
+
skipped.push(run);
|
|
3694
|
+
}
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
3698
|
+
}
|
|
3617
3699
|
async function tick(deps) {
|
|
3618
3700
|
const now = deps.now?.() ?? new Date;
|
|
3619
3701
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
@@ -3797,6 +3879,13 @@ function intervalFromEnv() {
|
|
|
3797
3879
|
const value = Number(raw);
|
|
3798
3880
|
return Number.isFinite(value) && value > 0 ? value : undefined;
|
|
3799
3881
|
}
|
|
3882
|
+
function concurrencyFromEnv() {
|
|
3883
|
+
const raw = process.env.LOOPS_DAEMON_CONCURRENCY;
|
|
3884
|
+
if (!raw)
|
|
3885
|
+
return;
|
|
3886
|
+
const value = Number(raw);
|
|
3887
|
+
return Number.isInteger(value) && value > 0 ? value : undefined;
|
|
3888
|
+
}
|
|
3800
3889
|
async function runDaemon(opts = {}) {
|
|
3801
3890
|
ensureDataDir();
|
|
3802
3891
|
const pidPath = opts.pidPath ?? pidFilePath();
|
|
@@ -3811,6 +3900,7 @@ async function runDaemon(opts = {}) {
|
|
|
3811
3900
|
const runnerId = `${hostname2()}:${process.pid}:${leaseId}`;
|
|
3812
3901
|
const intervalMs = opts.intervalMs ?? intervalFromEnv() ?? 1000;
|
|
3813
3902
|
const leaseTtlMs = opts.leaseTtlMs ?? Math.max(60000, intervalMs * 10);
|
|
3903
|
+
const concurrency = Math.max(1, opts.concurrency ?? concurrencyFromEnv() ?? 4);
|
|
3814
3904
|
const log = opts.log ?? ((message) => console.error(`[loops-daemon] ${message}`));
|
|
3815
3905
|
const lease = store.acquireDaemonLease({
|
|
3816
3906
|
id: leaseId,
|
|
@@ -3825,6 +3915,7 @@ async function runDaemon(opts = {}) {
|
|
|
3825
3915
|
let stopFlag = false;
|
|
3826
3916
|
let leaseLost = false;
|
|
3827
3917
|
const runAbort = new AbortController;
|
|
3918
|
+
const activeRuns = new Map;
|
|
3828
3919
|
const requestStop = (message) => {
|
|
3829
3920
|
stopFlag = true;
|
|
3830
3921
|
if (!runAbort.signal.aborted)
|
|
@@ -3848,6 +3939,48 @@ async function runDaemon(opts = {}) {
|
|
|
3848
3939
|
opts.signal?.addEventListener("abort", onSignal, { once: true });
|
|
3849
3940
|
process.on("SIGINT", onSignal);
|
|
3850
3941
|
process.on("SIGTERM", onSignal);
|
|
3942
|
+
const executeDaemonRun = async (claim) => {
|
|
3943
|
+
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
3944
|
+
const timer = setInterval(() => {
|
|
3945
|
+
try {
|
|
3946
|
+
ensureLease();
|
|
3947
|
+
} catch (err) {
|
|
3948
|
+
log(err instanceof Error ? err.message : String(err));
|
|
3949
|
+
}
|
|
3950
|
+
}, heartbeatMs);
|
|
3951
|
+
timer.unref();
|
|
3952
|
+
try {
|
|
3953
|
+
const finalRun = await executeClaimedRun({
|
|
3954
|
+
store,
|
|
3955
|
+
runnerId,
|
|
3956
|
+
loop: claim.loop,
|
|
3957
|
+
run: claim.run,
|
|
3958
|
+
daemonLeaseId: leaseId,
|
|
3959
|
+
beforeFinalize: () => ensureLease(),
|
|
3960
|
+
execute: opts.execute ?? ((loop, run) => executeLoopTarget(store, loop, run, {
|
|
3961
|
+
signal: runAbort.signal,
|
|
3962
|
+
beforePersist: () => ensureLease(),
|
|
3963
|
+
daemonLeaseId: leaseId,
|
|
3964
|
+
onSpawn: (pid) => {
|
|
3965
|
+
ensureLease();
|
|
3966
|
+
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
3967
|
+
}
|
|
3968
|
+
})),
|
|
3969
|
+
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
3970
|
+
});
|
|
3971
|
+
ensureLease();
|
|
3972
|
+
if (leaseLost)
|
|
3973
|
+
throw new Error("daemon lease lost during run");
|
|
3974
|
+
advanceLoop(store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: leaseId });
|
|
3975
|
+
log(`run ${finalRun.id} ${finalRun.status} loop=${claim.loop.id}`);
|
|
3976
|
+
} finally {
|
|
3977
|
+
clearInterval(timer);
|
|
3978
|
+
}
|
|
3979
|
+
};
|
|
3980
|
+
const startClaim = (claim) => {
|
|
3981
|
+
const task = executeDaemonRun(claim).catch((err) => log(`run ${claim.run.id} error: ${err instanceof Error ? err.message : String(err)}`)).finally(() => activeRuns.delete(claim.run.id));
|
|
3982
|
+
activeRuns.set(claim.run.id, task);
|
|
3983
|
+
};
|
|
3851
3984
|
try {
|
|
3852
3985
|
await runLoop({
|
|
3853
3986
|
intervalMs,
|
|
@@ -3856,49 +3989,26 @@ async function runDaemon(opts = {}) {
|
|
|
3856
3989
|
onTickError: (err) => log(`tick error: ${err instanceof Error ? err.message : String(err)}`),
|
|
3857
3990
|
tickFn: async () => {
|
|
3858
3991
|
ensureLease();
|
|
3859
|
-
const
|
|
3992
|
+
const available = Math.max(0, concurrency - activeRuns.size);
|
|
3993
|
+
const result = claimDueRuns({
|
|
3860
3994
|
store,
|
|
3861
3995
|
runnerId,
|
|
3862
3996
|
daemonLeaseId: leaseId,
|
|
3863
3997
|
beforeRun: () => ensureLease(),
|
|
3864
|
-
|
|
3865
|
-
const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
|
|
3866
|
-
const timer = setInterval(() => {
|
|
3867
|
-
try {
|
|
3868
|
-
ensureLease();
|
|
3869
|
-
} catch (err) {
|
|
3870
|
-
log(err instanceof Error ? err.message : String(err));
|
|
3871
|
-
}
|
|
3872
|
-
}, heartbeatMs);
|
|
3873
|
-
timer.unref();
|
|
3874
|
-
try {
|
|
3875
|
-
const result2 = await executeLoopTarget(store, loop, run, {
|
|
3876
|
-
signal: runAbort.signal,
|
|
3877
|
-
beforePersist: () => ensureLease(),
|
|
3878
|
-
daemonLeaseId: leaseId,
|
|
3879
|
-
onSpawn: (pid) => {
|
|
3880
|
-
ensureLease();
|
|
3881
|
-
store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
|
|
3882
|
-
}
|
|
3883
|
-
});
|
|
3884
|
-
ensureLease();
|
|
3885
|
-
if (leaseLost)
|
|
3886
|
-
throw new Error("daemon lease lost during run");
|
|
3887
|
-
return result2;
|
|
3888
|
-
} finally {
|
|
3889
|
-
clearInterval(timer);
|
|
3890
|
-
}
|
|
3891
|
-
},
|
|
3892
|
-
beforeFinalize: () => ensureLease(),
|
|
3893
|
-
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
3998
|
+
maxClaims: available
|
|
3894
3999
|
});
|
|
3895
|
-
const
|
|
4000
|
+
for (const claim of result.claims)
|
|
4001
|
+
startClaim(claim);
|
|
4002
|
+
const changed = result.claims.length + result.skipped.length + result.recovered.length + result.expired.length;
|
|
3896
4003
|
if (changed > 0) {
|
|
3897
|
-
log(`tick
|
|
4004
|
+
log(`tick claimed=${result.claims.length} active=${activeRuns.size} skipped=${result.skipped.length} recovered=${result.recovered.length} expired=${result.expired.length}`);
|
|
3898
4005
|
}
|
|
3899
4006
|
}
|
|
3900
4007
|
});
|
|
3901
4008
|
} finally {
|
|
4009
|
+
if (activeRuns.size > 0 && !runAbort.signal.aborted)
|
|
4010
|
+
runAbort.abort();
|
|
4011
|
+
await Promise.allSettled([...activeRuns.values()]);
|
|
3902
4012
|
opts.signal?.removeEventListener("abort", onSignal);
|
|
3903
4013
|
process.off("SIGINT", onSignal);
|
|
3904
4014
|
process.off("SIGTERM", onSignal);
|
|
@@ -4020,7 +4130,7 @@ function enableStartup(result) {
|
|
|
4020
4130
|
// package.json
|
|
4021
4131
|
var package_default = {
|
|
4022
4132
|
name: "@hasna/loops",
|
|
4023
|
-
version: "0.3.
|
|
4133
|
+
version: "0.3.11",
|
|
4024
4134
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4025
4135
|
type: "module",
|
|
4026
4136
|
main: "dist/index.js",
|
|
@@ -4110,7 +4220,7 @@ function packageVersion() {
|
|
|
4110
4220
|
// src/daemon/index.ts
|
|
4111
4221
|
var program = new Command;
|
|
4112
4222
|
program.name("loops-daemon").description("OpenLoops daemon helper").version(packageVersion());
|
|
4113
|
-
program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs }));
|
|
4223
|
+
program.command("run").option("--interval-ms <ms>", "tick interval", (value) => Number(value)).option("--concurrency <n>", "maximum loop runs to execute concurrently", (value) => Number(value)).action(async (opts) => runDaemon({ intervalMs: opts.intervalMs, concurrency: opts.concurrency }));
|
|
4114
4224
|
program.command("start").action(async () => {
|
|
4115
4225
|
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops-daemon", args: ["run"] });
|
|
4116
4226
|
console.log(JSON.stringify(result, null, 2));
|
package/dist/index.js
CHANGED
|
@@ -858,31 +858,23 @@ class Store {
|
|
|
858
858
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
859
859
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
860
860
|
`);
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
this.db.query("ALTER TABLE loop_runs ADD COLUMN goal_run_id TEXT").run();
|
|
869
|
-
} catch {}
|
|
870
|
-
try {
|
|
871
|
-
this.db.query("ALTER TABLE workflow_specs ADD COLUMN goal_json TEXT").run();
|
|
872
|
-
} catch {}
|
|
873
|
-
try {
|
|
874
|
-
this.db.query("ALTER TABLE workflow_runs ADD COLUMN goal_run_id TEXT").run();
|
|
875
|
-
} catch {}
|
|
876
|
-
try {
|
|
877
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
878
|
-
} catch {}
|
|
879
|
-
try {
|
|
880
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN goal_run_id TEXT").run();
|
|
881
|
-
} catch {}
|
|
861
|
+
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
862
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
863
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
864
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
865
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
866
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
867
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
882
868
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
883
869
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
884
870
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
885
871
|
}
|
|
872
|
+
addColumnIfMissing(table, column, definition) {
|
|
873
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
874
|
+
if (columns.some((c) => c.name === column))
|
|
875
|
+
return;
|
|
876
|
+
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
877
|
+
}
|
|
886
878
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
887
879
|
if (!opts.daemonLeaseId)
|
|
888
880
|
return;
|
|
@@ -1668,6 +1660,10 @@ class Store {
|
|
|
1668
1660
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1669
1661
|
return (row?.count ?? 0) > 0;
|
|
1670
1662
|
}
|
|
1663
|
+
hasRunningRunForSlot(loopId, scheduledFor) {
|
|
1664
|
+
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
1665
|
+
return (row?.count ?? 0) > 0;
|
|
1666
|
+
}
|
|
1671
1667
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1672
1668
|
const now = (opts.now ?? new Date).toISOString();
|
|
1673
1669
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
@@ -3604,6 +3600,92 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
3604
3600
|
deps.onRun?.(finalRun);
|
|
3605
3601
|
return finalRun;
|
|
3606
3602
|
}
|
|
3603
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
3604
|
+
const now = deps.now?.() ?? new Date;
|
|
3605
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
3606
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
3607
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
3608
|
+
return;
|
|
3609
|
+
let skipped;
|
|
3610
|
+
try {
|
|
3611
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
3612
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3613
|
+
});
|
|
3614
|
+
} catch (error) {
|
|
3615
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3616
|
+
return;
|
|
3617
|
+
throw error;
|
|
3618
|
+
}
|
|
3619
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
3620
|
+
deps.onRun?.(skipped);
|
|
3621
|
+
return skipped;
|
|
3622
|
+
}
|
|
3623
|
+
let claim;
|
|
3624
|
+
try {
|
|
3625
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3626
|
+
} catch (error) {
|
|
3627
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3628
|
+
return;
|
|
3629
|
+
throw error;
|
|
3630
|
+
}
|
|
3631
|
+
if (!claim)
|
|
3632
|
+
return;
|
|
3633
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
3634
|
+
deps.onRun?.(claim.run);
|
|
3635
|
+
return claim;
|
|
3636
|
+
}
|
|
3637
|
+
function claimDueRuns(deps) {
|
|
3638
|
+
const now = deps.now?.() ?? new Date;
|
|
3639
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3640
|
+
const recoveredByLoop = new Map;
|
|
3641
|
+
for (const run of recovered) {
|
|
3642
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
3643
|
+
}
|
|
3644
|
+
for (const runs of recoveredByLoop.values()) {
|
|
3645
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
3646
|
+
if (!loop)
|
|
3647
|
+
continue;
|
|
3648
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
3649
|
+
if (retryable) {
|
|
3650
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
3651
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3652
|
+
});
|
|
3653
|
+
continue;
|
|
3654
|
+
}
|
|
3655
|
+
for (const run of runs) {
|
|
3656
|
+
const current = deps.store.getLoop(run.loopId);
|
|
3657
|
+
if (current) {
|
|
3658
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
3659
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3660
|
+
});
|
|
3661
|
+
}
|
|
3662
|
+
}
|
|
3663
|
+
}
|
|
3664
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3665
|
+
const claims = [];
|
|
3666
|
+
const claimed = [];
|
|
3667
|
+
const skipped = [];
|
|
3668
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
3669
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
3670
|
+
if (claims.length >= maxClaims)
|
|
3671
|
+
break;
|
|
3672
|
+
const plan = dueSlots(loop, now);
|
|
3673
|
+
for (const slot of plan.slots) {
|
|
3674
|
+
if (claims.length >= maxClaims)
|
|
3675
|
+
break;
|
|
3676
|
+
const run = claimSlot(deps, loop, slot);
|
|
3677
|
+
if (!run)
|
|
3678
|
+
continue;
|
|
3679
|
+
if ("loop" in run) {
|
|
3680
|
+
claims.push(run);
|
|
3681
|
+
claimed.push(run.run);
|
|
3682
|
+
} else if (run.status === "skipped") {
|
|
3683
|
+
skipped.push(run);
|
|
3684
|
+
}
|
|
3685
|
+
}
|
|
3686
|
+
}
|
|
3687
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
3688
|
+
}
|
|
3607
3689
|
async function tick(deps) {
|
|
3608
3690
|
const now = deps.now?.() ?? new Date;
|
|
3609
3691
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
package/dist/lib/scheduler.d.ts
CHANGED
|
@@ -18,6 +18,13 @@ export interface TickResult {
|
|
|
18
18
|
recovered: LoopRun[];
|
|
19
19
|
expired: Loop[];
|
|
20
20
|
}
|
|
21
|
+
export interface ClaimedLoopRun {
|
|
22
|
+
loop: Loop;
|
|
23
|
+
run: LoopRun;
|
|
24
|
+
}
|
|
25
|
+
export interface ClaimDueRunsResult extends TickResult {
|
|
26
|
+
claims: ClaimedLoopRun[];
|
|
27
|
+
}
|
|
21
28
|
export declare function manualRunScheduledFor(loop: Loop, now?: Date): string;
|
|
22
29
|
export declare function shouldAdvanceManualRun(loop: Loop, scheduledFor: string, now?: Date): boolean;
|
|
23
30
|
export type ManualRunSource = "ad_hoc" | "due_slot" | "retry_slot";
|
|
@@ -36,4 +43,7 @@ export declare function executeClaimedRun(deps: {
|
|
|
36
43
|
execute?: (loop: Loop, run: LoopRun) => Promise<ExecutorResult>;
|
|
37
44
|
onError?: (loop: Loop, error: unknown) => void;
|
|
38
45
|
}): Promise<LoopRun>;
|
|
46
|
+
export declare function claimDueRuns(deps: SchedulerDeps & {
|
|
47
|
+
maxClaims?: number;
|
|
48
|
+
}): ClaimDueRunsResult;
|
|
39
49
|
export declare function tick(deps: SchedulerDeps): Promise<TickResult>;
|
package/dist/lib/store.d.ts
CHANGED
|
@@ -58,6 +58,14 @@ export declare class Store {
|
|
|
58
58
|
private db;
|
|
59
59
|
constructor(path?: string);
|
|
60
60
|
private migrate;
|
|
61
|
+
/**
|
|
62
|
+
* Add a column only if it does not already exist. Idempotent — avoids the
|
|
63
|
+
* "duplicate column name" error that SQLite logs (via libsqlite3, before any
|
|
64
|
+
* JS try/catch) when re-running an additive migration on a database that has
|
|
65
|
+
* already been upgraded. Table/column/definition come from hardcoded literals
|
|
66
|
+
* in {@link migrate}, never user input, so interpolation here is safe.
|
|
67
|
+
*/
|
|
68
|
+
private addColumnIfMissing;
|
|
61
69
|
private assertDaemonLeaseFence;
|
|
62
70
|
createLoop(input: CreateLoopInput, from?: Date): Loop;
|
|
63
71
|
getLoop(id: string): Loop | undefined;
|
|
@@ -130,6 +138,7 @@ export declare class Store {
|
|
|
130
138
|
appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>): WorkflowEvent;
|
|
131
139
|
listWorkflowEvents(workflowRunId: string, limit?: number): WorkflowEvent[];
|
|
132
140
|
hasRunningRun(loopId: string): boolean;
|
|
141
|
+
hasRunningRunForSlot(loopId: string, scheduledFor: string): boolean;
|
|
133
142
|
markRunPid(id: string, pid: number, claimedBy?: string, opts?: DaemonLeaseFence): LoopRun | undefined;
|
|
134
143
|
private hasLiveWorkflowStepProcesses;
|
|
135
144
|
createSkippedRun(loop: Loop, scheduledFor: string, reason: string, opts?: DaemonLeaseFence): LoopRun;
|
package/dist/lib/store.js
CHANGED
|
@@ -858,31 +858,23 @@ class Store {
|
|
|
858
858
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
859
859
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
860
860
|
`);
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
this.db.query("ALTER TABLE loop_runs ADD COLUMN goal_run_id TEXT").run();
|
|
869
|
-
} catch {}
|
|
870
|
-
try {
|
|
871
|
-
this.db.query("ALTER TABLE workflow_specs ADD COLUMN goal_json TEXT").run();
|
|
872
|
-
} catch {}
|
|
873
|
-
try {
|
|
874
|
-
this.db.query("ALTER TABLE workflow_runs ADD COLUMN goal_run_id TEXT").run();
|
|
875
|
-
} catch {}
|
|
876
|
-
try {
|
|
877
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
878
|
-
} catch {}
|
|
879
|
-
try {
|
|
880
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN goal_run_id TEXT").run();
|
|
881
|
-
} catch {}
|
|
861
|
+
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
862
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
863
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
864
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
865
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
866
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
867
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
882
868
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
883
869
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
884
870
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
885
871
|
}
|
|
872
|
+
addColumnIfMissing(table, column, definition) {
|
|
873
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
874
|
+
if (columns.some((c) => c.name === column))
|
|
875
|
+
return;
|
|
876
|
+
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
877
|
+
}
|
|
886
878
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
887
879
|
if (!opts.daemonLeaseId)
|
|
888
880
|
return;
|
|
@@ -1668,6 +1660,10 @@ class Store {
|
|
|
1668
1660
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1669
1661
|
return (row?.count ?? 0) > 0;
|
|
1670
1662
|
}
|
|
1663
|
+
hasRunningRunForSlot(loopId, scheduledFor) {
|
|
1664
|
+
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
1665
|
+
return (row?.count ?? 0) > 0;
|
|
1666
|
+
}
|
|
1671
1667
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1672
1668
|
const now = (opts.now ?? new Date).toISOString();
|
|
1673
1669
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
package/dist/sdk/index.js
CHANGED
|
@@ -858,31 +858,23 @@ class Store {
|
|
|
858
858
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
859
859
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
860
860
|
`);
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
this.db.query("ALTER TABLE loop_runs ADD COLUMN goal_run_id TEXT").run();
|
|
869
|
-
} catch {}
|
|
870
|
-
try {
|
|
871
|
-
this.db.query("ALTER TABLE workflow_specs ADD COLUMN goal_json TEXT").run();
|
|
872
|
-
} catch {}
|
|
873
|
-
try {
|
|
874
|
-
this.db.query("ALTER TABLE workflow_runs ADD COLUMN goal_run_id TEXT").run();
|
|
875
|
-
} catch {}
|
|
876
|
-
try {
|
|
877
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
|
|
878
|
-
} catch {}
|
|
879
|
-
try {
|
|
880
|
-
this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN goal_run_id TEXT").run();
|
|
881
|
-
} catch {}
|
|
861
|
+
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
862
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
863
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
864
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
865
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
866
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
867
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
882
868
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
883
869
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
884
870
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
885
871
|
}
|
|
872
|
+
addColumnIfMissing(table, column, definition) {
|
|
873
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
874
|
+
if (columns.some((c) => c.name === column))
|
|
875
|
+
return;
|
|
876
|
+
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
877
|
+
}
|
|
886
878
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
887
879
|
if (!opts.daemonLeaseId)
|
|
888
880
|
return;
|
|
@@ -1668,6 +1660,10 @@ class Store {
|
|
|
1668
1660
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
|
|
1669
1661
|
return (row?.count ?? 0) > 0;
|
|
1670
1662
|
}
|
|
1663
|
+
hasRunningRunForSlot(loopId, scheduledFor) {
|
|
1664
|
+
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
1665
|
+
return (row?.count ?? 0) > 0;
|
|
1666
|
+
}
|
|
1671
1667
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
1672
1668
|
const now = (opts.now ?? new Date).toISOString();
|
|
1673
1669
|
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
@@ -3604,6 +3600,92 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
3604
3600
|
deps.onRun?.(finalRun);
|
|
3605
3601
|
return finalRun;
|
|
3606
3602
|
}
|
|
3603
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
3604
|
+
const now = deps.now?.() ?? new Date;
|
|
3605
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
3606
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
3607
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
3608
|
+
return;
|
|
3609
|
+
let skipped;
|
|
3610
|
+
try {
|
|
3611
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
3612
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3613
|
+
});
|
|
3614
|
+
} catch (error) {
|
|
3615
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3616
|
+
return;
|
|
3617
|
+
throw error;
|
|
3618
|
+
}
|
|
3619
|
+
advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
|
|
3620
|
+
deps.onRun?.(skipped);
|
|
3621
|
+
return skipped;
|
|
3622
|
+
}
|
|
3623
|
+
let claim;
|
|
3624
|
+
try {
|
|
3625
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3626
|
+
} catch (error) {
|
|
3627
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
3628
|
+
return;
|
|
3629
|
+
throw error;
|
|
3630
|
+
}
|
|
3631
|
+
if (!claim)
|
|
3632
|
+
return;
|
|
3633
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
3634
|
+
deps.onRun?.(claim.run);
|
|
3635
|
+
return claim;
|
|
3636
|
+
}
|
|
3637
|
+
function claimDueRuns(deps) {
|
|
3638
|
+
const now = deps.now?.() ?? new Date;
|
|
3639
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3640
|
+
const recoveredByLoop = new Map;
|
|
3641
|
+
for (const run of recovered) {
|
|
3642
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
3643
|
+
}
|
|
3644
|
+
for (const runs of recoveredByLoop.values()) {
|
|
3645
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
3646
|
+
if (!loop)
|
|
3647
|
+
continue;
|
|
3648
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
3649
|
+
if (retryable) {
|
|
3650
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
3651
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3652
|
+
});
|
|
3653
|
+
continue;
|
|
3654
|
+
}
|
|
3655
|
+
for (const run of runs) {
|
|
3656
|
+
const current = deps.store.getLoop(run.loopId);
|
|
3657
|
+
if (current) {
|
|
3658
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
3659
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
3660
|
+
});
|
|
3661
|
+
}
|
|
3662
|
+
}
|
|
3663
|
+
}
|
|
3664
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
3665
|
+
const claims = [];
|
|
3666
|
+
const claimed = [];
|
|
3667
|
+
const skipped = [];
|
|
3668
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
3669
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
3670
|
+
if (claims.length >= maxClaims)
|
|
3671
|
+
break;
|
|
3672
|
+
const plan = dueSlots(loop, now);
|
|
3673
|
+
for (const slot of plan.slots) {
|
|
3674
|
+
if (claims.length >= maxClaims)
|
|
3675
|
+
break;
|
|
3676
|
+
const run = claimSlot(deps, loop, slot);
|
|
3677
|
+
if (!run)
|
|
3678
|
+
continue;
|
|
3679
|
+
if ("loop" in run) {
|
|
3680
|
+
claims.push(run);
|
|
3681
|
+
claimed.push(run.run);
|
|
3682
|
+
} else if (run.status === "skipped") {
|
|
3683
|
+
skipped.push(run);
|
|
3684
|
+
}
|
|
3685
|
+
}
|
|
3686
|
+
}
|
|
3687
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
3688
|
+
}
|
|
3607
3689
|
async function tick(deps) {
|
|
3608
3690
|
const now = deps.now?.() ?? new Date;
|
|
3609
3691
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|