@terreno/ai 57.10.1 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/harness/errors.d.ts +6 -0
- package/dist/harness/errors.js +6 -0
- package/dist/harness/harness.d.ts +2 -2
- package/dist/harness/harness.js +4 -2
- package/dist/harness/models/harnessTask.js +5 -0
- package/dist/harness/runners/inProcessRunner.d.ts +11 -4
- package/dist/harness/runners/inProcessRunner.js +44 -13
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +20 -2
- package/dist/harness/runtime.js +84 -31
- package/dist/types/harness.d.ts +19 -1
- package/package.json +17 -6
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/errors.ts +6 -0
- package/src/harness/events.test.ts +17 -5
- package/src/harness/harness.ts +14 -3
- package/src/harness/models/harnessTask.ts +5 -0
- package/src/harness/ownership.test.ts +9 -9
- package/src/harness/recovery.test.ts +2 -7
- package/src/harness/runners/inProcessRunner.test.ts +128 -0
- package/src/harness/runners/inProcessRunner.ts +54 -14
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +117 -29
- package/src/types/harness.ts +25 -1
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import {randomUUID} from "node:crypto";
|
|
2
|
+
import {hostname} from "node:os";
|
|
3
|
+
import {logger} from "@terreno/api";
|
|
4
|
+
import {type JobDocument, type JobsApp, tryGetJobsService} from "@terreno/jobs";
|
|
5
|
+
import {DateTime, Duration, type DurationLike} from "luxon";
|
|
6
|
+
|
|
7
|
+
import type {
|
|
8
|
+
HarnessLeaseSettings,
|
|
9
|
+
HarnessRunnableTask,
|
|
10
|
+
HarnessRunner,
|
|
11
|
+
HarnessRunnerContext,
|
|
12
|
+
} from "../../types/harness";
|
|
13
|
+
import {errorMessage, harnessError} from "../errors";
|
|
14
|
+
|
|
15
|
+
/** Job that runs one phase of one harness task. */
|
|
16
|
+
export const HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
|
|
17
|
+
|
|
18
|
+
/** Job statuses that still lead to a handler run; any other status will never claim. */
|
|
19
|
+
const IN_FLIGHT_JOB_STATUSES = new Set<JobDocument["status"]>(["pending", "running", "scheduled"]);
|
|
20
|
+
|
|
21
|
+
export interface JobsRunnerOptions {
|
|
22
|
+
/** How many runnable tasks one dispatch tick enqueues at most. */
|
|
23
|
+
dispatchBatchSize?: number;
|
|
24
|
+
/** How often a running phase renews its task lease, and how often expired leases are swept. */
|
|
25
|
+
heartbeatInterval?: DurationLike;
|
|
26
|
+
/** The registered `JobsApp`; the runner defines `terreno.harness.phase` on it. */
|
|
27
|
+
jobs: JobsApp;
|
|
28
|
+
/** How long a task lease lives without a heartbeat. */
|
|
29
|
+
leaseDuration?: DurationLike;
|
|
30
|
+
/** Runner instance id written into task leases. Defaults to `hostname:pid:uuid`. */
|
|
31
|
+
ownerId?: string;
|
|
32
|
+
/** How long the dispatcher sleeps between scans for runnable tasks. */
|
|
33
|
+
pollInterval?: DurationLike;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
interface PhaseJobPayload {
|
|
37
|
+
taskId: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const parsePayload = (payload: unknown): PhaseJobPayload => {
|
|
41
|
+
const taskId = (payload as Partial<PhaseJobPayload> | null)?.taskId;
|
|
42
|
+
if (typeof taskId !== "string" || !taskId) {
|
|
43
|
+
throw harnessError({
|
|
44
|
+
detail: `${HARNESS_PHASE_JOB_NAME} payload needs a taskId string`,
|
|
45
|
+
kind: "invalidRequest",
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return {taskId};
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** `taskId:phase:attempt:claims` — one key per runnable visit of a phase. */
|
|
52
|
+
const phaseJobKey = (task: HarnessRunnableTask): string =>
|
|
53
|
+
`${task.taskId}:${task.phase}:${task.attempt}:${task.claims}`;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
|
|
57
|
+
* work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
|
|
58
|
+
* runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
|
|
59
|
+
* handler claims its task under a fenced lease, runs one phase while heartbeating, and
|
|
60
|
+
* hands a still-running task back as `pending` for the next job. The task lease, not the
|
|
61
|
+
* job lock, decides who may commit: a duplicate or late job finds the task claimed and
|
|
62
|
+
* does nothing, and a stale lease holder's commit is rejected.
|
|
63
|
+
*/
|
|
64
|
+
export class JobsRunner implements HarnessRunner {
|
|
65
|
+
private context: HarnessRunnerContext | undefined;
|
|
66
|
+
private readonly dispatchBatchSize: number;
|
|
67
|
+
private hasPendingWake = false;
|
|
68
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
69
|
+
private isStopping = false;
|
|
70
|
+
private readonly lease: HarnessLeaseSettings;
|
|
71
|
+
private loop: Promise<void> | undefined;
|
|
72
|
+
private nextSweepAt = DateTime.fromMillis(0);
|
|
73
|
+
private readonly pollIntervalMs: number;
|
|
74
|
+
private wakeSleeper: (() => void) | undefined;
|
|
75
|
+
|
|
76
|
+
constructor(options: JobsRunnerOptions) {
|
|
77
|
+
const duration = Duration.fromDurationLike(options.leaseDuration ?? {seconds: 30});
|
|
78
|
+
const heartbeat = Duration.fromDurationLike(options.heartbeatInterval ?? {seconds: 10});
|
|
79
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
80
|
+
throw harnessError({
|
|
81
|
+
detail: "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
82
|
+
kind: "configInvalid",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
this.lease = {
|
|
86
|
+
duration,
|
|
87
|
+
heartbeat,
|
|
88
|
+
owner: options.ownerId ?? `${hostname()}:${process.pid}:${randomUUID()}`,
|
|
89
|
+
};
|
|
90
|
+
this.pollIntervalMs = Duration.fromDurationLike(
|
|
91
|
+
options.pollInterval ?? {milliseconds: 500}
|
|
92
|
+
).toMillis();
|
|
93
|
+
const dispatchBatchSize = options.dispatchBatchSize ?? 100;
|
|
94
|
+
if (!Number.isInteger(dispatchBatchSize) || dispatchBatchSize < 1) {
|
|
95
|
+
throw harnessError({
|
|
96
|
+
detail: "JobsRunner dispatchBatchSize must be a positive integer",
|
|
97
|
+
kind: "configInvalid",
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
this.dispatchBatchSize = dispatchBatchSize;
|
|
101
|
+
options.jobs.define(HARNESS_PHASE_JOB_NAME, {
|
|
102
|
+
handler: (payload) => this.runPhaseJob(payload),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Id written into every task lease this runner holds. */
|
|
107
|
+
get ownerId(): string {
|
|
108
|
+
return this.lease.owner;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async start(context: HarnessRunnerContext): Promise<void> {
|
|
112
|
+
if (this.loop !== undefined) {
|
|
113
|
+
throw harnessError({detail: "JobsRunner is already started", kind: "alreadyStarted"});
|
|
114
|
+
}
|
|
115
|
+
if (!tryGetJobsService()) {
|
|
116
|
+
throw harnessError({
|
|
117
|
+
detail:
|
|
118
|
+
"JobsRunner needs a registered JobsApp: register it and build the app before harness.start()",
|
|
119
|
+
kind: "configInvalid",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
this.context = context;
|
|
123
|
+
this.isStopping = false;
|
|
124
|
+
this.nextSweepAt = DateTime.fromMillis(0);
|
|
125
|
+
this.loop = this.dispatchLoop();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Stop dispatching, then wait for every phase this instance is running. */
|
|
129
|
+
async stop(): Promise<void> {
|
|
130
|
+
if (this.loop === undefined) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
this.isStopping = true;
|
|
134
|
+
this.wake();
|
|
135
|
+
await this.loop;
|
|
136
|
+
await Promise.allSettled([...this.inFlight]);
|
|
137
|
+
this.loop = undefined;
|
|
138
|
+
this.context = undefined;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Dispatch now instead of after the rest of the idle sleep. */
|
|
142
|
+
wake(): void {
|
|
143
|
+
if (this.wakeSleeper) {
|
|
144
|
+
this.wakeSleeper();
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.hasPendingWake = true;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Job handler: claim the task, run one phase, and hand it back if it still runs. */
|
|
151
|
+
private async runPhaseJob(payload: unknown): Promise<void> {
|
|
152
|
+
const {taskId} = parsePayload(payload);
|
|
153
|
+
// Tracked from before the claim, so stop() also waits for a claim in flight.
|
|
154
|
+
const job = this.claimAndRunPhase(taskId);
|
|
155
|
+
this.inFlight.add(job);
|
|
156
|
+
try {
|
|
157
|
+
await job;
|
|
158
|
+
} finally {
|
|
159
|
+
this.inFlight.delete(job);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private async claimAndRunPhase(taskId: string): Promise<void> {
|
|
164
|
+
const context = this.context;
|
|
165
|
+
if (!context || this.isStopping) {
|
|
166
|
+
// Throwing makes the jobs worker retry later, on this instance or another one.
|
|
167
|
+
throw harnessError({
|
|
168
|
+
detail: `JobsRunner is not running; task ${taskId} stays runnable`,
|
|
169
|
+
kind: "runnerStopped",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const task = await context.claimTask(taskId, this.lease);
|
|
173
|
+
if (!task) {
|
|
174
|
+
// Another job claimed it first, or it stopped being runnable since dispatch.
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
await context.runTask(task, this.lease, {maxPhases: 1});
|
|
178
|
+
// The next phase (or a retry that is already due) can be dispatched right away.
|
|
179
|
+
this.wake();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private async dispatchLoop(): Promise<void> {
|
|
183
|
+
while (!this.isStopping && this.context) {
|
|
184
|
+
await this.tick(this.context);
|
|
185
|
+
if (!this.isStopping) {
|
|
186
|
+
await this.sleep(this.pollIntervalMs);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
|
|
192
|
+
private async tick(context: HarnessRunnerContext): Promise<void> {
|
|
193
|
+
try {
|
|
194
|
+
if (DateTime.now() >= this.nextSweepAt) {
|
|
195
|
+
this.nextSweepAt = DateTime.now().plus(this.lease.heartbeat);
|
|
196
|
+
await context.recoverExpired();
|
|
197
|
+
}
|
|
198
|
+
const runnable = await context.listRunnable(this.dispatchBatchSize);
|
|
199
|
+
for (const task of runnable) {
|
|
200
|
+
if (this.isStopping) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
await this.dispatch(task);
|
|
204
|
+
}
|
|
205
|
+
} catch (error: unknown) {
|
|
206
|
+
logger.error(`JobsRunner dispatch failed: ${errorMessage(error)}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Enqueue the job for this runnable visit. When the job with that key already ended
|
|
212
|
+
* without claiming the task (its handler kept failing, or it ran where the task was not
|
|
213
|
+
* runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
|
|
214
|
+
* the task is not stranded. A newly created job is always in flight, so this ends.
|
|
215
|
+
*/
|
|
216
|
+
private async dispatch(task: HarnessRunnableTask): Promise<void> {
|
|
217
|
+
const jobs = tryGetJobsService();
|
|
218
|
+
if (!jobs) {
|
|
219
|
+
throw harnessError({detail: "JobsApp is no longer registered", kind: "configInvalid"});
|
|
220
|
+
}
|
|
221
|
+
let idempotencyKey = phaseJobKey(task);
|
|
222
|
+
for (;;) {
|
|
223
|
+
const job = await jobs.enqueue({
|
|
224
|
+
idempotencyKey,
|
|
225
|
+
name: HARNESS_PHASE_JOB_NAME,
|
|
226
|
+
payload: {taskId: task.taskId} satisfies PhaseJobPayload,
|
|
227
|
+
});
|
|
228
|
+
if (IN_FLIGHT_JOB_STATUSES.has(job.status)) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
idempotencyKey = `${phaseJobKey(task)}:${job._id}`;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private sleep(ms: number): Promise<void> {
|
|
236
|
+
if (this.hasPendingWake) {
|
|
237
|
+
this.hasPendingWake = false;
|
|
238
|
+
return Promise.resolve();
|
|
239
|
+
}
|
|
240
|
+
return new Promise((resolve) => {
|
|
241
|
+
const timer = setTimeout(() => {
|
|
242
|
+
this.wakeSleeper = undefined;
|
|
243
|
+
resolve();
|
|
244
|
+
}, ms);
|
|
245
|
+
this.wakeSleeper = () => {
|
|
246
|
+
clearTimeout(timer);
|
|
247
|
+
this.wakeSleeper = undefined;
|
|
248
|
+
resolve();
|
|
249
|
+
};
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
package/src/harness/runtime.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
HarnessCommit,
|
|
13
13
|
HarnessLeaseSettings,
|
|
14
14
|
HarnessRunAgentOptions,
|
|
15
|
+
HarnessRunnableTask,
|
|
15
16
|
HarnessTaskDefinition,
|
|
16
17
|
HarnessTaskDocument,
|
|
17
18
|
HarnessTaskRuntime,
|
|
@@ -99,6 +100,42 @@ const registeredFilter = (
|
|
|
99
100
|
return [...definitions.values()].map(({name, version}) => ({name, version}));
|
|
100
101
|
};
|
|
101
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Mongo filter for a registered task a runner may claim now: `pending` and due, or
|
|
105
|
+
* `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
|
|
106
|
+
*/
|
|
107
|
+
const runnableTaskFilter = (
|
|
108
|
+
definitions: Map<string, HarnessTaskDefinition>,
|
|
109
|
+
now: DateTime
|
|
110
|
+
): Record<string, unknown> => ({
|
|
111
|
+
$and: [
|
|
112
|
+
{$or: registeredFilter(definitions)},
|
|
113
|
+
{
|
|
114
|
+
$or: [
|
|
115
|
+
{
|
|
116
|
+
$or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now.toJSDate()}}],
|
|
117
|
+
status: HARNESS_TASK_STATUSES.pending,
|
|
118
|
+
},
|
|
119
|
+
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
120
|
+
{
|
|
121
|
+
status: HARNESS_TASK_STATUSES.waiting,
|
|
122
|
+
"waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
|
|
123
|
+
"waiting.timeoutAt": {$lte: now.toJSDate()},
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
// A task being aborted never starts another run.
|
|
129
|
+
"abortRequested.at": {$exists: false},
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
/** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
|
|
133
|
+
const claimUpdate = (lease: HarnessLeaseSettings): Record<string, unknown> => ({
|
|
134
|
+
$inc: {claims: 1},
|
|
135
|
+
$set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
|
|
136
|
+
$unset: {waiting: 1},
|
|
137
|
+
});
|
|
138
|
+
|
|
102
139
|
/**
|
|
103
140
|
* Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
|
|
104
141
|
* an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
|
|
@@ -115,36 +152,78 @@ export const claimNextTask = async ({
|
|
|
115
152
|
if (definitions.size === 0) {
|
|
116
153
|
return null;
|
|
117
154
|
}
|
|
118
|
-
const registered = registeredFilter(definitions);
|
|
119
|
-
const now = DateTime.now().toJSDate();
|
|
120
155
|
return models.task.findOneAndUpdate(
|
|
156
|
+
runnableTaskFilter(definitions, DateTime.now()),
|
|
157
|
+
claimUpdate(lease),
|
|
158
|
+
{returnDocument: "after", sort: {created: 1}}
|
|
159
|
+
);
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
|
|
164
|
+
* runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
|
|
165
|
+
*/
|
|
166
|
+
export const claimTaskById = async ({
|
|
167
|
+
definitions,
|
|
168
|
+
lease,
|
|
169
|
+
models,
|
|
170
|
+
taskId,
|
|
171
|
+
}: {
|
|
172
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
173
|
+
lease: HarnessLeaseSettings;
|
|
174
|
+
models: HarnessModels;
|
|
175
|
+
taskId: string;
|
|
176
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
177
|
+
if (definitions.size === 0 || !mongoose.isValidObjectId(taskId)) {
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
return models.task.findOneAndUpdate(
|
|
181
|
+
{...runnableTaskFilter(definitions, DateTime.now()), _id: toObjectId(taskId)},
|
|
182
|
+
claimUpdate(lease),
|
|
183
|
+
{returnDocument: "after"}
|
|
184
|
+
);
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
|
|
188
|
+
export const listRunnableTasks = async ({
|
|
189
|
+
definitions,
|
|
190
|
+
limit,
|
|
191
|
+
models,
|
|
192
|
+
}: {
|
|
193
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
194
|
+
limit: number;
|
|
195
|
+
models: HarnessModels;
|
|
196
|
+
}): Promise<HarnessRunnableTask[]> => {
|
|
197
|
+
if (definitions.size === 0) {
|
|
198
|
+
return [];
|
|
199
|
+
}
|
|
200
|
+
const tasks = await models.task
|
|
201
|
+
.find(runnableTaskFilter(definitions, DateTime.now()))
|
|
202
|
+
.select({attempt: 1, claims: 1, phase: 1})
|
|
203
|
+
.sort({created: 1})
|
|
204
|
+
.limit(limit);
|
|
205
|
+
return tasks.map((task) => ({
|
|
206
|
+
attempt: task.attempt ?? 0,
|
|
207
|
+
claims: task.claims ?? 0,
|
|
208
|
+
phase: task.phase,
|
|
209
|
+
taskId: String(task._id),
|
|
210
|
+
}));
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Give a task that is still `running` under `task`'s lease back to the runners as
|
|
215
|
+
* `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
|
|
216
|
+
*/
|
|
217
|
+
const releaseTask = async (models: HarnessModels, task: HarnessTaskDocument): Promise<boolean> => {
|
|
218
|
+
const result = await models.task.updateOne(
|
|
121
219
|
{
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
$or: [
|
|
126
|
-
{
|
|
127
|
-
$or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now}}],
|
|
128
|
-
status: HARNESS_TASK_STATUSES.pending,
|
|
129
|
-
},
|
|
130
|
-
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
131
|
-
{
|
|
132
|
-
status: HARNESS_TASK_STATUSES.waiting,
|
|
133
|
-
"waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
|
|
134
|
-
"waiting.timeoutAt": {$lte: now},
|
|
135
|
-
},
|
|
136
|
-
],
|
|
137
|
-
},
|
|
138
|
-
],
|
|
139
|
-
// A task being aborted never starts another run.
|
|
140
|
-
"abortRequested.at": {$exists: false},
|
|
141
|
-
},
|
|
142
|
-
{
|
|
143
|
-
$set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
|
|
144
|
-
$unset: {waiting: 1},
|
|
220
|
+
_id: task._id,
|
|
221
|
+
"lease.token": task.lease?.token ?? null,
|
|
222
|
+
status: HARNESS_TASK_STATUSES.running,
|
|
145
223
|
},
|
|
146
|
-
{
|
|
224
|
+
{$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {lease: 1}}
|
|
147
225
|
);
|
|
226
|
+
return result.modifiedCount > 0;
|
|
148
227
|
};
|
|
149
228
|
|
|
150
229
|
/**
|
|
@@ -219,10 +298,13 @@ const interruptionAction = (
|
|
|
219
298
|
export const runClaimedTask = async ({
|
|
220
299
|
engine,
|
|
221
300
|
lease,
|
|
301
|
+
maxPhases,
|
|
222
302
|
task,
|
|
223
303
|
}: {
|
|
224
304
|
engine: HarnessEngine;
|
|
225
305
|
lease: HarnessLeaseSettings;
|
|
306
|
+
/** Hand the task back as `pending` once this many phases committed and it still runs. */
|
|
307
|
+
maxPhases?: number;
|
|
226
308
|
task: HarnessTaskDocument;
|
|
227
309
|
}): Promise<void> => {
|
|
228
310
|
const definition = engine.definitions.get(taskDefinitionKey(task));
|
|
@@ -238,14 +320,20 @@ export const runClaimedTask = async ({
|
|
|
238
320
|
engine.controllers.set(taskId, controller);
|
|
239
321
|
try {
|
|
240
322
|
let current = task;
|
|
241
|
-
let
|
|
323
|
+
let phasesRun = 0;
|
|
242
324
|
while (current.status === HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
|
|
325
|
+
if (maxPhases !== undefined && phasesRun >= maxPhases) {
|
|
326
|
+
if (!(await releaseTask(engine.models, current))) {
|
|
327
|
+
logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
|
|
328
|
+
}
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
243
331
|
// An abort from another process may have fenced the task since the last commit.
|
|
244
|
-
if (
|
|
332
|
+
if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
|
|
245
333
|
logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
|
|
246
334
|
return;
|
|
247
335
|
}
|
|
248
|
-
|
|
336
|
+
phasesRun += 1;
|
|
249
337
|
const result = await runPhase({controller, definition, engine, lease, task: current});
|
|
250
338
|
if (!result) {
|
|
251
339
|
return;
|
package/src/types/harness.ts
CHANGED
|
@@ -150,6 +150,8 @@ export interface HarnessTaskDocument extends mongoose.Document<mongoose.Types.Ob
|
|
|
150
150
|
ancestorIds?: mongoose.Types.ObjectId[];
|
|
151
151
|
attempt: number;
|
|
152
152
|
background: boolean;
|
|
153
|
+
/** Times a runner claimed the task; numbers each runnable visit for job dispatch. */
|
|
154
|
+
claims?: number;
|
|
153
155
|
created: Date;
|
|
154
156
|
deleted: boolean;
|
|
155
157
|
input?: unknown;
|
|
@@ -491,6 +493,20 @@ export interface HarnessLeaseSettings {
|
|
|
491
493
|
owner: string;
|
|
492
494
|
}
|
|
493
495
|
|
|
496
|
+
/** A task a runner may claim now, as listed for dispatch. */
|
|
497
|
+
export interface HarnessRunnableTask {
|
|
498
|
+
attempt: number;
|
|
499
|
+
/** Claims so far; changes every time the task becomes runnable again after a claim. */
|
|
500
|
+
claims: number;
|
|
501
|
+
phase: string;
|
|
502
|
+
taskId: string;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
export interface HarnessRunTaskOptions {
|
|
506
|
+
/** Stop after this many phases and hand a still-running task back as `pending`. */
|
|
507
|
+
maxPhases?: number;
|
|
508
|
+
}
|
|
509
|
+
|
|
494
510
|
/**
|
|
495
511
|
* What the harness hands a runner: hold the owner lease, recover tasks whose lease
|
|
496
512
|
* expired, claim one runnable task, then run it to a stop.
|
|
@@ -499,6 +515,10 @@ export interface HarnessRunnerContext {
|
|
|
499
515
|
/** Take or renew the singleton `HarnessOwner` lease; false when another owner holds it. */
|
|
500
516
|
acquireOwnerLease: (lease: HarnessLeaseSettings) => Promise<boolean>;
|
|
501
517
|
claimNext: (lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
|
|
518
|
+
/** Claim one task by id when it is runnable now; null when it is not (or another runner won). */
|
|
519
|
+
claimTask: (taskId: string, lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
|
|
520
|
+
/** Oldest runnable tasks first, at most `limit`, without claiming them. */
|
|
521
|
+
listRunnable: (limit: number) => Promise<HarnessRunnableTask[]>;
|
|
502
522
|
/**
|
|
503
523
|
* Resume or park every `running` task whose lease expired. Returns how many were made
|
|
504
524
|
* runnable again.
|
|
@@ -506,7 +526,11 @@ export interface HarnessRunnerContext {
|
|
|
506
526
|
recoverExpired: () => Promise<number>;
|
|
507
527
|
/** Give up the owner lease so a standby can take over without waiting for expiry. */
|
|
508
528
|
releaseOwnerLease: (lease: HarnessLeaseSettings) => Promise<void>;
|
|
509
|
-
runTask: (
|
|
529
|
+
runTask: (
|
|
530
|
+
task: HarnessTaskDocument,
|
|
531
|
+
lease: HarnessLeaseSettings,
|
|
532
|
+
options?: HarnessRunTaskOptions
|
|
533
|
+
) => Promise<void>;
|
|
510
534
|
}
|
|
511
535
|
|
|
512
536
|
/** Decides who executes runnable tasks and when. */
|