@terreno/ai 57.10.1 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,252 @@
1
+ import {randomUUID} from "node:crypto";
2
+ import {hostname} from "node:os";
3
+ import {logger} from "@terreno/api";
4
+ import {type JobDocument, type JobsApp, tryGetJobsService} from "@terreno/jobs";
5
+ import {DateTime, Duration, type DurationLike} from "luxon";
6
+
7
+ import type {
8
+ HarnessLeaseSettings,
9
+ HarnessRunnableTask,
10
+ HarnessRunner,
11
+ HarnessRunnerContext,
12
+ } from "../../types/harness";
13
+ import {errorMessage, harnessError} from "../errors";
14
+
15
+ /** Job that runs one phase of one harness task. */
16
+ export const HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
17
+
18
+ /** Job statuses that still lead to a handler run; any other status will never claim. */
19
+ const IN_FLIGHT_JOB_STATUSES = new Set<JobDocument["status"]>(["pending", "running", "scheduled"]);
20
+
21
+ export interface JobsRunnerOptions {
22
+ /** How many runnable tasks one dispatch tick enqueues at most. */
23
+ dispatchBatchSize?: number;
24
+ /** How often a running phase renews its task lease, and how often expired leases are swept. */
25
+ heartbeatInterval?: DurationLike;
26
+ /** The registered `JobsApp`; the runner defines `terreno.harness.phase` on it. */
27
+ jobs: JobsApp;
28
+ /** How long a task lease lives without a heartbeat. */
29
+ leaseDuration?: DurationLike;
30
+ /** Runner instance id written into task leases. Defaults to `hostname:pid:uuid`. */
31
+ ownerId?: string;
32
+ /** How long the dispatcher sleeps between scans for runnable tasks. */
33
+ pollInterval?: DurationLike;
34
+ }
35
+
36
+ interface PhaseJobPayload {
37
+ taskId: string;
38
+ }
39
+
40
+ const parsePayload = (payload: unknown): PhaseJobPayload => {
41
+ const taskId = (payload as Partial<PhaseJobPayload> | null)?.taskId;
42
+ if (typeof taskId !== "string" || !taskId) {
43
+ throw harnessError({
44
+ detail: `${HARNESS_PHASE_JOB_NAME} payload needs a taskId string`,
45
+ kind: "invalidRequest",
46
+ });
47
+ }
48
+ return {taskId};
49
+ };
50
+
51
+ /** `taskId:phase:attempt:claims` — one key per runnable visit of a phase. */
52
+ const phaseJobKey = (task: HarnessRunnableTask): string =>
53
+ `${task.taskId}:${task.phase}:${task.attempt}:${task.claims}`;
54
+
55
+ /**
56
+ * Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
57
+ * work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
58
+ * runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
59
+ * handler claims its task under a fenced lease, runs one phase while heartbeating, and
60
+ * hands a still-running task back as `pending` for the next job. The task lease, not the
61
+ * job lock, decides who may commit: a duplicate or late job finds the task claimed and
62
+ * does nothing, and a stale lease holder's commit is rejected.
63
+ */
64
+ export class JobsRunner implements HarnessRunner {
65
+ private context: HarnessRunnerContext | undefined;
66
+ private readonly dispatchBatchSize: number;
67
+ private hasPendingWake = false;
68
+ private readonly inFlight = new Set<Promise<void>>();
69
+ private isStopping = false;
70
+ private readonly lease: HarnessLeaseSettings;
71
+ private loop: Promise<void> | undefined;
72
+ private nextSweepAt = DateTime.fromMillis(0);
73
+ private readonly pollIntervalMs: number;
74
+ private wakeSleeper: (() => void) | undefined;
75
+
76
+ constructor(options: JobsRunnerOptions) {
77
+ const duration = Duration.fromDurationLike(options.leaseDuration ?? {seconds: 30});
78
+ const heartbeat = Duration.fromDurationLike(options.heartbeatInterval ?? {seconds: 10});
79
+ if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
80
+ throw harnessError({
81
+ detail: "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration",
82
+ kind: "configInvalid",
83
+ });
84
+ }
85
+ this.lease = {
86
+ duration,
87
+ heartbeat,
88
+ owner: options.ownerId ?? `${hostname()}:${process.pid}:${randomUUID()}`,
89
+ };
90
+ this.pollIntervalMs = Duration.fromDurationLike(
91
+ options.pollInterval ?? {milliseconds: 500}
92
+ ).toMillis();
93
+ const dispatchBatchSize = options.dispatchBatchSize ?? 100;
94
+ if (!Number.isInteger(dispatchBatchSize) || dispatchBatchSize < 1) {
95
+ throw harnessError({
96
+ detail: "JobsRunner dispatchBatchSize must be a positive integer",
97
+ kind: "configInvalid",
98
+ });
99
+ }
100
+ this.dispatchBatchSize = dispatchBatchSize;
101
+ options.jobs.define(HARNESS_PHASE_JOB_NAME, {
102
+ handler: (payload) => this.runPhaseJob(payload),
103
+ });
104
+ }
105
+
106
+ /** Id written into every task lease this runner holds. */
107
+ get ownerId(): string {
108
+ return this.lease.owner;
109
+ }
110
+
111
+ async start(context: HarnessRunnerContext): Promise<void> {
112
+ if (this.loop !== undefined) {
113
+ throw harnessError({detail: "JobsRunner is already started", kind: "alreadyStarted"});
114
+ }
115
+ if (!tryGetJobsService()) {
116
+ throw harnessError({
117
+ detail:
118
+ "JobsRunner needs a registered JobsApp: register it and build the app before harness.start()",
119
+ kind: "configInvalid",
120
+ });
121
+ }
122
+ this.context = context;
123
+ this.isStopping = false;
124
+ this.nextSweepAt = DateTime.fromMillis(0);
125
+ this.loop = this.dispatchLoop();
126
+ }
127
+
128
+ /** Stop dispatching, then wait for every phase this instance is running. */
129
+ async stop(): Promise<void> {
130
+ if (this.loop === undefined) {
131
+ return;
132
+ }
133
+ this.isStopping = true;
134
+ this.wake();
135
+ await this.loop;
136
+ await Promise.allSettled([...this.inFlight]);
137
+ this.loop = undefined;
138
+ this.context = undefined;
139
+ }
140
+
141
+ /** Dispatch now instead of after the rest of the idle sleep. */
142
+ wake(): void {
143
+ if (this.wakeSleeper) {
144
+ this.wakeSleeper();
145
+ return;
146
+ }
147
+ this.hasPendingWake = true;
148
+ }
149
+
150
+ /** Job handler: claim the task, run one phase, and hand it back if it still runs. */
151
+ private async runPhaseJob(payload: unknown): Promise<void> {
152
+ const {taskId} = parsePayload(payload);
153
+ // Tracked from before the claim, so stop() also waits for a claim in flight.
154
+ const job = this.claimAndRunPhase(taskId);
155
+ this.inFlight.add(job);
156
+ try {
157
+ await job;
158
+ } finally {
159
+ this.inFlight.delete(job);
160
+ }
161
+ }
162
+
163
+ private async claimAndRunPhase(taskId: string): Promise<void> {
164
+ const context = this.context;
165
+ if (!context || this.isStopping) {
166
+ // Throwing makes the jobs worker retry later, on this instance or another one.
167
+ throw harnessError({
168
+ detail: `JobsRunner is not running; task ${taskId} stays runnable`,
169
+ kind: "runnerStopped",
170
+ });
171
+ }
172
+ const task = await context.claimTask(taskId, this.lease);
173
+ if (!task) {
174
+ // Another job claimed it first, or it stopped being runnable since dispatch.
175
+ return;
176
+ }
177
+ await context.runTask(task, this.lease, {maxPhases: 1});
178
+ // The next phase (or a retry that is already due) can be dispatched right away.
179
+ this.wake();
180
+ }
181
+
182
+ private async dispatchLoop(): Promise<void> {
183
+ while (!this.isStopping && this.context) {
184
+ await this.tick(this.context);
185
+ if (!this.isStopping) {
186
+ await this.sleep(this.pollIntervalMs);
187
+ }
188
+ }
189
+ }
190
+
191
+ /** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
192
+ private async tick(context: HarnessRunnerContext): Promise<void> {
193
+ try {
194
+ if (DateTime.now() >= this.nextSweepAt) {
195
+ this.nextSweepAt = DateTime.now().plus(this.lease.heartbeat);
196
+ await context.recoverExpired();
197
+ }
198
+ const runnable = await context.listRunnable(this.dispatchBatchSize);
199
+ for (const task of runnable) {
200
+ if (this.isStopping) {
201
+ return;
202
+ }
203
+ await this.dispatch(task);
204
+ }
205
+ } catch (error: unknown) {
206
+ logger.error(`JobsRunner dispatch failed: ${errorMessage(error)}`);
207
+ }
208
+ }
209
+
210
+ /**
211
+ * Enqueue the job for this runnable visit. When the job with that key already ended
212
+ * without claiming the task (its handler kept failing, or it ran where the task was not
213
+ * runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
214
+ * the task is not stranded. A newly created job is always in flight, so this ends.
215
+ */
216
+ private async dispatch(task: HarnessRunnableTask): Promise<void> {
217
+ const jobs = tryGetJobsService();
218
+ if (!jobs) {
219
+ throw harnessError({detail: "JobsApp is no longer registered", kind: "configInvalid"});
220
+ }
221
+ let idempotencyKey = phaseJobKey(task);
222
+ for (;;) {
223
+ const job = await jobs.enqueue({
224
+ idempotencyKey,
225
+ name: HARNESS_PHASE_JOB_NAME,
226
+ payload: {taskId: task.taskId} satisfies PhaseJobPayload,
227
+ });
228
+ if (IN_FLIGHT_JOB_STATUSES.has(job.status)) {
229
+ return;
230
+ }
231
+ idempotencyKey = `${phaseJobKey(task)}:${job._id}`;
232
+ }
233
+ }
234
+
235
+ private sleep(ms: number): Promise<void> {
236
+ if (this.hasPendingWake) {
237
+ this.hasPendingWake = false;
238
+ return Promise.resolve();
239
+ }
240
+ return new Promise((resolve) => {
241
+ const timer = setTimeout(() => {
242
+ this.wakeSleeper = undefined;
243
+ resolve();
244
+ }, ms);
245
+ this.wakeSleeper = () => {
246
+ clearTimeout(timer);
247
+ this.wakeSleeper = undefined;
248
+ resolve();
249
+ };
250
+ });
251
+ }
252
+ }
@@ -12,6 +12,7 @@ import type {
12
12
  HarnessCommit,
13
13
  HarnessLeaseSettings,
14
14
  HarnessRunAgentOptions,
15
+ HarnessRunnableTask,
15
16
  HarnessTaskDefinition,
16
17
  HarnessTaskDocument,
17
18
  HarnessTaskRuntime,
@@ -99,6 +100,42 @@ const registeredFilter = (
99
100
  return [...definitions.values()].map(({name, version}) => ({name, version}));
100
101
  };
101
102
 
103
+ /**
104
+ * Mongo filter for a registered task a runner may claim now: `pending` and due, or
105
+ * `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
106
+ */
107
+ const runnableTaskFilter = (
108
+ definitions: Map<string, HarnessTaskDefinition>,
109
+ now: DateTime
110
+ ): Record<string, unknown> => ({
111
+ $and: [
112
+ {$or: registeredFilter(definitions)},
113
+ {
114
+ $or: [
115
+ {
116
+ $or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now.toJSDate()}}],
117
+ status: HARNESS_TASK_STATUSES.pending,
118
+ },
119
+ // An event wait or sleep whose timeout passed resumes like a due retry.
120
+ {
121
+ status: HARNESS_TASK_STATUSES.waiting,
122
+ "waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
123
+ "waiting.timeoutAt": {$lte: now.toJSDate()},
124
+ },
125
+ ],
126
+ },
127
+ ],
128
+ // A task being aborted never starts another run.
129
+ "abortRequested.at": {$exists: false},
130
+ });
131
+
132
+ /** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
133
+ const claimUpdate = (lease: HarnessLeaseSettings): Record<string, unknown> => ({
134
+ $inc: {claims: 1},
135
+ $set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
136
+ $unset: {waiting: 1},
137
+ });
138
+
102
139
  /**
103
140
  * Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
104
141
  * an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
@@ -115,36 +152,78 @@ export const claimNextTask = async ({
115
152
  if (definitions.size === 0) {
116
153
  return null;
117
154
  }
118
- const registered = registeredFilter(definitions);
119
- const now = DateTime.now().toJSDate();
120
155
  return models.task.findOneAndUpdate(
156
+ runnableTaskFilter(definitions, DateTime.now()),
157
+ claimUpdate(lease),
158
+ {returnDocument: "after", sort: {created: 1}}
159
+ );
160
+ };
161
+
162
+ /**
163
+ * Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
164
+ * runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
165
+ */
166
+ export const claimTaskById = async ({
167
+ definitions,
168
+ lease,
169
+ models,
170
+ taskId,
171
+ }: {
172
+ definitions: Map<string, HarnessTaskDefinition>;
173
+ lease: HarnessLeaseSettings;
174
+ models: HarnessModels;
175
+ taskId: string;
176
+ }): Promise<HarnessTaskDocument | null> => {
177
+ if (definitions.size === 0 || !mongoose.isValidObjectId(taskId)) {
178
+ return null;
179
+ }
180
+ return models.task.findOneAndUpdate(
181
+ {...runnableTaskFilter(definitions, DateTime.now()), _id: toObjectId(taskId)},
182
+ claimUpdate(lease),
183
+ {returnDocument: "after"}
184
+ );
185
+ };
186
+
187
+ /** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
188
+ export const listRunnableTasks = async ({
189
+ definitions,
190
+ limit,
191
+ models,
192
+ }: {
193
+ definitions: Map<string, HarnessTaskDefinition>;
194
+ limit: number;
195
+ models: HarnessModels;
196
+ }): Promise<HarnessRunnableTask[]> => {
197
+ if (definitions.size === 0) {
198
+ return [];
199
+ }
200
+ const tasks = await models.task
201
+ .find(runnableTaskFilter(definitions, DateTime.now()))
202
+ .select({attempt: 1, claims: 1, phase: 1})
203
+ .sort({created: 1})
204
+ .limit(limit);
205
+ return tasks.map((task) => ({
206
+ attempt: task.attempt ?? 0,
207
+ claims: task.claims ?? 0,
208
+ phase: task.phase,
209
+ taskId: String(task._id),
210
+ }));
211
+ };
212
+
213
+ /**
214
+ * Give a task that is still `running` under `task`'s lease back to the runners as
215
+ * `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
216
+ */
217
+ const releaseTask = async (models: HarnessModels, task: HarnessTaskDocument): Promise<boolean> => {
218
+ const result = await models.task.updateOne(
121
219
  {
122
- $and: [
123
- {$or: registered},
124
- {
125
- $or: [
126
- {
127
- $or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now}}],
128
- status: HARNESS_TASK_STATUSES.pending,
129
- },
130
- // An event wait or sleep whose timeout passed resumes like a due retry.
131
- {
132
- status: HARNESS_TASK_STATUSES.waiting,
133
- "waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
134
- "waiting.timeoutAt": {$lte: now},
135
- },
136
- ],
137
- },
138
- ],
139
- // A task being aborted never starts another run.
140
- "abortRequested.at": {$exists: false},
141
- },
142
- {
143
- $set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
144
- $unset: {waiting: 1},
220
+ _id: task._id,
221
+ "lease.token": task.lease?.token ?? null,
222
+ status: HARNESS_TASK_STATUSES.running,
145
223
  },
146
- {returnDocument: "after", sort: {created: 1}}
224
+ {$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {lease: 1}}
147
225
  );
226
+ return result.modifiedCount > 0;
148
227
  };
149
228
 
150
229
  /**
@@ -219,10 +298,13 @@ const interruptionAction = (
219
298
  export const runClaimedTask = async ({
220
299
  engine,
221
300
  lease,
301
+ maxPhases,
222
302
  task,
223
303
  }: {
224
304
  engine: HarnessEngine;
225
305
  lease: HarnessLeaseSettings;
306
+ /** Hand the task back as `pending` once this many phases committed and it still runs. */
307
+ maxPhases?: number;
226
308
  task: HarnessTaskDocument;
227
309
  }): Promise<void> => {
228
310
  const definition = engine.definitions.get(taskDefinitionKey(task));
@@ -238,14 +320,20 @@ export const runClaimedTask = async ({
238
320
  engine.controllers.set(taskId, controller);
239
321
  try {
240
322
  let current = task;
241
- let isFirstPhase = true;
323
+ let phasesRun = 0;
242
324
  while (current.status === HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
325
+ if (maxPhases !== undefined && phasesRun >= maxPhases) {
326
+ if (!(await releaseTask(engine.models, current))) {
327
+ logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
328
+ }
329
+ return;
330
+ }
243
331
  // An abort from another process may have fenced the task since the last commit.
244
- if (!isFirstPhase && !(await holdsLease(engine.models, current))) {
332
+ if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
245
333
  logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
246
334
  return;
247
335
  }
248
- isFirstPhase = false;
336
+ phasesRun += 1;
249
337
  const result = await runPhase({controller, definition, engine, lease, task: current});
250
338
  if (!result) {
251
339
  return;
@@ -150,6 +150,8 @@ export interface HarnessTaskDocument extends mongoose.Document<mongoose.Types.Ob
150
150
  ancestorIds?: mongoose.Types.ObjectId[];
151
151
  attempt: number;
152
152
  background: boolean;
153
+ /** Times a runner claimed the task; numbers each runnable visit for job dispatch. */
154
+ claims?: number;
153
155
  created: Date;
154
156
  deleted: boolean;
155
157
  input?: unknown;
@@ -491,6 +493,20 @@ export interface HarnessLeaseSettings {
491
493
  owner: string;
492
494
  }
493
495
 
496
+ /** A task a runner may claim now, as listed for dispatch. */
497
+ export interface HarnessRunnableTask {
498
+ attempt: number;
499
+ /** Claims so far; changes every time the task becomes runnable again after a claim. */
500
+ claims: number;
501
+ phase: string;
502
+ taskId: string;
503
+ }
504
+
505
+ export interface HarnessRunTaskOptions {
506
+ /** Stop after this many phases and hand a still-running task back as `pending`. */
507
+ maxPhases?: number;
508
+ }
509
+
494
510
  /**
495
511
  * What the harness hands a runner: hold the owner lease, recover tasks whose lease
496
512
  * expired, claim one runnable task, then run it to a stop.
@@ -499,6 +515,10 @@ export interface HarnessRunnerContext {
499
515
  /** Take or renew the singleton `HarnessOwner` lease; false when another owner holds it. */
500
516
  acquireOwnerLease: (lease: HarnessLeaseSettings) => Promise<boolean>;
501
517
  claimNext: (lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
518
+ /** Claim one task by id when it is runnable now; null when it is not (or another runner won). */
519
+ claimTask: (taskId: string, lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
520
+ /** Oldest runnable tasks first, at most `limit`, without claiming them. */
521
+ listRunnable: (limit: number) => Promise<HarnessRunnableTask[]>;
502
522
  /**
503
523
  * Resume or park every `running` task whose lease expired. Returns how many were made
504
524
  * runnable again.
@@ -506,7 +526,11 @@ export interface HarnessRunnerContext {
506
526
  recoverExpired: () => Promise<number>;
507
527
  /** Give up the owner lease so a standby can take over without waiting for expiry. */
508
528
  releaseOwnerLease: (lease: HarnessLeaseSettings) => Promise<void>;
509
- runTask: (task: HarnessTaskDocument, lease: HarnessLeaseSettings) => Promise<void>;
529
+ runTask: (
530
+ task: HarnessTaskDocument,
531
+ lease: HarnessLeaseSettings,
532
+ options?: HarnessRunTaskOptions
533
+ ) => Promise<void>;
510
534
  }
511
535
 
512
536
  /** Decides who executes runnable tasks and when. */