@terreno/ai 57.10.1 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.runClaimedTask = exports.recoverExpiredTasks = exports.claimNextTask = void 0;
6
+ exports.runClaimedTask = exports.recoverExpiredTasks = exports.listRunnableTasks = exports.claimTaskById = exports.claimNextTask = void 0;
7
7
  const api_1 = require("@terreno/api");
8
8
  const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
@@ -42,6 +42,37 @@ const assertValidNext = (definition, next) => {
42
42
  const registeredFilter = (definitions) => {
43
43
  return [...definitions.values()].map(({ name, version }) => ({ name, version }));
44
44
  };
45
+ /**
46
+ * Mongo filter for a registered task a runner may claim now: `pending` and due, or
47
+ * `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
48
+ */
49
+ const runnableTaskFilter = (definitions, now) => ({
50
+ $and: [
51
+ { $or: registeredFilter(definitions) },
52
+ {
53
+ $or: [
54
+ {
55
+ $or: [{ runAt: { $exists: false } }, { runAt: null }, { runAt: { $lte: now.toJSDate() } }],
56
+ status: harness_1.HARNESS_TASK_STATUSES.pending,
57
+ },
58
+ // An event wait or sleep whose timeout passed resumes like a due retry.
59
+ {
60
+ status: harness_1.HARNESS_TASK_STATUSES.waiting,
61
+ "waiting.kind": { $in: [harness_1.HARNESS_WAIT_KINDS.event, harness_1.HARNESS_WAIT_KINDS.sleep] },
62
+ "waiting.timeoutAt": { $lte: now.toJSDate() },
63
+ },
64
+ ],
65
+ },
66
+ ],
67
+ // A task being aborted never starts another run.
68
+ "abortRequested.at": { $exists: false },
69
+ });
70
+ /** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
71
+ const claimUpdate = (lease) => ({
72
+ $inc: { claims: 1 },
73
+ $set: { lease: (0, commit_1.newTaskLease)(lease), status: harness_1.HARNESS_TASK_STATUSES.running },
74
+ $unset: { waiting: 1 },
75
+ });
45
76
  /**
46
77
  * Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
47
78
  * an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
@@ -50,34 +81,50 @@ const claimNextTask = async ({ definitions, lease, models, }) => {
50
81
  if (definitions.size === 0) {
51
82
  return null;
52
83
  }
53
- const registered = registeredFilter(definitions);
54
- const now = luxon_1.DateTime.now().toJSDate();
55
- return models.task.findOneAndUpdate({
56
- $and: [
57
- { $or: registered },
58
- {
59
- $or: [
60
- {
61
- $or: [{ runAt: { $exists: false } }, { runAt: null }, { runAt: { $lte: now } }],
62
- status: harness_1.HARNESS_TASK_STATUSES.pending,
63
- },
64
- // An event wait or sleep whose timeout passed resumes like a due retry.
65
- {
66
- status: harness_1.HARNESS_TASK_STATUSES.waiting,
67
- "waiting.kind": { $in: [harness_1.HARNESS_WAIT_KINDS.event, harness_1.HARNESS_WAIT_KINDS.sleep] },
68
- "waiting.timeoutAt": { $lte: now },
69
- },
70
- ],
71
- },
72
- ],
73
- // A task being aborted never starts another run.
74
- "abortRequested.at": { $exists: false },
75
- }, {
76
- $set: { lease: (0, commit_1.newTaskLease)(lease), status: harness_1.HARNESS_TASK_STATUSES.running },
77
- $unset: { waiting: 1 },
78
- }, { returnDocument: "after", sort: { created: 1 } });
84
+ return models.task.findOneAndUpdate(runnableTaskFilter(definitions, luxon_1.DateTime.now()), claimUpdate(lease), { returnDocument: "after", sort: { created: 1 } });
79
85
  };
80
86
  exports.claimNextTask = claimNextTask;
87
+ /**
88
+ * Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
89
+ * runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
90
+ */
91
+ const claimTaskById = async ({ definitions, lease, models, taskId, }) => {
92
+ if (definitions.size === 0 || !mongoose_1.default.isValidObjectId(taskId)) {
93
+ return null;
94
+ }
95
+ return models.task.findOneAndUpdate({ ...runnableTaskFilter(definitions, luxon_1.DateTime.now()), _id: toObjectId(taskId) }, claimUpdate(lease), { returnDocument: "after" });
96
+ };
97
+ exports.claimTaskById = claimTaskById;
98
+ /** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
99
+ const listRunnableTasks = async ({ definitions, limit, models, }) => {
100
+ if (definitions.size === 0) {
101
+ return [];
102
+ }
103
+ const tasks = await models.task
104
+ .find(runnableTaskFilter(definitions, luxon_1.DateTime.now()))
105
+ .select({ attempt: 1, claims: 1, phase: 1 })
106
+ .sort({ created: 1 })
107
+ .limit(limit);
108
+ return tasks.map((task) => ({
109
+ attempt: task.attempt ?? 0,
110
+ claims: task.claims ?? 0,
111
+ phase: task.phase,
112
+ taskId: String(task._id),
113
+ }));
114
+ };
115
+ exports.listRunnableTasks = listRunnableTasks;
116
+ /**
117
+ * Give a task that is still `running` under `task`'s lease back to the runners as
118
+ * `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
119
+ */
120
+ const releaseTask = async (models, task) => {
121
+ const result = await models.task.updateOne({
122
+ _id: task._id,
123
+ "lease.token": task.lease?.token ?? null,
124
+ status: harness_1.HARNESS_TASK_STATUSES.running,
125
+ }, { $set: { status: harness_1.HARNESS_TASK_STATUSES.pending }, $unset: { lease: 1 } });
126
+ return result.modifiedCount > 0;
127
+ };
81
128
  /**
82
129
  * Find `running` tasks whose lease expired (their runner died or froze mid-phase) and
83
130
  * resolve each by its current phase's `replay`: `safe` goes back to `pending` at the same
@@ -140,7 +187,7 @@ const interruptionAction = (definition, phase) => {
140
187
  * was lost, the task is left at its last checkpoint and expired-lease recovery decides
141
188
  * what happens next. A terminal task re-checks the task that owns it.
142
189
  */
143
- const runClaimedTask = async ({ engine, lease, task, }) => {
190
+ const runClaimedTask = async ({ engine, lease, maxPhases, task, }) => {
144
191
  const definition = engine.definitions.get((0, defineTask_1.taskDefinitionKey)(task));
145
192
  if (!definition) {
146
193
  throw (0, errors_1.harnessError)({
@@ -153,14 +200,20 @@ const runClaimedTask = async ({ engine, lease, task, }) => {
153
200
  engine.controllers.set(taskId, controller);
154
201
  try {
155
202
  let current = task;
156
- let isFirstPhase = true;
203
+ let phasesRun = 0;
157
204
  while (current.status === harness_1.HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
205
+ if (maxPhases !== undefined && phasesRun >= maxPhases) {
206
+ if (!(await releaseTask(engine.models, current))) {
207
+ api_1.logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
208
+ }
209
+ return;
210
+ }
158
211
  // An abort from another process may have fenced the task since the last commit.
159
- if (!isFirstPhase && !(await holdsLease(engine.models, current))) {
212
+ if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
160
213
  api_1.logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
161
214
  return;
162
215
  }
163
- isFirstPhase = false;
216
+ phasesRun += 1;
164
217
  const result = await runPhase({ controller, definition, engine, lease, task: current });
165
218
  if (!result) {
166
219
  return;
@@ -119,6 +119,8 @@ export interface HarnessTaskDocument extends mongoose.Document<mongoose.Types.Ob
119
119
  ancestorIds?: mongoose.Types.ObjectId[];
120
120
  attempt: number;
121
121
  background: boolean;
122
+ /** Times a runner claimed the task; numbers each runnable visit for job dispatch. */
123
+ claims?: number;
122
124
  created: Date;
123
125
  deleted: boolean;
124
126
  input?: unknown;
@@ -425,6 +427,18 @@ export interface HarnessLeaseSettings {
425
427
  /** Runner instance id written into every lease it holds. */
426
428
  owner: string;
427
429
  }
430
+ /** A task a runner may claim now, as listed for dispatch. */
431
+ export interface HarnessRunnableTask {
432
+ attempt: number;
433
+ /** Claims so far; changes every time the task becomes runnable again after a claim. */
434
+ claims: number;
435
+ phase: string;
436
+ taskId: string;
437
+ }
438
+ export interface HarnessRunTaskOptions {
439
+ /** Stop after this many phases and hand a still-running task back as `pending`. */
440
+ maxPhases?: number;
441
+ }
428
442
  /**
429
443
  * What the harness hands a runner: hold the owner lease, recover tasks whose lease
430
444
  * expired, claim one runnable task, then run it to a stop.
@@ -433,6 +447,10 @@ export interface HarnessRunnerContext {
433
447
  /** Take or renew the singleton `HarnessOwner` lease; false when another owner holds it. */
434
448
  acquireOwnerLease: (lease: HarnessLeaseSettings) => Promise<boolean>;
435
449
  claimNext: (lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
450
+ /** Claim one task by id when it is runnable now; null when it is not (or another runner won). */
451
+ claimTask: (taskId: string, lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
452
+ /** Oldest runnable tasks first, at most `limit`, without claiming them. */
453
+ listRunnable: (limit: number) => Promise<HarnessRunnableTask[]>;
436
454
  /**
437
455
  * Resume or park every `running` task whose lease expired. Returns how many were made
438
456
  * runnable again.
@@ -440,7 +458,7 @@ export interface HarnessRunnerContext {
440
458
  recoverExpired: () => Promise<number>;
441
459
  /** Give up the owner lease so a standby can take over without waiting for expiry. */
442
460
  releaseOwnerLease: (lease: HarnessLeaseSettings) => Promise<void>;
443
- runTask: (task: HarnessTaskDocument, lease: HarnessLeaseSettings) => Promise<void>;
461
+ runTask: (task: HarnessTaskDocument, lease: HarnessLeaseSettings, options?: HarnessRunTaskOptions) => Promise<void>;
444
462
  }
445
463
  /** Decides who executes runnable tasks and when. */
446
464
  export interface HarnessRunner {
package/package.json CHANGED
@@ -10,7 +10,7 @@
10
10
  "@langfuse/otel": "^5.0.1",
11
11
  "@opentelemetry/api": "^1.9.0",
12
12
  "@opentelemetry/sdk-node": "0.217.0",
13
- "@terreno/blocks": "57.10.1",
13
+ "@terreno/blocks": "57.11.0",
14
14
  "ai": "^6.0.97",
15
15
  "ajv": "8.20.0",
16
16
  "csv-parse": "^6.1.0",
@@ -22,9 +22,10 @@
22
22
  "description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
23
23
  "devDependencies": {
24
24
  "@biomejs/biome": "^2.3.6",
25
- "@terreno/admin-backend": "57.10.1",
26
- "@terreno/api": "57.10.1",
27
- "@terreno/test": "57.10.1",
25
+ "@terreno/admin-backend": "57.11.0",
26
+ "@terreno/api": "57.11.0",
27
+ "@terreno/jobs": "57.11.0",
28
+ "@terreno/test": "57.11.0",
28
29
  "@types/bun": "^1.4.0",
29
30
  "@types/express": "^5.0.6",
30
31
  "@types/luxon": "^3.7.1",
@@ -50,6 +51,12 @@
50
51
  "require": "./dist/harness/harness.js",
51
52
  "types": "./dist/harness/harness.d.ts"
52
53
  },
54
+ "./harness/jobsRunner": {
55
+ "default": "./dist/harness/runners/jobsRunner.js",
56
+ "import": "./dist/harness/runners/jobsRunner.js",
57
+ "require": "./dist/harness/runners/jobsRunner.js",
58
+ "types": "./dist/harness/runners/jobsRunner.d.ts"
59
+ },
53
60
  "./langfuseApp": {
54
61
  "default": "./dist/langfuseApp.js",
55
62
  "import": "./dist/langfuseApp.js",
@@ -81,12 +88,16 @@
81
88
  "name": "@terreno/ai",
82
89
  "peerDependencies": {
83
90
  "@ai-sdk/google-vertex": "^4.0.142",
84
- "@terreno/api": "57.10.1",
91
+ "@terreno/api": "57.11.0",
92
+ "@terreno/jobs": "57.11.0",
85
93
  "mongoose": "^8.0.0 || ^9.0.0"
86
94
  },
87
95
  "peerDependenciesMeta": {
88
96
  "@ai-sdk/google-vertex": {
89
97
  "optional": true
98
+ },
99
+ "@terreno/jobs": {
100
+ "optional": true
90
101
  }
91
102
  },
92
103
  "publishConfig": {
@@ -109,5 +120,5 @@
109
120
  "test:coverage": "bun run ../scripts/check-coverage.ts"
110
121
  },
111
122
  "types": "dist/index.d.ts",
112
- "version": "57.10.1"
123
+ "version": "57.11.0"
113
124
  }
@@ -0,0 +1,155 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+
3
+ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
4
+ import {registerObsSpan} from "../observability/local/models/obsSpan";
5
+ import {registerObsTrace} from "../observability/local/models/obsTrace";
6
+ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
7
+ import {type AnyHarnessTaskDefinition, defineTask, Harness, InProcessRunner} from "./harness";
8
+ import {registerHarnessOwner} from "./models/harnessOwner";
9
+ import {registerHarnessTask} from "./models/harnessTask";
10
+
11
+ const TaskModel = registerHarnessTask();
12
+ const OwnerModel = registerHarnessOwner();
13
+ let SpanModel: ObsSpanModel;
14
+ let TraceModel: ObsTraceModel;
15
+
16
+ const pause = (ms: number): Promise<void> => new Promise((done) => setTimeout(done, ms));
17
+
18
+ const openHarnesses: Harness[] = [];
19
+ const releases: Array<() => void> = [];
20
+
21
+ const openHarness = async ({
22
+ concurrency,
23
+ registry,
24
+ }: {
25
+ concurrency: number;
26
+ registry: ReadonlyArray<AnyHarnessTaskDefinition>;
27
+ }): Promise<Harness> => {
28
+ const harness = await Harness.open({
29
+ registry,
30
+ runner: new InProcessRunner({concurrency, pollInterval: {milliseconds: 20}}),
31
+ });
32
+ openHarnesses.push(harness);
33
+ await harness.start();
34
+ return harness;
35
+ };
36
+
37
+ describe("InProcessRunner concurrency against Mongo", () => {
38
+ beforeAll(() => {
39
+ createLocalObservabilityPlugin();
40
+ SpanModel = registerObsSpan();
41
+ TraceModel = registerObsTrace();
42
+ });
43
+
44
+ beforeEach(async () => {
45
+ await Promise.all([
46
+ TaskModel.deleteMany({}),
47
+ OwnerModel.deleteMany({}),
48
+ SpanModel.deleteMany({}),
49
+ TraceModel.deleteMany({}),
50
+ ]);
51
+ });
52
+
53
+ afterEach(async () => {
54
+ for (const release of releases.splice(0)) {
55
+ release();
56
+ }
57
+ await Promise.all(openHarnesses.splice(0).map((harness) => harness.stop()));
58
+ });
59
+
60
+ it("completes a second task while the first is blocked on a gate", async () => {
61
+ let releaseGate: () => void = () => {};
62
+ const gated = new Promise<void>((done) => {
63
+ releaseGate = done;
64
+ });
65
+ releases.push(releaseGate);
66
+ const definition = defineTask<{blocked: boolean}, Record<string, never>, {status: string}>({
67
+ initial: () => ({phase: "work", state: {}}),
68
+ name: "test.concurrencyGate",
69
+ phases: {
70
+ work: {
71
+ run: async (task, rt) => {
72
+ if (task.input.blocked) {
73
+ await gated;
74
+ }
75
+ await rt.commit({terminal: {result: {status: "done"}, status: "completed"}});
76
+ },
77
+ },
78
+ },
79
+ version: 1,
80
+ }) as unknown as AnyHarnessTaskDefinition;
81
+ const harness = await openHarness({concurrency: 2, registry: [definition]});
82
+
83
+ const blocked = await harness.createTask(definition as never, {blocked: true});
84
+ const free = await harness.createTask(definition as never, {blocked: false});
85
+ const freeDone = await harness.waitForTask(free._id, {timeout: {seconds: 5}});
86
+
87
+ expect(freeDone.status).toBe("completed");
88
+ expect((await TaskModel.findExactlyOne({_id: blocked._id})).status).toBe("running");
89
+ releaseGate();
90
+ expect((await harness.waitForTask(blocked._id, {timeout: {seconds: 5}})).status).toBe(
91
+ "completed"
92
+ );
93
+ });
94
+
95
+ it("never runs one task twice at once and never exceeds the concurrency", async () => {
96
+ const activeByTask = new Map<string, number>();
97
+ let active = 0;
98
+ let peak = 0;
99
+ let peakPerTask = 0;
100
+ const enter = (taskId: string): void => {
101
+ active += 1;
102
+ peak = Math.max(peak, active);
103
+ const count = (activeByTask.get(taskId) ?? 0) + 1;
104
+ activeByTask.set(taskId, count);
105
+ peakPerTask = Math.max(peakPerTask, count);
106
+ };
107
+ const leave = (taskId: string): void => {
108
+ active -= 1;
109
+ activeByTask.set(taskId, (activeByTask.get(taskId) ?? 1) - 1);
110
+ };
111
+ // The first phase holds until three run at once (or 2 s pass), so the peak does not
112
+ // depend on how fast CI Mongo claims.
113
+ const fullHouse = async (): Promise<void> => {
114
+ for (let i = 0; i < 200 && active < 3; i++) {
115
+ await pause(10);
116
+ }
117
+ };
118
+ const definition = defineTask<{n: number}, Record<string, never>, {status: string}>({
119
+ initial: () => ({phase: "first", state: {}}),
120
+ name: "test.concurrencyPeak",
121
+ phases: {
122
+ first: {
123
+ run: async (task, rt) => {
124
+ enter(task.id);
125
+ await fullHouse();
126
+ await pause(30);
127
+ leave(task.id);
128
+ await rt.commit({phase: "second"});
129
+ },
130
+ },
131
+ second: {
132
+ run: async (task, rt) => {
133
+ enter(task.id);
134
+ await pause(30);
135
+ leave(task.id);
136
+ await rt.commit({terminal: {result: {status: "done"}, status: "completed"}});
137
+ },
138
+ },
139
+ },
140
+ version: 1,
141
+ }) as unknown as AnyHarnessTaskDefinition;
142
+ const harness = await openHarness({concurrency: 3, registry: [definition]});
143
+
144
+ const created = await Promise.all(
145
+ [1, 2, 3, 4, 5, 6, 7].map((n) => harness.createTask(definition as never, {n}))
146
+ );
147
+ const done = await Promise.all(
148
+ created.map((task) => harness.waitForTask(task._id, {timeout: {seconds: 10}}))
149
+ );
150
+
151
+ expect(done.map((task) => task.status)).toEqual(Array(7).fill("completed"));
152
+ expect(peakPerTask).toBe(1);
153
+ expect(peak).toBe(3);
154
+ });
155
+ });
@@ -73,6 +73,12 @@ export const HARNESS_ERRORS = {
73
73
  status: 409,
74
74
  title: "requestId is already in use",
75
75
  },
76
+ /** A `JobsRunner` phase job ran on an instance whose harness is not started. */
77
+ runnerStopped: {
78
+ code: "harness-runner-stopped",
79
+ status: 503,
80
+ title: "Harness runner is not running",
81
+ },
76
82
  /** `retry` on a task whose abort is still running. */
77
83
  taskAborting: {code: "harness-task-aborting", status: 409, title: "Task is being aborted"},
78
84
  /** `resolveInterrupted` on a task that is not interrupted. */
@@ -577,21 +577,33 @@ describe("Harness event stream", () => {
577
577
  const who = frame.data.taskId === String(created._id) ? "parent" : "child";
578
578
  return `${who} ${frame.event} ${payload.status ?? payload.text ?? ""}`.trim();
579
579
  };
580
- expect(parentStream.frames.map(describe)).toEqual([
580
+ // Parent and child run concurrently, so only each task's own order is fixed.
581
+ const lines = parentStream.frames.map(describe);
582
+ expect(lines.filter((line) => line.startsWith("parent"))).toEqual([
581
583
  "parent task.status pending",
582
584
  "parent output parent starting",
583
- "child task.status pending",
584
585
  "parent task.status waiting",
586
+ // Each phase re-runs after its wait, so its output is sent again.
587
+ "parent output parent starting",
588
+ "parent task.status completed",
589
+ ]);
590
+ expect(lines.filter((line) => line.startsWith("child"))).toEqual([
591
+ "child task.status pending",
585
592
  "child output child 1 working",
586
593
  "child approval.requested pending",
587
594
  "child task.status waiting",
588
595
  "child approval.decided approved",
589
- // Each phase re-runs after its wait, so its output is sent again.
590
596
  "child output child 1 working",
591
597
  "child task.status completed",
592
- "parent output parent starting",
593
- "parent task.status completed",
594
598
  ]);
599
+ // The child is created inside the parent's first phase, and the parent resumes only
600
+ // after the child completes.
601
+ expect(lines.indexOf("child task.status pending")).toBeLessThan(
602
+ lines.indexOf("parent task.status waiting")
603
+ );
604
+ expect(lines.indexOf("child task.status completed")).toBeLessThan(
605
+ lines.lastIndexOf("parent output parent starting")
606
+ );
595
607
  // The child's stream is the same log narrowed to the child: no parent events.
596
608
  expect(childStream.frames.map(describe)).toEqual(
597
609
  parentStream.frames.map(describe).filter((line) => line.startsWith("child"))
@@ -69,7 +69,13 @@ import {
69
69
  splitRegistry,
70
70
  } from "./registry";
71
71
  import {InProcessRunner} from "./runners/inProcessRunner";
72
- import {claimNextTask, recoverExpiredTasks, runClaimedTask} from "./runtime";
72
+ import {
73
+ claimNextTask,
74
+ claimTaskById,
75
+ listRunnableTasks,
76
+ recoverExpiredTasks,
77
+ runClaimedTask,
78
+ } from "./runtime";
73
79
  import {sendEventRecords} from "./waits";
74
80
 
75
81
  export type {
@@ -121,8 +127,10 @@ export type {
121
127
  HarnessResolveInterruptedOptions,
122
128
  HarnessRetryPolicy,
123
129
  HarnessRunAgentOptions,
130
+ HarnessRunnableTask,
124
131
  HarnessRunner,
125
132
  HarnessRunnerContext,
133
+ HarnessRunTaskOptions,
126
134
  HarnessSection,
127
135
  HarnessSectionInput,
128
136
  HarnessSendEventOptions,
@@ -426,9 +434,12 @@ export class Harness {
426
434
  await this.runner.start({
427
435
  acquireOwnerLease: (lease) => acquireOwnerLease({lease, models, testHooks}),
428
436
  claimNext: (lease) => this.claimNextOrSweep(lease),
437
+ claimTask: (taskId, lease) => claimTaskById({definitions, lease, models, taskId}),
438
+ listRunnable: (limit) => listRunnableTasks({definitions, limit, models}),
429
439
  recoverExpired: () => recoverExpiredTasks(engine),
430
440
  releaseOwnerLease: (lease) => releaseOwnerLease({lease, models}),
431
- runTask: (task, lease) => runClaimedTask({engine, lease, task}),
441
+ runTask: (task, lease, options) =>
442
+ runClaimedTask({engine, lease, maxPhases: options?.maxPhases, task}),
432
443
  });
433
444
  }
434
445
 
@@ -450,7 +461,7 @@ export class Harness {
450
461
  return claimNextTask({definitions, lease, models});
451
462
  }
452
463
 
453
- /** Stop claiming work and wait for the phase in flight to settle. */
464
+ /** Stop claiming work and wait for every phase in flight to settle. */
454
465
  async stop(): Promise<void> {
455
466
  if (!this.isStarted) {
456
467
  return;
@@ -39,6 +39,11 @@ const harnessTaskSchema = new mongoose.Schema<HarnessTaskDocument, HarnessTaskMo
39
39
  description: "When true, the task outlives its owning conversation turn",
40
40
  type: Boolean,
41
41
  },
42
+ claims: {
43
+ default: 0,
44
+ description: "Times a runner claimed the task; numbers each runnable visit for job dispatch",
45
+ type: Number,
46
+ },
42
47
  eventSeq: {
43
48
  default: 0,
44
49
  description: "Events received by harness.sendEvent; numbers the task's inbox",
@@ -43,11 +43,12 @@ const waitUntil = async (predicate: () => Promise<boolean>, label: string): Prom
43
43
 
44
44
  const openHarness = async (
45
45
  registry: ReadonlyArray<AnyHarnessTaskDefinition>,
46
- testHooks?: HarnessTestHooks
46
+ testHooks?: HarnessTestHooks,
47
+ concurrency?: number
47
48
  ): Promise<Harness> => {
48
49
  const harness = await Harness.open({
49
50
  registry,
50
- runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
51
+ runner: new InProcessRunner({concurrency, pollInterval: {milliseconds: 20}}),
51
52
  testHooks,
52
53
  });
53
54
  openHarnesses.push(harness);
@@ -500,13 +501,12 @@ describe("Harness ownership tree", () => {
500
501
  policy: "failFast",
501
502
  runs,
502
503
  });
503
- const harness = await openHarness([
504
- parent,
505
- failing,
506
- asAny(waitingSibling),
507
- asAny(pendingSibling),
508
- asAny(grandchild),
509
- ]);
504
+ // One task at a time keeps the last sibling unstarted (pending) when the failure lands.
505
+ const harness = await openHarness(
506
+ [parent, failing, asAny(waitingSibling), asAny(pendingSibling), asAny(grandchild)],
507
+ undefined,
508
+ 1
509
+ );
510
510
  await harness.start();
511
511
 
512
512
  const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
@@ -838,10 +838,8 @@ describe("Harness leases and crash recovery", () => {
838
838
  });
839
839
 
840
840
  describe("owner lease", () => {
841
- it("never lets two live harnesses run tasks at the same time", async () => {
841
+ it("lets only the owner of two live harnesses run tasks, each task once", async () => {
842
842
  const activeByTask = new Map<string, number>();
843
- let maxConcurrent = 0;
844
- let active = 0;
845
843
  const ranBy: string[] = [];
846
844
  const counted = defineTask<unknown, unknown, unknown>({
847
845
  initial: () => ({phase: "work"}),
@@ -850,12 +848,9 @@ describe("Harness leases and crash recovery", () => {
850
848
  work: {
851
849
  replay: "safe",
852
850
  run: async (task, rt) => {
853
- active += 1;
854
851
  activeByTask.set(task.id, (activeByTask.get(task.id) ?? 0) + 1);
855
- maxConcurrent = Math.max(maxConcurrent, active);
856
852
  ranBy.push(String((await TaskModel.findExactlyOne({_id: task.id})).lease?.owner));
857
853
  await pause(15);
858
- active -= 1;
859
854
  await rt.commit({terminal: {status: "completed"}});
860
855
  },
861
856
  },
@@ -882,7 +877,7 @@ describe("Harness leases and crash recovery", () => {
882
877
  created.map((task) => a.harness.waitForTask(task._id, {timeout: {seconds: 5}}))
883
878
  );
884
879
 
885
- expect(maxConcurrent).toBe(1);
880
+ // Only the owner runs work (concurrently, up to its limit); each task runs once.
886
881
  expect([...activeByTask.values()]).toEqual([1, 1, 1, 1, 1, 1]);
887
882
  expect(new Set(ranBy).size).toBe(1);
888
883
  expect([a.runner.role, b.runner.role].sort()).toEqual(["owner", "standby"]);