@terreno/ai 57.10.1 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/harness/errors.d.ts +6 -0
- package/dist/harness/errors.js +6 -0
- package/dist/harness/harness.d.ts +2 -2
- package/dist/harness/harness.js +4 -2
- package/dist/harness/models/harnessTask.js +5 -0
- package/dist/harness/runners/inProcessRunner.d.ts +11 -4
- package/dist/harness/runners/inProcessRunner.js +44 -13
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +20 -2
- package/dist/harness/runtime.js +84 -31
- package/dist/types/harness.d.ts +19 -1
- package/package.json +17 -6
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/errors.ts +6 -0
- package/src/harness/events.test.ts +17 -5
- package/src/harness/harness.ts +14 -3
- package/src/harness/models/harnessTask.ts +5 -0
- package/src/harness/ownership.test.ts +9 -9
- package/src/harness/recovery.test.ts +2 -7
- package/src/harness/runners/inProcessRunner.test.ts +128 -0
- package/src/harness/runners/inProcessRunner.ts +54 -14
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +117 -29
- package/src/types/harness.ts +25 -1
package/dist/harness/runtime.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.runClaimedTask = exports.recoverExpiredTasks = exports.claimNextTask = void 0;
|
|
6
|
+
exports.runClaimedTask = exports.recoverExpiredTasks = exports.listRunnableTasks = exports.claimTaskById = exports.claimNextTask = void 0;
|
|
7
7
|
const api_1 = require("@terreno/api");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
@@ -42,6 +42,37 @@ const assertValidNext = (definition, next) => {
|
|
|
42
42
|
const registeredFilter = (definitions) => {
|
|
43
43
|
return [...definitions.values()].map(({ name, version }) => ({ name, version }));
|
|
44
44
|
};
|
|
45
|
+
/**
|
|
46
|
+
* Mongo filter for a registered task a runner may claim now: `pending` and due, or
|
|
47
|
+
* `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
|
|
48
|
+
*/
|
|
49
|
+
const runnableTaskFilter = (definitions, now) => ({
|
|
50
|
+
$and: [
|
|
51
|
+
{ $or: registeredFilter(definitions) },
|
|
52
|
+
{
|
|
53
|
+
$or: [
|
|
54
|
+
{
|
|
55
|
+
$or: [{ runAt: { $exists: false } }, { runAt: null }, { runAt: { $lte: now.toJSDate() } }],
|
|
56
|
+
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
57
|
+
},
|
|
58
|
+
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
59
|
+
{
|
|
60
|
+
status: harness_1.HARNESS_TASK_STATUSES.waiting,
|
|
61
|
+
"waiting.kind": { $in: [harness_1.HARNESS_WAIT_KINDS.event, harness_1.HARNESS_WAIT_KINDS.sleep] },
|
|
62
|
+
"waiting.timeoutAt": { $lte: now.toJSDate() },
|
|
63
|
+
},
|
|
64
|
+
],
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
// A task being aborted never starts another run.
|
|
68
|
+
"abortRequested.at": { $exists: false },
|
|
69
|
+
});
|
|
70
|
+
/** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
|
|
71
|
+
const claimUpdate = (lease) => ({
|
|
72
|
+
$inc: { claims: 1 },
|
|
73
|
+
$set: { lease: (0, commit_1.newTaskLease)(lease), status: harness_1.HARNESS_TASK_STATUSES.running },
|
|
74
|
+
$unset: { waiting: 1 },
|
|
75
|
+
});
|
|
45
76
|
/**
|
|
46
77
|
* Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
|
|
47
78
|
* an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
|
|
@@ -50,34 +81,50 @@ const claimNextTask = async ({ definitions, lease, models, }) => {
|
|
|
50
81
|
if (definitions.size === 0) {
|
|
51
82
|
return null;
|
|
52
83
|
}
|
|
53
|
-
|
|
54
|
-
const now = luxon_1.DateTime.now().toJSDate();
|
|
55
|
-
return models.task.findOneAndUpdate({
|
|
56
|
-
$and: [
|
|
57
|
-
{ $or: registered },
|
|
58
|
-
{
|
|
59
|
-
$or: [
|
|
60
|
-
{
|
|
61
|
-
$or: [{ runAt: { $exists: false } }, { runAt: null }, { runAt: { $lte: now } }],
|
|
62
|
-
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
63
|
-
},
|
|
64
|
-
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
65
|
-
{
|
|
66
|
-
status: harness_1.HARNESS_TASK_STATUSES.waiting,
|
|
67
|
-
"waiting.kind": { $in: [harness_1.HARNESS_WAIT_KINDS.event, harness_1.HARNESS_WAIT_KINDS.sleep] },
|
|
68
|
-
"waiting.timeoutAt": { $lte: now },
|
|
69
|
-
},
|
|
70
|
-
],
|
|
71
|
-
},
|
|
72
|
-
],
|
|
73
|
-
// A task being aborted never starts another run.
|
|
74
|
-
"abortRequested.at": { $exists: false },
|
|
75
|
-
}, {
|
|
76
|
-
$set: { lease: (0, commit_1.newTaskLease)(lease), status: harness_1.HARNESS_TASK_STATUSES.running },
|
|
77
|
-
$unset: { waiting: 1 },
|
|
78
|
-
}, { returnDocument: "after", sort: { created: 1 } });
|
|
84
|
+
return models.task.findOneAndUpdate(runnableTaskFilter(definitions, luxon_1.DateTime.now()), claimUpdate(lease), { returnDocument: "after", sort: { created: 1 } });
|
|
79
85
|
};
|
|
80
86
|
exports.claimNextTask = claimNextTask;
|
|
87
|
+
/**
|
|
88
|
+
* Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
|
|
89
|
+
* runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
|
|
90
|
+
*/
|
|
91
|
+
const claimTaskById = async ({ definitions, lease, models, taskId, }) => {
|
|
92
|
+
if (definitions.size === 0 || !mongoose_1.default.isValidObjectId(taskId)) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
return models.task.findOneAndUpdate({ ...runnableTaskFilter(definitions, luxon_1.DateTime.now()), _id: toObjectId(taskId) }, claimUpdate(lease), { returnDocument: "after" });
|
|
96
|
+
};
|
|
97
|
+
exports.claimTaskById = claimTaskById;
|
|
98
|
+
/** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
|
|
99
|
+
const listRunnableTasks = async ({ definitions, limit, models, }) => {
|
|
100
|
+
if (definitions.size === 0) {
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
const tasks = await models.task
|
|
104
|
+
.find(runnableTaskFilter(definitions, luxon_1.DateTime.now()))
|
|
105
|
+
.select({ attempt: 1, claims: 1, phase: 1 })
|
|
106
|
+
.sort({ created: 1 })
|
|
107
|
+
.limit(limit);
|
|
108
|
+
return tasks.map((task) => ({
|
|
109
|
+
attempt: task.attempt ?? 0,
|
|
110
|
+
claims: task.claims ?? 0,
|
|
111
|
+
phase: task.phase,
|
|
112
|
+
taskId: String(task._id),
|
|
113
|
+
}));
|
|
114
|
+
};
|
|
115
|
+
exports.listRunnableTasks = listRunnableTasks;
|
|
116
|
+
/**
|
|
117
|
+
* Give a task that is still `running` under `task`'s lease back to the runners as
|
|
118
|
+
* `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
|
|
119
|
+
*/
|
|
120
|
+
const releaseTask = async (models, task) => {
|
|
121
|
+
const result = await models.task.updateOne({
|
|
122
|
+
_id: task._id,
|
|
123
|
+
"lease.token": task.lease?.token ?? null,
|
|
124
|
+
status: harness_1.HARNESS_TASK_STATUSES.running,
|
|
125
|
+
}, { $set: { status: harness_1.HARNESS_TASK_STATUSES.pending }, $unset: { lease: 1 } });
|
|
126
|
+
return result.modifiedCount > 0;
|
|
127
|
+
};
|
|
81
128
|
/**
|
|
82
129
|
* Find `running` tasks whose lease expired (their runner died or froze mid-phase) and
|
|
83
130
|
* resolve each by its current phase's `replay`: `safe` goes back to `pending` at the same
|
|
@@ -140,7 +187,7 @@ const interruptionAction = (definition, phase) => {
|
|
|
140
187
|
* was lost, the task is left at its last checkpoint and expired-lease recovery decides
|
|
141
188
|
* what happens next. A terminal task re-checks the task that owns it.
|
|
142
189
|
*/
|
|
143
|
-
const runClaimedTask = async ({ engine, lease, task, }) => {
|
|
190
|
+
const runClaimedTask = async ({ engine, lease, maxPhases, task, }) => {
|
|
144
191
|
const definition = engine.definitions.get((0, defineTask_1.taskDefinitionKey)(task));
|
|
145
192
|
if (!definition) {
|
|
146
193
|
throw (0, errors_1.harnessError)({
|
|
@@ -153,14 +200,20 @@ const runClaimedTask = async ({ engine, lease, task, }) => {
|
|
|
153
200
|
engine.controllers.set(taskId, controller);
|
|
154
201
|
try {
|
|
155
202
|
let current = task;
|
|
156
|
-
let
|
|
203
|
+
let phasesRun = 0;
|
|
157
204
|
while (current.status === harness_1.HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
|
|
205
|
+
if (maxPhases !== undefined && phasesRun >= maxPhases) {
|
|
206
|
+
if (!(await releaseTask(engine.models, current))) {
|
|
207
|
+
api_1.logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
|
|
208
|
+
}
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
158
211
|
// An abort from another process may have fenced the task since the last commit.
|
|
159
|
-
if (
|
|
212
|
+
if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
|
|
160
213
|
api_1.logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
|
|
161
214
|
return;
|
|
162
215
|
}
|
|
163
|
-
|
|
216
|
+
phasesRun += 1;
|
|
164
217
|
const result = await runPhase({ controller, definition, engine, lease, task: current });
|
|
165
218
|
if (!result) {
|
|
166
219
|
return;
|
package/dist/types/harness.d.ts
CHANGED
|
@@ -119,6 +119,8 @@ export interface HarnessTaskDocument extends mongoose.Document<mongoose.Types.Ob
|
|
|
119
119
|
ancestorIds?: mongoose.Types.ObjectId[];
|
|
120
120
|
attempt: number;
|
|
121
121
|
background: boolean;
|
|
122
|
+
/** Times a runner claimed the task; numbers each runnable visit for job dispatch. */
|
|
123
|
+
claims?: number;
|
|
122
124
|
created: Date;
|
|
123
125
|
deleted: boolean;
|
|
124
126
|
input?: unknown;
|
|
@@ -425,6 +427,18 @@ export interface HarnessLeaseSettings {
|
|
|
425
427
|
/** Runner instance id written into every lease it holds. */
|
|
426
428
|
owner: string;
|
|
427
429
|
}
|
|
430
|
+
/** A task a runner may claim now, as listed for dispatch. */
|
|
431
|
+
export interface HarnessRunnableTask {
|
|
432
|
+
attempt: number;
|
|
433
|
+
/** Claims so far; changes every time the task becomes runnable again after a claim. */
|
|
434
|
+
claims: number;
|
|
435
|
+
phase: string;
|
|
436
|
+
taskId: string;
|
|
437
|
+
}
|
|
438
|
+
export interface HarnessRunTaskOptions {
|
|
439
|
+
/** Stop after this many phases and hand a still-running task back as `pending`. */
|
|
440
|
+
maxPhases?: number;
|
|
441
|
+
}
|
|
428
442
|
/**
|
|
429
443
|
* What the harness hands a runner: hold the owner lease, recover tasks whose lease
|
|
430
444
|
* expired, claim one runnable task, then run it to a stop.
|
|
@@ -433,6 +447,10 @@ export interface HarnessRunnerContext {
|
|
|
433
447
|
/** Take or renew the singleton `HarnessOwner` lease; false when another owner holds it. */
|
|
434
448
|
acquireOwnerLease: (lease: HarnessLeaseSettings) => Promise<boolean>;
|
|
435
449
|
claimNext: (lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
|
|
450
|
+
/** Claim one task by id when it is runnable now; null when it is not (or another runner won). */
|
|
451
|
+
claimTask: (taskId: string, lease: HarnessLeaseSettings) => Promise<HarnessTaskDocument | null>;
|
|
452
|
+
/** Oldest runnable tasks first, at most `limit`, without claiming them. */
|
|
453
|
+
listRunnable: (limit: number) => Promise<HarnessRunnableTask[]>;
|
|
436
454
|
/**
|
|
437
455
|
* Resume or park every `running` task whose lease expired. Returns how many were made
|
|
438
456
|
* runnable again.
|
|
@@ -440,7 +458,7 @@ export interface HarnessRunnerContext {
|
|
|
440
458
|
recoverExpired: () => Promise<number>;
|
|
441
459
|
/** Give up the owner lease so a standby can take over without waiting for expiry. */
|
|
442
460
|
releaseOwnerLease: (lease: HarnessLeaseSettings) => Promise<void>;
|
|
443
|
-
runTask: (task: HarnessTaskDocument, lease: HarnessLeaseSettings) => Promise<void>;
|
|
461
|
+
runTask: (task: HarnessTaskDocument, lease: HarnessLeaseSettings, options?: HarnessRunTaskOptions) => Promise<void>;
|
|
444
462
|
}
|
|
445
463
|
/** Decides who executes runnable tasks and when. */
|
|
446
464
|
export interface HarnessRunner {
|
package/package.json
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"@langfuse/otel": "^5.0.1",
|
|
11
11
|
"@opentelemetry/api": "^1.9.0",
|
|
12
12
|
"@opentelemetry/sdk-node": "0.217.0",
|
|
13
|
-
"@terreno/blocks": "57.
|
|
13
|
+
"@terreno/blocks": "57.11.0",
|
|
14
14
|
"ai": "^6.0.97",
|
|
15
15
|
"ajv": "8.20.0",
|
|
16
16
|
"csv-parse": "^6.1.0",
|
|
@@ -22,9 +22,10 @@
|
|
|
22
22
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
23
23
|
"devDependencies": {
|
|
24
24
|
"@biomejs/biome": "^2.3.6",
|
|
25
|
-
"@terreno/admin-backend": "57.
|
|
26
|
-
"@terreno/api": "57.
|
|
27
|
-
"@terreno/
|
|
25
|
+
"@terreno/admin-backend": "57.11.0",
|
|
26
|
+
"@terreno/api": "57.11.0",
|
|
27
|
+
"@terreno/jobs": "57.11.0",
|
|
28
|
+
"@terreno/test": "57.11.0",
|
|
28
29
|
"@types/bun": "^1.4.0",
|
|
29
30
|
"@types/express": "^5.0.6",
|
|
30
31
|
"@types/luxon": "^3.7.1",
|
|
@@ -50,6 +51,12 @@
|
|
|
50
51
|
"require": "./dist/harness/harness.js",
|
|
51
52
|
"types": "./dist/harness/harness.d.ts"
|
|
52
53
|
},
|
|
54
|
+
"./harness/jobsRunner": {
|
|
55
|
+
"default": "./dist/harness/runners/jobsRunner.js",
|
|
56
|
+
"import": "./dist/harness/runners/jobsRunner.js",
|
|
57
|
+
"require": "./dist/harness/runners/jobsRunner.js",
|
|
58
|
+
"types": "./dist/harness/runners/jobsRunner.d.ts"
|
|
59
|
+
},
|
|
53
60
|
"./langfuseApp": {
|
|
54
61
|
"default": "./dist/langfuseApp.js",
|
|
55
62
|
"import": "./dist/langfuseApp.js",
|
|
@@ -81,12 +88,16 @@
|
|
|
81
88
|
"name": "@terreno/ai",
|
|
82
89
|
"peerDependencies": {
|
|
83
90
|
"@ai-sdk/google-vertex": "^4.0.142",
|
|
84
|
-
"@terreno/api": "57.
|
|
91
|
+
"@terreno/api": "57.11.0",
|
|
92
|
+
"@terreno/jobs": "57.11.0",
|
|
85
93
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
86
94
|
},
|
|
87
95
|
"peerDependenciesMeta": {
|
|
88
96
|
"@ai-sdk/google-vertex": {
|
|
89
97
|
"optional": true
|
|
98
|
+
},
|
|
99
|
+
"@terreno/jobs": {
|
|
100
|
+
"optional": true
|
|
90
101
|
}
|
|
91
102
|
},
|
|
92
103
|
"publishConfig": {
|
|
@@ -109,5 +120,5 @@
|
|
|
109
120
|
"test:coverage": "bun run ../scripts/check-coverage.ts"
|
|
110
121
|
},
|
|
111
122
|
"types": "dist/index.d.ts",
|
|
112
|
-
"version": "57.
|
|
123
|
+
"version": "57.11.0"
|
|
113
124
|
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
4
|
+
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
5
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
6
|
+
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
7
|
+
import {type AnyHarnessTaskDefinition, defineTask, Harness, InProcessRunner} from "./harness";
|
|
8
|
+
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
9
|
+
import {registerHarnessTask} from "./models/harnessTask";
|
|
10
|
+
|
|
11
|
+
const TaskModel = registerHarnessTask();
|
|
12
|
+
const OwnerModel = registerHarnessOwner();
|
|
13
|
+
let SpanModel: ObsSpanModel;
|
|
14
|
+
let TraceModel: ObsTraceModel;
|
|
15
|
+
|
|
16
|
+
const pause = (ms: number): Promise<void> => new Promise((done) => setTimeout(done, ms));
|
|
17
|
+
|
|
18
|
+
const openHarnesses: Harness[] = [];
|
|
19
|
+
const releases: Array<() => void> = [];
|
|
20
|
+
|
|
21
|
+
const openHarness = async ({
|
|
22
|
+
concurrency,
|
|
23
|
+
registry,
|
|
24
|
+
}: {
|
|
25
|
+
concurrency: number;
|
|
26
|
+
registry: ReadonlyArray<AnyHarnessTaskDefinition>;
|
|
27
|
+
}): Promise<Harness> => {
|
|
28
|
+
const harness = await Harness.open({
|
|
29
|
+
registry,
|
|
30
|
+
runner: new InProcessRunner({concurrency, pollInterval: {milliseconds: 20}}),
|
|
31
|
+
});
|
|
32
|
+
openHarnesses.push(harness);
|
|
33
|
+
await harness.start();
|
|
34
|
+
return harness;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
describe("InProcessRunner concurrency against Mongo", () => {
|
|
38
|
+
beforeAll(() => {
|
|
39
|
+
createLocalObservabilityPlugin();
|
|
40
|
+
SpanModel = registerObsSpan();
|
|
41
|
+
TraceModel = registerObsTrace();
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
beforeEach(async () => {
|
|
45
|
+
await Promise.all([
|
|
46
|
+
TaskModel.deleteMany({}),
|
|
47
|
+
OwnerModel.deleteMany({}),
|
|
48
|
+
SpanModel.deleteMany({}),
|
|
49
|
+
TraceModel.deleteMany({}),
|
|
50
|
+
]);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
afterEach(async () => {
|
|
54
|
+
for (const release of releases.splice(0)) {
|
|
55
|
+
release();
|
|
56
|
+
}
|
|
57
|
+
await Promise.all(openHarnesses.splice(0).map((harness) => harness.stop()));
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("completes a second task while the first is blocked on a gate", async () => {
|
|
61
|
+
let releaseGate: () => void = () => {};
|
|
62
|
+
const gated = new Promise<void>((done) => {
|
|
63
|
+
releaseGate = done;
|
|
64
|
+
});
|
|
65
|
+
releases.push(releaseGate);
|
|
66
|
+
const definition = defineTask<{blocked: boolean}, Record<string, never>, {status: string}>({
|
|
67
|
+
initial: () => ({phase: "work", state: {}}),
|
|
68
|
+
name: "test.concurrencyGate",
|
|
69
|
+
phases: {
|
|
70
|
+
work: {
|
|
71
|
+
run: async (task, rt) => {
|
|
72
|
+
if (task.input.blocked) {
|
|
73
|
+
await gated;
|
|
74
|
+
}
|
|
75
|
+
await rt.commit({terminal: {result: {status: "done"}, status: "completed"}});
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
version: 1,
|
|
80
|
+
}) as unknown as AnyHarnessTaskDefinition;
|
|
81
|
+
const harness = await openHarness({concurrency: 2, registry: [definition]});
|
|
82
|
+
|
|
83
|
+
const blocked = await harness.createTask(definition as never, {blocked: true});
|
|
84
|
+
const free = await harness.createTask(definition as never, {blocked: false});
|
|
85
|
+
const freeDone = await harness.waitForTask(free._id, {timeout: {seconds: 5}});
|
|
86
|
+
|
|
87
|
+
expect(freeDone.status).toBe("completed");
|
|
88
|
+
expect((await TaskModel.findExactlyOne({_id: blocked._id})).status).toBe("running");
|
|
89
|
+
releaseGate();
|
|
90
|
+
expect((await harness.waitForTask(blocked._id, {timeout: {seconds: 5}})).status).toBe(
|
|
91
|
+
"completed"
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("never runs one task twice at once and never exceeds the concurrency", async () => {
|
|
96
|
+
const activeByTask = new Map<string, number>();
|
|
97
|
+
let active = 0;
|
|
98
|
+
let peak = 0;
|
|
99
|
+
let peakPerTask = 0;
|
|
100
|
+
const enter = (taskId: string): void => {
|
|
101
|
+
active += 1;
|
|
102
|
+
peak = Math.max(peak, active);
|
|
103
|
+
const count = (activeByTask.get(taskId) ?? 0) + 1;
|
|
104
|
+
activeByTask.set(taskId, count);
|
|
105
|
+
peakPerTask = Math.max(peakPerTask, count);
|
|
106
|
+
};
|
|
107
|
+
const leave = (taskId: string): void => {
|
|
108
|
+
active -= 1;
|
|
109
|
+
activeByTask.set(taskId, (activeByTask.get(taskId) ?? 1) - 1);
|
|
110
|
+
};
|
|
111
|
+
// The first phase holds until three run at once (or 2 s pass), so the peak does not
|
|
112
|
+
// depend on how fast CI Mongo claims.
|
|
113
|
+
const fullHouse = async (): Promise<void> => {
|
|
114
|
+
for (let i = 0; i < 200 && active < 3; i++) {
|
|
115
|
+
await pause(10);
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
const definition = defineTask<{n: number}, Record<string, never>, {status: string}>({
|
|
119
|
+
initial: () => ({phase: "first", state: {}}),
|
|
120
|
+
name: "test.concurrencyPeak",
|
|
121
|
+
phases: {
|
|
122
|
+
first: {
|
|
123
|
+
run: async (task, rt) => {
|
|
124
|
+
enter(task.id);
|
|
125
|
+
await fullHouse();
|
|
126
|
+
await pause(30);
|
|
127
|
+
leave(task.id);
|
|
128
|
+
await rt.commit({phase: "second"});
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
second: {
|
|
132
|
+
run: async (task, rt) => {
|
|
133
|
+
enter(task.id);
|
|
134
|
+
await pause(30);
|
|
135
|
+
leave(task.id);
|
|
136
|
+
await rt.commit({terminal: {result: {status: "done"}, status: "completed"}});
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
},
|
|
140
|
+
version: 1,
|
|
141
|
+
}) as unknown as AnyHarnessTaskDefinition;
|
|
142
|
+
const harness = await openHarness({concurrency: 3, registry: [definition]});
|
|
143
|
+
|
|
144
|
+
const created = await Promise.all(
|
|
145
|
+
[1, 2, 3, 4, 5, 6, 7].map((n) => harness.createTask(definition as never, {n}))
|
|
146
|
+
);
|
|
147
|
+
const done = await Promise.all(
|
|
148
|
+
created.map((task) => harness.waitForTask(task._id, {timeout: {seconds: 10}}))
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
expect(done.map((task) => task.status)).toEqual(Array(7).fill("completed"));
|
|
152
|
+
expect(peakPerTask).toBe(1);
|
|
153
|
+
expect(peak).toBe(3);
|
|
154
|
+
});
|
|
155
|
+
});
|
package/src/harness/errors.ts
CHANGED
|
@@ -73,6 +73,12 @@ export const HARNESS_ERRORS = {
|
|
|
73
73
|
status: 409,
|
|
74
74
|
title: "requestId is already in use",
|
|
75
75
|
},
|
|
76
|
+
/** A `JobsRunner` phase job ran on an instance whose harness is not started. */
|
|
77
|
+
runnerStopped: {
|
|
78
|
+
code: "harness-runner-stopped",
|
|
79
|
+
status: 503,
|
|
80
|
+
title: "Harness runner is not running",
|
|
81
|
+
},
|
|
76
82
|
/** `retry` on a task whose abort is still running. */
|
|
77
83
|
taskAborting: {code: "harness-task-aborting", status: 409, title: "Task is being aborted"},
|
|
78
84
|
/** `resolveInterrupted` on a task that is not interrupted. */
|
|
@@ -577,21 +577,33 @@ describe("Harness event stream", () => {
|
|
|
577
577
|
const who = frame.data.taskId === String(created._id) ? "parent" : "child";
|
|
578
578
|
return `${who} ${frame.event} ${payload.status ?? payload.text ?? ""}`.trim();
|
|
579
579
|
};
|
|
580
|
-
|
|
580
|
+
// Parent and child run concurrently, so only each task's own order is fixed.
|
|
581
|
+
const lines = parentStream.frames.map(describe);
|
|
582
|
+
expect(lines.filter((line) => line.startsWith("parent"))).toEqual([
|
|
581
583
|
"parent task.status pending",
|
|
582
584
|
"parent output parent starting",
|
|
583
|
-
"child task.status pending",
|
|
584
585
|
"parent task.status waiting",
|
|
586
|
+
// Each phase re-runs after its wait, so its output is sent again.
|
|
587
|
+
"parent output parent starting",
|
|
588
|
+
"parent task.status completed",
|
|
589
|
+
]);
|
|
590
|
+
expect(lines.filter((line) => line.startsWith("child"))).toEqual([
|
|
591
|
+
"child task.status pending",
|
|
585
592
|
"child output child 1 working",
|
|
586
593
|
"child approval.requested pending",
|
|
587
594
|
"child task.status waiting",
|
|
588
595
|
"child approval.decided approved",
|
|
589
|
-
// Each phase re-runs after its wait, so its output is sent again.
|
|
590
596
|
"child output child 1 working",
|
|
591
597
|
"child task.status completed",
|
|
592
|
-
"parent output parent starting",
|
|
593
|
-
"parent task.status completed",
|
|
594
598
|
]);
|
|
599
|
+
// The child is created inside the parent's first phase, and the parent resumes only
|
|
600
|
+
// after the child completes.
|
|
601
|
+
expect(lines.indexOf("child task.status pending")).toBeLessThan(
|
|
602
|
+
lines.indexOf("parent task.status waiting")
|
|
603
|
+
);
|
|
604
|
+
expect(lines.indexOf("child task.status completed")).toBeLessThan(
|
|
605
|
+
lines.lastIndexOf("parent output parent starting")
|
|
606
|
+
);
|
|
595
607
|
// The child's stream is the same log narrowed to the child: no parent events.
|
|
596
608
|
expect(childStream.frames.map(describe)).toEqual(
|
|
597
609
|
parentStream.frames.map(describe).filter((line) => line.startsWith("child"))
|
package/src/harness/harness.ts
CHANGED
|
@@ -69,7 +69,13 @@ import {
|
|
|
69
69
|
splitRegistry,
|
|
70
70
|
} from "./registry";
|
|
71
71
|
import {InProcessRunner} from "./runners/inProcessRunner";
|
|
72
|
-
import {
|
|
72
|
+
import {
|
|
73
|
+
claimNextTask,
|
|
74
|
+
claimTaskById,
|
|
75
|
+
listRunnableTasks,
|
|
76
|
+
recoverExpiredTasks,
|
|
77
|
+
runClaimedTask,
|
|
78
|
+
} from "./runtime";
|
|
73
79
|
import {sendEventRecords} from "./waits";
|
|
74
80
|
|
|
75
81
|
export type {
|
|
@@ -121,8 +127,10 @@ export type {
|
|
|
121
127
|
HarnessResolveInterruptedOptions,
|
|
122
128
|
HarnessRetryPolicy,
|
|
123
129
|
HarnessRunAgentOptions,
|
|
130
|
+
HarnessRunnableTask,
|
|
124
131
|
HarnessRunner,
|
|
125
132
|
HarnessRunnerContext,
|
|
133
|
+
HarnessRunTaskOptions,
|
|
126
134
|
HarnessSection,
|
|
127
135
|
HarnessSectionInput,
|
|
128
136
|
HarnessSendEventOptions,
|
|
@@ -426,9 +434,12 @@ export class Harness {
|
|
|
426
434
|
await this.runner.start({
|
|
427
435
|
acquireOwnerLease: (lease) => acquireOwnerLease({lease, models, testHooks}),
|
|
428
436
|
claimNext: (lease) => this.claimNextOrSweep(lease),
|
|
437
|
+
claimTask: (taskId, lease) => claimTaskById({definitions, lease, models, taskId}),
|
|
438
|
+
listRunnable: (limit) => listRunnableTasks({definitions, limit, models}),
|
|
429
439
|
recoverExpired: () => recoverExpiredTasks(engine),
|
|
430
440
|
releaseOwnerLease: (lease) => releaseOwnerLease({lease, models}),
|
|
431
|
-
runTask: (task, lease) =>
|
|
441
|
+
runTask: (task, lease, options) =>
|
|
442
|
+
runClaimedTask({engine, lease, maxPhases: options?.maxPhases, task}),
|
|
432
443
|
});
|
|
433
444
|
}
|
|
434
445
|
|
|
@@ -450,7 +461,7 @@ export class Harness {
|
|
|
450
461
|
return claimNextTask({definitions, lease, models});
|
|
451
462
|
}
|
|
452
463
|
|
|
453
|
-
/** Stop claiming work and wait for
|
|
464
|
+
/** Stop claiming work and wait for every phase in flight to settle. */
|
|
454
465
|
async stop(): Promise<void> {
|
|
455
466
|
if (!this.isStarted) {
|
|
456
467
|
return;
|
|
@@ -39,6 +39,11 @@ const harnessTaskSchema = new mongoose.Schema<HarnessTaskDocument, HarnessTaskMo
|
|
|
39
39
|
description: "When true, the task outlives its owning conversation turn",
|
|
40
40
|
type: Boolean,
|
|
41
41
|
},
|
|
42
|
+
claims: {
|
|
43
|
+
default: 0,
|
|
44
|
+
description: "Times a runner claimed the task; numbers each runnable visit for job dispatch",
|
|
45
|
+
type: Number,
|
|
46
|
+
},
|
|
42
47
|
eventSeq: {
|
|
43
48
|
default: 0,
|
|
44
49
|
description: "Events received by harness.sendEvent; numbers the task's inbox",
|
|
@@ -43,11 +43,12 @@ const waitUntil = async (predicate: () => Promise<boolean>, label: string): Prom
|
|
|
43
43
|
|
|
44
44
|
const openHarness = async (
|
|
45
45
|
registry: ReadonlyArray<AnyHarnessTaskDefinition>,
|
|
46
|
-
testHooks?: HarnessTestHooks
|
|
46
|
+
testHooks?: HarnessTestHooks,
|
|
47
|
+
concurrency?: number
|
|
47
48
|
): Promise<Harness> => {
|
|
48
49
|
const harness = await Harness.open({
|
|
49
50
|
registry,
|
|
50
|
-
runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
|
|
51
|
+
runner: new InProcessRunner({concurrency, pollInterval: {milliseconds: 20}}),
|
|
51
52
|
testHooks,
|
|
52
53
|
});
|
|
53
54
|
openHarnesses.push(harness);
|
|
@@ -500,13 +501,12 @@ describe("Harness ownership tree", () => {
|
|
|
500
501
|
policy: "failFast",
|
|
501
502
|
runs,
|
|
502
503
|
});
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
failing,
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
]);
|
|
504
|
+
// One task at a time keeps the last sibling unstarted (pending) when the failure lands.
|
|
505
|
+
const harness = await openHarness(
|
|
506
|
+
[parent, failing, asAny(waitingSibling), asAny(pendingSibling), asAny(grandchild)],
|
|
507
|
+
undefined,
|
|
508
|
+
1
|
|
509
|
+
);
|
|
510
510
|
await harness.start();
|
|
511
511
|
|
|
512
512
|
const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
|
|
@@ -838,10 +838,8 @@ describe("Harness leases and crash recovery", () => {
|
|
|
838
838
|
});
|
|
839
839
|
|
|
840
840
|
describe("owner lease", () => {
|
|
841
|
-
it("
|
|
841
|
+
it("lets only the owner of two live harnesses run tasks, each task once", async () => {
|
|
842
842
|
const activeByTask = new Map<string, number>();
|
|
843
|
-
let maxConcurrent = 0;
|
|
844
|
-
let active = 0;
|
|
845
843
|
const ranBy: string[] = [];
|
|
846
844
|
const counted = defineTask<unknown, unknown, unknown>({
|
|
847
845
|
initial: () => ({phase: "work"}),
|
|
@@ -850,12 +848,9 @@ describe("Harness leases and crash recovery", () => {
|
|
|
850
848
|
work: {
|
|
851
849
|
replay: "safe",
|
|
852
850
|
run: async (task, rt) => {
|
|
853
|
-
active += 1;
|
|
854
851
|
activeByTask.set(task.id, (activeByTask.get(task.id) ?? 0) + 1);
|
|
855
|
-
maxConcurrent = Math.max(maxConcurrent, active);
|
|
856
852
|
ranBy.push(String((await TaskModel.findExactlyOne({_id: task.id})).lease?.owner));
|
|
857
853
|
await pause(15);
|
|
858
|
-
active -= 1;
|
|
859
854
|
await rt.commit({terminal: {status: "completed"}});
|
|
860
855
|
},
|
|
861
856
|
},
|
|
@@ -882,7 +877,7 @@ describe("Harness leases and crash recovery", () => {
|
|
|
882
877
|
created.map((task) => a.harness.waitForTask(task._id, {timeout: {seconds: 5}}))
|
|
883
878
|
);
|
|
884
879
|
|
|
885
|
-
|
|
880
|
+
// Only the owner runs work (concurrently, up to its limit); each task runs once.
|
|
886
881
|
expect([...activeByTask.values()]).toEqual([1, 1, 1, 1, 1, 1]);
|
|
887
882
|
expect(new Set(ranBy).size).toBe(1);
|
|
888
883
|
expect([a.runner.role, b.runner.role].sort()).toEqual(["owner", "standby"]);
|