@terreno/ai 57.10.1 → 57.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aiApp.d.ts +5 -0
- package/dist/aiApp.js +4 -2
- package/dist/harness/commit.d.ts +17 -1
- package/dist/harness/commit.js +86 -2
- package/dist/harness/errors.d.ts +6 -0
- package/dist/harness/errors.js +6 -0
- package/dist/harness/harness.d.ts +2 -2
- package/dist/harness/harness.js +4 -2
- package/dist/harness/models/harnessTask.js +5 -0
- package/dist/harness/routes/approvals.js +2 -2
- package/dist/harness/routes/conversations.js +1 -1
- package/dist/harness/routes/tasks.js +85 -3
- package/dist/harness/runners/inProcessRunner.d.ts +11 -4
- package/dist/harness/runners/inProcessRunner.js +44 -13
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +20 -2
- package/dist/harness/runtime.js +91 -32
- package/dist/harness/subagent.d.ts +17 -2
- package/dist/harness/subagent.js +26 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3 -2
- package/dist/models/project.d.ts +13 -0
- package/dist/models/project.js +32 -2
- package/dist/observability/local/models/obsTrace.js +16 -2
- package/dist/observability/local/traceStore.d.ts +5 -0
- package/dist/observability/local/traceStore.js +5 -0
- package/dist/observability/routes/traces.js +2 -0
- package/dist/routes/projects.js +3 -3
- package/dist/service/chatTurn.js +1 -1
- package/dist/types/harness.d.ts +47 -3
- package/dist/types/observability.d.ts +3 -0
- package/package.json +65 -6
- package/src/aiApp.test.ts +15 -0
- package/src/aiApp.ts +9 -1
- package/src/harness/commit.ts +103 -1
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/errors.ts +6 -0
- package/src/harness/events.test.ts +17 -5
- package/src/harness/harness.ts +14 -3
- package/src/harness/models/harnessTask.ts +5 -0
- package/src/harness/ownership.test.ts +9 -9
- package/src/harness/recovery.test.ts +2 -7
- package/src/harness/routes/approvals.ts +2 -2
- package/src/harness/routes/conversations.ts +1 -1
- package/src/harness/routes/tasks.test.ts +34 -0
- package/src/harness/routes/tasks.ts +98 -3
- package/src/harness/runners/inProcessRunner.test.ts +128 -0
- package/src/harness/runners/inProcessRunner.ts +54 -14
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +125 -30
- package/src/harness/subagent.test.ts +124 -3
- package/src/harness/subagent.ts +43 -1
- package/src/index.ts +1 -1
- package/src/isolated/lazyProjectModel.isolated.ts +25 -0
- package/src/models/project.ts +31 -1
- package/src/observability/local/models/obsTrace.ts +22 -5
- package/src/observability/local/traceStore.test.ts +7 -0
- package/src/observability/local/traceStore.ts +12 -0
- package/src/observability/routes/traces.test.ts +15 -0
- package/src/observability/routes/traces.ts +2 -0
- package/src/routes/projects.ts +4 -4
- package/src/service/chatTurn.ts +2 -2
- package/src/types/harness.ts +56 -3
- package/src/types/observability.ts +3 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import {randomUUID} from "node:crypto";
|
|
2
|
+
import {hostname} from "node:os";
|
|
3
|
+
import {logger} from "@terreno/api";
|
|
4
|
+
import {type JobDocument, type JobsApp, tryGetJobsService} from "@terreno/jobs";
|
|
5
|
+
import {DateTime, Duration, type DurationLike} from "luxon";
|
|
6
|
+
|
|
7
|
+
import type {
|
|
8
|
+
HarnessLeaseSettings,
|
|
9
|
+
HarnessRunnableTask,
|
|
10
|
+
HarnessRunner,
|
|
11
|
+
HarnessRunnerContext,
|
|
12
|
+
} from "../../types/harness";
|
|
13
|
+
import {errorMessage, harnessError} from "../errors";
|
|
14
|
+
|
|
15
|
+
/** Job that runs one phase of one harness task. */
|
|
16
|
+
export const HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
|
|
17
|
+
|
|
18
|
+
/** Job statuses that still lead to a handler run; any other status will never claim. */
|
|
19
|
+
const IN_FLIGHT_JOB_STATUSES = new Set<JobDocument["status"]>(["pending", "running", "scheduled"]);
|
|
20
|
+
|
|
21
|
+
export interface JobsRunnerOptions {
|
|
22
|
+
/** How many runnable tasks one dispatch tick enqueues at most. */
|
|
23
|
+
dispatchBatchSize?: number;
|
|
24
|
+
/** How often a running phase renews its task lease, and how often expired leases are swept. */
|
|
25
|
+
heartbeatInterval?: DurationLike;
|
|
26
|
+
/** The registered `JobsApp`; the runner defines `terreno.harness.phase` on it. */
|
|
27
|
+
jobs: JobsApp;
|
|
28
|
+
/** How long a task lease lives without a heartbeat. */
|
|
29
|
+
leaseDuration?: DurationLike;
|
|
30
|
+
/** Runner instance id written into task leases. Defaults to `hostname:pid:uuid`. */
|
|
31
|
+
ownerId?: string;
|
|
32
|
+
/** How long the dispatcher sleeps between scans for runnable tasks. */
|
|
33
|
+
pollInterval?: DurationLike;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
interface PhaseJobPayload {
|
|
37
|
+
taskId: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const parsePayload = (payload: unknown): PhaseJobPayload => {
|
|
41
|
+
const taskId = (payload as Partial<PhaseJobPayload> | null)?.taskId;
|
|
42
|
+
if (typeof taskId !== "string" || !taskId) {
|
|
43
|
+
throw harnessError({
|
|
44
|
+
detail: `${HARNESS_PHASE_JOB_NAME} payload needs a taskId string`,
|
|
45
|
+
kind: "invalidRequest",
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return {taskId};
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** `taskId:phase:attempt:claims` — one key per runnable visit of a phase. */
|
|
52
|
+
const phaseJobKey = (task: HarnessRunnableTask): string =>
|
|
53
|
+
`${task.taskId}:${task.phase}:${task.attempt}:${task.claims}`;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
|
|
57
|
+
* work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
|
|
58
|
+
* runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
|
|
59
|
+
* handler claims its task under a fenced lease, runs one phase while heartbeating, and
|
|
60
|
+
* hands a still-running task back as `pending` for the next job. The task lease, not the
|
|
61
|
+
* job lock, decides who may commit: a duplicate or late job finds the task claimed and
|
|
62
|
+
* does nothing, and a stale lease holder's commit is rejected.
|
|
63
|
+
*/
|
|
64
|
+
export class JobsRunner implements HarnessRunner {
|
|
65
|
+
private context: HarnessRunnerContext | undefined;
|
|
66
|
+
private readonly dispatchBatchSize: number;
|
|
67
|
+
private hasPendingWake = false;
|
|
68
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
69
|
+
private isStopping = false;
|
|
70
|
+
private readonly lease: HarnessLeaseSettings;
|
|
71
|
+
private loop: Promise<void> | undefined;
|
|
72
|
+
private nextSweepAt = DateTime.fromMillis(0);
|
|
73
|
+
private readonly pollIntervalMs: number;
|
|
74
|
+
private wakeSleeper: (() => void) | undefined;
|
|
75
|
+
|
|
76
|
+
constructor(options: JobsRunnerOptions) {
|
|
77
|
+
const duration = Duration.fromDurationLike(options.leaseDuration ?? {seconds: 30});
|
|
78
|
+
const heartbeat = Duration.fromDurationLike(options.heartbeatInterval ?? {seconds: 10});
|
|
79
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
80
|
+
throw harnessError({
|
|
81
|
+
detail: "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
82
|
+
kind: "configInvalid",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
this.lease = {
|
|
86
|
+
duration,
|
|
87
|
+
heartbeat,
|
|
88
|
+
owner: options.ownerId ?? `${hostname()}:${process.pid}:${randomUUID()}`,
|
|
89
|
+
};
|
|
90
|
+
this.pollIntervalMs = Duration.fromDurationLike(
|
|
91
|
+
options.pollInterval ?? {milliseconds: 500}
|
|
92
|
+
).toMillis();
|
|
93
|
+
const dispatchBatchSize = options.dispatchBatchSize ?? 100;
|
|
94
|
+
if (!Number.isInteger(dispatchBatchSize) || dispatchBatchSize < 1) {
|
|
95
|
+
throw harnessError({
|
|
96
|
+
detail: "JobsRunner dispatchBatchSize must be a positive integer",
|
|
97
|
+
kind: "configInvalid",
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
this.dispatchBatchSize = dispatchBatchSize;
|
|
101
|
+
options.jobs.define(HARNESS_PHASE_JOB_NAME, {
|
|
102
|
+
handler: (payload) => this.runPhaseJob(payload),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Id written into every task lease this runner holds. */
|
|
107
|
+
get ownerId(): string {
|
|
108
|
+
return this.lease.owner;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async start(context: HarnessRunnerContext): Promise<void> {
|
|
112
|
+
if (this.loop !== undefined) {
|
|
113
|
+
throw harnessError({detail: "JobsRunner is already started", kind: "alreadyStarted"});
|
|
114
|
+
}
|
|
115
|
+
if (!tryGetJobsService()) {
|
|
116
|
+
throw harnessError({
|
|
117
|
+
detail:
|
|
118
|
+
"JobsRunner needs a registered JobsApp: register it and build the app before harness.start()",
|
|
119
|
+
kind: "configInvalid",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
this.context = context;
|
|
123
|
+
this.isStopping = false;
|
|
124
|
+
this.nextSweepAt = DateTime.fromMillis(0);
|
|
125
|
+
this.loop = this.dispatchLoop();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Stop dispatching, then wait for every phase this instance is running. */
|
|
129
|
+
async stop(): Promise<void> {
|
|
130
|
+
if (this.loop === undefined) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
this.isStopping = true;
|
|
134
|
+
this.wake();
|
|
135
|
+
await this.loop;
|
|
136
|
+
await Promise.allSettled([...this.inFlight]);
|
|
137
|
+
this.loop = undefined;
|
|
138
|
+
this.context = undefined;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Dispatch now instead of after the rest of the idle sleep. */
|
|
142
|
+
wake(): void {
|
|
143
|
+
if (this.wakeSleeper) {
|
|
144
|
+
this.wakeSleeper();
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.hasPendingWake = true;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Job handler: claim the task, run one phase, and hand it back if it still runs. */
|
|
151
|
+
private async runPhaseJob(payload: unknown): Promise<void> {
|
|
152
|
+
const {taskId} = parsePayload(payload);
|
|
153
|
+
// Tracked from before the claim, so stop() also waits for a claim in flight.
|
|
154
|
+
const job = this.claimAndRunPhase(taskId);
|
|
155
|
+
this.inFlight.add(job);
|
|
156
|
+
try {
|
|
157
|
+
await job;
|
|
158
|
+
} finally {
|
|
159
|
+
this.inFlight.delete(job);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private async claimAndRunPhase(taskId: string): Promise<void> {
|
|
164
|
+
const context = this.context;
|
|
165
|
+
if (!context || this.isStopping) {
|
|
166
|
+
// Throwing makes the jobs worker retry later, on this instance or another one.
|
|
167
|
+
throw harnessError({
|
|
168
|
+
detail: `JobsRunner is not running; task ${taskId} stays runnable`,
|
|
169
|
+
kind: "runnerStopped",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const task = await context.claimTask(taskId, this.lease);
|
|
173
|
+
if (!task) {
|
|
174
|
+
// Another job claimed it first, or it stopped being runnable since dispatch.
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
await context.runTask(task, this.lease, {maxPhases: 1});
|
|
178
|
+
// The next phase (or a retry that is already due) can be dispatched right away.
|
|
179
|
+
this.wake();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private async dispatchLoop(): Promise<void> {
|
|
183
|
+
while (!this.isStopping && this.context) {
|
|
184
|
+
await this.tick(this.context);
|
|
185
|
+
if (!this.isStopping) {
|
|
186
|
+
await this.sleep(this.pollIntervalMs);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
|
|
192
|
+
private async tick(context: HarnessRunnerContext): Promise<void> {
|
|
193
|
+
try {
|
|
194
|
+
if (DateTime.now() >= this.nextSweepAt) {
|
|
195
|
+
this.nextSweepAt = DateTime.now().plus(this.lease.heartbeat);
|
|
196
|
+
await context.recoverExpired();
|
|
197
|
+
}
|
|
198
|
+
const runnable = await context.listRunnable(this.dispatchBatchSize);
|
|
199
|
+
for (const task of runnable) {
|
|
200
|
+
if (this.isStopping) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
await this.dispatch(task);
|
|
204
|
+
}
|
|
205
|
+
} catch (error: unknown) {
|
|
206
|
+
logger.error(`JobsRunner dispatch failed: ${errorMessage(error)}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Enqueue the job for this runnable visit. When the job with that key already ended
|
|
212
|
+
* without claiming the task (its handler kept failing, or it ran where the task was not
|
|
213
|
+
* runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
|
|
214
|
+
* the task is not stranded. A newly created job is always in flight, so this ends.
|
|
215
|
+
*/
|
|
216
|
+
private async dispatch(task: HarnessRunnableTask): Promise<void> {
|
|
217
|
+
const jobs = tryGetJobsService();
|
|
218
|
+
if (!jobs) {
|
|
219
|
+
throw harnessError({detail: "JobsApp is no longer registered", kind: "configInvalid"});
|
|
220
|
+
}
|
|
221
|
+
let idempotencyKey = phaseJobKey(task);
|
|
222
|
+
for (;;) {
|
|
223
|
+
const job = await jobs.enqueue({
|
|
224
|
+
idempotencyKey,
|
|
225
|
+
name: HARNESS_PHASE_JOB_NAME,
|
|
226
|
+
payload: {taskId: task.taskId} satisfies PhaseJobPayload,
|
|
227
|
+
});
|
|
228
|
+
if (IN_FLIGHT_JOB_STATUSES.has(job.status)) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
idempotencyKey = `${phaseJobKey(task)}:${job._id}`;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private sleep(ms: number): Promise<void> {
|
|
236
|
+
if (this.hasPendingWake) {
|
|
237
|
+
this.hasPendingWake = false;
|
|
238
|
+
return Promise.resolve();
|
|
239
|
+
}
|
|
240
|
+
return new Promise((resolve) => {
|
|
241
|
+
const timer = setTimeout(() => {
|
|
242
|
+
this.wakeSleeper = undefined;
|
|
243
|
+
resolve();
|
|
244
|
+
}, ms);
|
|
245
|
+
this.wakeSleeper = () => {
|
|
246
|
+
clearTimeout(timer);
|
|
247
|
+
this.wakeSleeper = undefined;
|
|
248
|
+
resolve();
|
|
249
|
+
};
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
package/src/harness/runtime.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
HarnessCommit,
|
|
13
13
|
HarnessLeaseSettings,
|
|
14
14
|
HarnessRunAgentOptions,
|
|
15
|
+
HarnessRunnableTask,
|
|
15
16
|
HarnessTaskDefinition,
|
|
16
17
|
HarnessTaskDocument,
|
|
17
18
|
HarnessTaskRuntime,
|
|
@@ -62,6 +63,7 @@ import {
|
|
|
62
63
|
serializeOutputSchema,
|
|
63
64
|
startSubagentTurn,
|
|
64
65
|
subagentContent,
|
|
66
|
+
subagentPrompts,
|
|
65
67
|
subagentResult,
|
|
66
68
|
} from "./subagent";
|
|
67
69
|
import {HarnessSuspendSignal} from "./suspend";
|
|
@@ -99,6 +101,42 @@ const registeredFilter = (
|
|
|
99
101
|
return [...definitions.values()].map(({name, version}) => ({name, version}));
|
|
100
102
|
};
|
|
101
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Mongo filter for a registered task a runner may claim now: `pending` and due, or
|
|
106
|
+
* `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
|
|
107
|
+
*/
|
|
108
|
+
const runnableTaskFilter = (
|
|
109
|
+
definitions: Map<string, HarnessTaskDefinition>,
|
|
110
|
+
now: DateTime
|
|
111
|
+
): Record<string, unknown> => ({
|
|
112
|
+
$and: [
|
|
113
|
+
{$or: registeredFilter(definitions)},
|
|
114
|
+
{
|
|
115
|
+
$or: [
|
|
116
|
+
{
|
|
117
|
+
$or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now.toJSDate()}}],
|
|
118
|
+
status: HARNESS_TASK_STATUSES.pending,
|
|
119
|
+
},
|
|
120
|
+
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
121
|
+
{
|
|
122
|
+
status: HARNESS_TASK_STATUSES.waiting,
|
|
123
|
+
"waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
|
|
124
|
+
"waiting.timeoutAt": {$lte: now.toJSDate()},
|
|
125
|
+
},
|
|
126
|
+
],
|
|
127
|
+
},
|
|
128
|
+
],
|
|
129
|
+
// A task being aborted never starts another run.
|
|
130
|
+
"abortRequested.at": {$exists: false},
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
/** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
|
|
134
|
+
const claimUpdate = (lease: HarnessLeaseSettings): Record<string, unknown> => ({
|
|
135
|
+
$inc: {claims: 1},
|
|
136
|
+
$set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
|
|
137
|
+
$unset: {waiting: 1},
|
|
138
|
+
});
|
|
139
|
+
|
|
102
140
|
/**
|
|
103
141
|
* Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
|
|
104
142
|
* an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
|
|
@@ -115,36 +153,78 @@ export const claimNextTask = async ({
|
|
|
115
153
|
if (definitions.size === 0) {
|
|
116
154
|
return null;
|
|
117
155
|
}
|
|
118
|
-
const registered = registeredFilter(definitions);
|
|
119
|
-
const now = DateTime.now().toJSDate();
|
|
120
156
|
return models.task.findOneAndUpdate(
|
|
157
|
+
runnableTaskFilter(definitions, DateTime.now()),
|
|
158
|
+
claimUpdate(lease),
|
|
159
|
+
{returnDocument: "after", sort: {created: 1}}
|
|
160
|
+
);
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
|
|
165
|
+
* runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
|
|
166
|
+
*/
|
|
167
|
+
export const claimTaskById = async ({
|
|
168
|
+
definitions,
|
|
169
|
+
lease,
|
|
170
|
+
models,
|
|
171
|
+
taskId,
|
|
172
|
+
}: {
|
|
173
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
174
|
+
lease: HarnessLeaseSettings;
|
|
175
|
+
models: HarnessModels;
|
|
176
|
+
taskId: string;
|
|
177
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
178
|
+
if (definitions.size === 0 || !mongoose.isValidObjectId(taskId)) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
return models.task.findOneAndUpdate(
|
|
182
|
+
{...runnableTaskFilter(definitions, DateTime.now()), _id: toObjectId(taskId)},
|
|
183
|
+
claimUpdate(lease),
|
|
184
|
+
{returnDocument: "after"}
|
|
185
|
+
);
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
/** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
|
|
189
|
+
export const listRunnableTasks = async ({
|
|
190
|
+
definitions,
|
|
191
|
+
limit,
|
|
192
|
+
models,
|
|
193
|
+
}: {
|
|
194
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
195
|
+
limit: number;
|
|
196
|
+
models: HarnessModels;
|
|
197
|
+
}): Promise<HarnessRunnableTask[]> => {
|
|
198
|
+
if (definitions.size === 0) {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
const tasks = await models.task
|
|
202
|
+
.find(runnableTaskFilter(definitions, DateTime.now()))
|
|
203
|
+
.select({attempt: 1, claims: 1, phase: 1})
|
|
204
|
+
.sort({created: 1})
|
|
205
|
+
.limit(limit);
|
|
206
|
+
return tasks.map((task) => ({
|
|
207
|
+
attempt: task.attempt ?? 0,
|
|
208
|
+
claims: task.claims ?? 0,
|
|
209
|
+
phase: task.phase,
|
|
210
|
+
taskId: String(task._id),
|
|
211
|
+
}));
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Give a task that is still `running` under `task`'s lease back to the runners as
|
|
216
|
+
* `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
|
|
217
|
+
*/
|
|
218
|
+
const releaseTask = async (models: HarnessModels, task: HarnessTaskDocument): Promise<boolean> => {
|
|
219
|
+
const result = await models.task.updateOne(
|
|
121
220
|
{
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
$or: [
|
|
126
|
-
{
|
|
127
|
-
$or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now}}],
|
|
128
|
-
status: HARNESS_TASK_STATUSES.pending,
|
|
129
|
-
},
|
|
130
|
-
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
131
|
-
{
|
|
132
|
-
status: HARNESS_TASK_STATUSES.waiting,
|
|
133
|
-
"waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
|
|
134
|
-
"waiting.timeoutAt": {$lte: now},
|
|
135
|
-
},
|
|
136
|
-
],
|
|
137
|
-
},
|
|
138
|
-
],
|
|
139
|
-
// A task being aborted never starts another run.
|
|
140
|
-
"abortRequested.at": {$exists: false},
|
|
141
|
-
},
|
|
142
|
-
{
|
|
143
|
-
$set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
|
|
144
|
-
$unset: {waiting: 1},
|
|
221
|
+
_id: task._id,
|
|
222
|
+
"lease.token": task.lease?.token ?? null,
|
|
223
|
+
status: HARNESS_TASK_STATUSES.running,
|
|
145
224
|
},
|
|
146
|
-
{
|
|
225
|
+
{$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {lease: 1}}
|
|
147
226
|
);
|
|
227
|
+
return result.modifiedCount > 0;
|
|
148
228
|
};
|
|
149
229
|
|
|
150
230
|
/**
|
|
@@ -219,10 +299,13 @@ const interruptionAction = (
|
|
|
219
299
|
export const runClaimedTask = async ({
|
|
220
300
|
engine,
|
|
221
301
|
lease,
|
|
302
|
+
maxPhases,
|
|
222
303
|
task,
|
|
223
304
|
}: {
|
|
224
305
|
engine: HarnessEngine;
|
|
225
306
|
lease: HarnessLeaseSettings;
|
|
307
|
+
/** Hand the task back as `pending` once this many phases committed and it still runs. */
|
|
308
|
+
maxPhases?: number;
|
|
226
309
|
task: HarnessTaskDocument;
|
|
227
310
|
}): Promise<void> => {
|
|
228
311
|
const definition = engine.definitions.get(taskDefinitionKey(task));
|
|
@@ -238,14 +321,20 @@ export const runClaimedTask = async ({
|
|
|
238
321
|
engine.controllers.set(taskId, controller);
|
|
239
322
|
try {
|
|
240
323
|
let current = task;
|
|
241
|
-
let
|
|
324
|
+
let phasesRun = 0;
|
|
242
325
|
while (current.status === HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
|
|
326
|
+
if (maxPhases !== undefined && phasesRun >= maxPhases) {
|
|
327
|
+
if (!(await releaseTask(engine.models, current))) {
|
|
328
|
+
logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
|
|
329
|
+
}
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
243
332
|
// An abort from another process may have fenced the task since the last commit.
|
|
244
|
-
if (
|
|
333
|
+
if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
|
|
245
334
|
logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
|
|
246
335
|
return;
|
|
247
336
|
}
|
|
248
|
-
|
|
337
|
+
phasesRun += 1;
|
|
249
338
|
const result = await runPhase({controller, definition, engine, lease, task: current});
|
|
250
339
|
if (!result) {
|
|
251
340
|
return;
|
|
@@ -544,16 +633,22 @@ const runPhase = async ({
|
|
|
544
633
|
);
|
|
545
634
|
}
|
|
546
635
|
}
|
|
636
|
+
const {instructions, prompts} = subagentPrompts({
|
|
637
|
+
instructions: options.instructions,
|
|
638
|
+
prompts: options.prompts,
|
|
639
|
+
where: `${definition.key}: rt.runAgent`,
|
|
640
|
+
});
|
|
547
641
|
const {conversationId, turnTask} = await startSubagentTurn({
|
|
548
642
|
agent,
|
|
549
643
|
callIndex,
|
|
550
644
|
content,
|
|
551
645
|
extensions: engine.extensions,
|
|
552
|
-
instructions
|
|
646
|
+
instructions,
|
|
553
647
|
lease,
|
|
554
648
|
models,
|
|
555
649
|
outputSchema,
|
|
556
650
|
parent: task,
|
|
651
|
+
prompts,
|
|
557
652
|
turn,
|
|
558
653
|
});
|
|
559
654
|
engine.wake();
|
|
@@ -15,7 +15,12 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
15
15
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
16
16
|
import {generateToStream, withGenerateStreaming} from "../tests/generateStream";
|
|
17
17
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
18
|
-
import type {
|
|
18
|
+
import type {
|
|
19
|
+
HarnessAgentDefinition,
|
|
20
|
+
HarnessPromptInstructions,
|
|
21
|
+
HarnessPromptRef,
|
|
22
|
+
HarnessTestHooks,
|
|
23
|
+
} from "../types/harness";
|
|
19
24
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
20
25
|
import {
|
|
21
26
|
AGENT_TURN_TASK_NAME,
|
|
@@ -211,12 +216,14 @@ const parentTask = ({
|
|
|
211
216
|
instructions,
|
|
212
217
|
name = "test.parent",
|
|
213
218
|
output,
|
|
219
|
+
prompts,
|
|
214
220
|
}: {
|
|
215
221
|
agent: HarnessAgentDefinition;
|
|
216
222
|
inputs?: unknown[];
|
|
217
|
-
instructions?: string;
|
|
223
|
+
instructions?: HarnessPromptInstructions | string;
|
|
218
224
|
name?: string;
|
|
219
225
|
output?: z.ZodType;
|
|
226
|
+
prompts?: HarnessPromptRef[];
|
|
220
227
|
}) =>
|
|
221
228
|
defineTask<Record<string, never>, Record<string, never>, {results: unknown[]}>({
|
|
222
229
|
initial: () => ({phase: "summarize"}),
|
|
@@ -227,7 +234,7 @@ const parentTask = ({
|
|
|
227
234
|
run: async (_task, rt) => {
|
|
228
235
|
const results: unknown[] = [];
|
|
229
236
|
for (const input of inputs) {
|
|
230
|
-
results.push(await rt.runAgent(agent, {input, instructions, output}));
|
|
237
|
+
results.push(await rt.runAgent(agent, {input, instructions, output, prompts}));
|
|
231
238
|
}
|
|
232
239
|
await rt.commit({terminal: {result: {results}, status: "completed"}});
|
|
233
240
|
},
|
|
@@ -304,6 +311,120 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
304
311
|
);
|
|
305
312
|
});
|
|
306
313
|
|
|
314
|
+
it("records the task's trace scope, tags and prompt versions, and the subagent's prompts", async () => {
|
|
315
|
+
const agent = summarizerWith();
|
|
316
|
+
const parent = parentTask({
|
|
317
|
+
agent,
|
|
318
|
+
inputs: [{chart: "a"}, {chart: "b"}],
|
|
319
|
+
instructions: {
|
|
320
|
+
body: "Registry says be brief.",
|
|
321
|
+
label: "production",
|
|
322
|
+
name: "summary",
|
|
323
|
+
version: 3,
|
|
324
|
+
},
|
|
325
|
+
prompts: [{name: "style", version: 1}],
|
|
326
|
+
});
|
|
327
|
+
const {calls, model} = scriptedModel([{text: "A."}, {text: "B."}]);
|
|
328
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
329
|
+
|
|
330
|
+
const created = await harness.createTask(
|
|
331
|
+
parent,
|
|
332
|
+
{},
|
|
333
|
+
{
|
|
334
|
+
prompts: [{name: "plan", version: 2}],
|
|
335
|
+
trace: {scope: "workspace-1", tags: ["nightly", "nightly", "beta"]},
|
|
336
|
+
}
|
|
337
|
+
);
|
|
338
|
+
const done = await harness.waitForTask(created._id, {timeout: {seconds: 10}});
|
|
339
|
+
|
|
340
|
+
expect(done.status).toBe("completed");
|
|
341
|
+
const trace = await TraceModel.findExactlyOne({_id: done.traceId});
|
|
342
|
+
expect(trace.scope).toBe("workspace-1");
|
|
343
|
+
expect(plain(trace.tags)).toEqual(["nightly", "beta"]);
|
|
344
|
+
// Both runAgent calls used the same versions; each is recorded once.
|
|
345
|
+
expect(plain(trace.prompts)).toEqual([
|
|
346
|
+
{name: "plan", version: 2},
|
|
347
|
+
{label: "production", name: "summary", version: 3},
|
|
348
|
+
{name: "style", version: 1},
|
|
349
|
+
]);
|
|
350
|
+
const conversations = await ConversationModel.find({});
|
|
351
|
+
expect(conversations.map(({agent: snapshot}) => snapshot.instructions)).toEqual([
|
|
352
|
+
"Registry says be brief.",
|
|
353
|
+
"Registry says be brief.",
|
|
354
|
+
]);
|
|
355
|
+
expect(plain(calls[0]?.prompt[0])).toEqual({
|
|
356
|
+
content: "Registry says be brief.",
|
|
357
|
+
role: "system",
|
|
358
|
+
});
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
it("rejects a blank trace scope and a prompt ref without a version", async () => {
|
|
362
|
+
const agent = summarizerWith();
|
|
363
|
+
const parent = parentTask({agent});
|
|
364
|
+
const {model} = scriptedModel([]);
|
|
365
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
366
|
+
|
|
367
|
+
await expect(harness.createTask(parent, {}, {trace: {scope: " "}})).rejects.toThrow(
|
|
368
|
+
harnessErrorMatching("invalidRequest", "trace.scope must be a non-empty string")
|
|
369
|
+
);
|
|
370
|
+
await expect(
|
|
371
|
+
harness.createTask(parent, {}, {prompts: [{name: "plan"} as HarnessPromptRef]})
|
|
372
|
+
).rejects.toThrow(
|
|
373
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" needs a positive integer version')
|
|
374
|
+
);
|
|
375
|
+
await expect(
|
|
376
|
+
harness.createTask(parent, {}, {prompts: [{label: 3, name: "plan", version: 1} as never]})
|
|
377
|
+
).rejects.toThrow(
|
|
378
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
|
|
379
|
+
);
|
|
380
|
+
await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
|
|
381
|
+
harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
|
|
382
|
+
);
|
|
383
|
+
expect(await TaskModel.countDocuments({})).toBe(0);
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
it("records a repeated createTask prompt ref once", async () => {
|
|
387
|
+
const agent = summarizerWith();
|
|
388
|
+
const parent = parentTask({agent, inputs: []});
|
|
389
|
+
const {model} = scriptedModel([]);
|
|
390
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
391
|
+
|
|
392
|
+
const plan = {name: "plan", version: 2};
|
|
393
|
+
const created = await harness.createTask(parent, {}, {prompts: [plan, {...plan}]});
|
|
394
|
+
|
|
395
|
+
const trace = await TraceModel.findExactlyOne({_id: created.traceId});
|
|
396
|
+
expect(plain(trace.prompts)).toEqual([plan]);
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
|
|
400
|
+
const agent = summarizerWith();
|
|
401
|
+
const noBody = parentTask({
|
|
402
|
+
agent,
|
|
403
|
+
instructions: {name: "summary", version: 1} as HarnessPromptInstructions,
|
|
404
|
+
name: "test.noBody",
|
|
405
|
+
});
|
|
406
|
+
const badRef = parentTask({
|
|
407
|
+
agent,
|
|
408
|
+
name: "test.badRef",
|
|
409
|
+
prompts: [{name: "style", version: 0}],
|
|
410
|
+
});
|
|
411
|
+
const {calls, model} = scriptedModel([]);
|
|
412
|
+
const {harness} = await openProcess({model, registry: [agent, noBody, badRef]});
|
|
413
|
+
|
|
414
|
+
const first = await harness.createTask(noBody, {});
|
|
415
|
+
const second = await harness.createTask(badRef, {});
|
|
416
|
+
const failedNoBody = await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
417
|
+
const failedBadRef = await harness.waitForTask(second._id, {timeout: {seconds: 10}});
|
|
418
|
+
|
|
419
|
+
expect(failedNoBody.status).toBe("failed");
|
|
420
|
+
expect(failedNoBody.outcome?.error).toContain("instructions prompt needs a string body");
|
|
421
|
+
expect(failedBadRef.status).toBe("failed");
|
|
422
|
+
expect(failedBadRef.outcome?.error).toContain(
|
|
423
|
+
'Prompt ref "style" needs a positive integer version'
|
|
424
|
+
);
|
|
425
|
+
expect(calls).toHaveLength(0);
|
|
426
|
+
});
|
|
427
|
+
|
|
307
428
|
it("nests the subagent's AGENT span under the caller and its LLM and TOOL spans under it", async () => {
|
|
308
429
|
const lookup = defineTool({
|
|
309
430
|
description: "Look up a chart",
|