@terreno/ai 57.10.1 → 57.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aiApp.d.ts +5 -0
- package/dist/aiApp.js +4 -2
- package/dist/harness/commit.d.ts +17 -1
- package/dist/harness/commit.js +86 -2
- package/dist/harness/errors.d.ts +6 -0
- package/dist/harness/errors.js +6 -0
- package/dist/harness/harness.d.ts +2 -2
- package/dist/harness/harness.js +4 -2
- package/dist/harness/models/harnessTask.js +5 -0
- package/dist/harness/routes/approvals.js +2 -2
- package/dist/harness/routes/conversations.js +1 -1
- package/dist/harness/routes/tasks.js +85 -3
- package/dist/harness/runners/inProcessRunner.d.ts +11 -4
- package/dist/harness/runners/inProcessRunner.js +44 -13
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +20 -2
- package/dist/harness/runtime.js +91 -32
- package/dist/harness/subagent.d.ts +17 -2
- package/dist/harness/subagent.js +26 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3 -2
- package/dist/models/project.d.ts +13 -0
- package/dist/models/project.js +32 -2
- package/dist/observability/local/models/obsTrace.js +16 -2
- package/dist/observability/local/traceStore.d.ts +5 -0
- package/dist/observability/local/traceStore.js +5 -0
- package/dist/observability/routes/traces.js +2 -0
- package/dist/routes/projects.js +3 -3
- package/dist/service/chatTurn.js +1 -1
- package/dist/types/harness.d.ts +47 -3
- package/dist/types/observability.d.ts +3 -0
- package/package.json +65 -6
- package/src/aiApp.test.ts +15 -0
- package/src/aiApp.ts +9 -1
- package/src/harness/commit.ts +103 -1
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/errors.ts +6 -0
- package/src/harness/events.test.ts +17 -5
- package/src/harness/harness.ts +14 -3
- package/src/harness/models/harnessTask.ts +5 -0
- package/src/harness/ownership.test.ts +9 -9
- package/src/harness/recovery.test.ts +2 -7
- package/src/harness/routes/approvals.ts +2 -2
- package/src/harness/routes/conversations.ts +1 -1
- package/src/harness/routes/tasks.test.ts +34 -0
- package/src/harness/routes/tasks.ts +98 -3
- package/src/harness/runners/inProcessRunner.test.ts +128 -0
- package/src/harness/runners/inProcessRunner.ts +54 -14
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +125 -30
- package/src/harness/subagent.test.ts +124 -3
- package/src/harness/subagent.ts +43 -1
- package/src/index.ts +1 -1
- package/src/isolated/lazyProjectModel.isolated.ts +25 -0
- package/src/models/project.ts +31 -1
- package/src/observability/local/models/obsTrace.ts +22 -5
- package/src/observability/local/traceStore.test.ts +7 -0
- package/src/observability/local/traceStore.ts +12 -0
- package/src/observability/routes/traces.test.ts +15 -0
- package/src/observability/routes/traces.ts +2 -0
- package/src/routes/projects.ts +4 -4
- package/src/service/chatTurn.ts +2 -2
- package/src/types/harness.ts +56 -3
- package/src/types/observability.ts +3 -0
|
@@ -12,6 +12,8 @@ const fakeTask = (id: string): HarnessTaskDocument => ({_id: id}) as unknown as
|
|
|
12
12
|
const fakeContext = (overrides: Partial<HarnessRunnerContext>): HarnessRunnerContext => ({
|
|
13
13
|
acquireOwnerLease: async () => true,
|
|
14
14
|
claimNext: async () => null,
|
|
15
|
+
claimTask: async () => null,
|
|
16
|
+
listRunnable: async () => [],
|
|
15
17
|
recoverExpired: async () => 0,
|
|
16
18
|
releaseOwnerLease: async () => {},
|
|
17
19
|
runTask: async () => {},
|
|
@@ -156,6 +158,132 @@ describe("InProcessRunner", () => {
|
|
|
156
158
|
expect(isStopped).toBe(true);
|
|
157
159
|
});
|
|
158
160
|
|
|
161
|
+
describe("concurrency", () => {
|
|
162
|
+
const blockingTask = (): {done: Promise<void>; release: () => void} => {
|
|
163
|
+
let release: () => void = () => {};
|
|
164
|
+
const done = new Promise<void>((resolve) => {
|
|
165
|
+
release = resolve;
|
|
166
|
+
});
|
|
167
|
+
return {done, release};
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
it("defaults to 8 tasks at a time", async () => {
|
|
171
|
+
const gates = Array.from({length: 10}, () => blockingTask());
|
|
172
|
+
const queue = gates.map((_gate, index) => fakeTask(String(index)));
|
|
173
|
+
let running = 0;
|
|
174
|
+
let peak = 0;
|
|
175
|
+
const context = fakeContext({
|
|
176
|
+
claimNext: async () => queue.shift() ?? null,
|
|
177
|
+
runTask: async (task) => {
|
|
178
|
+
running += 1;
|
|
179
|
+
peak = Math.max(peak, running);
|
|
180
|
+
await gates[Number(task._id)].done;
|
|
181
|
+
running -= 1;
|
|
182
|
+
},
|
|
183
|
+
});
|
|
184
|
+
const runner = new InProcessRunner({pollInterval: {milliseconds: 5}});
|
|
185
|
+
await runner.start(context);
|
|
186
|
+
await waitUntil(() => running === 8);
|
|
187
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
188
|
+
|
|
189
|
+
expect(running).toBe(8);
|
|
190
|
+
expect(queue.length).toBe(2);
|
|
191
|
+
for (const gate of gates) {
|
|
192
|
+
gate.release();
|
|
193
|
+
}
|
|
194
|
+
await waitUntil(() => queue.length === 0 && running === 0);
|
|
195
|
+
await runner.stop();
|
|
196
|
+
expect(peak).toBe(8);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("lets a second task complete while the first is blocked on a gate", async () => {
|
|
200
|
+
const slow = blockingTask();
|
|
201
|
+
const finished: string[] = [];
|
|
202
|
+
const queue = [fakeTask("slow"), fakeTask("fast")];
|
|
203
|
+
const context = fakeContext({
|
|
204
|
+
claimNext: async () => queue.shift() ?? null,
|
|
205
|
+
runTask: async (task) => {
|
|
206
|
+
if (String(task._id) === "slow") {
|
|
207
|
+
await slow.done;
|
|
208
|
+
}
|
|
209
|
+
finished.push(String(task._id));
|
|
210
|
+
},
|
|
211
|
+
});
|
|
212
|
+
const runner = new InProcessRunner({concurrency: 2, pollInterval: {minutes: 10}});
|
|
213
|
+
await runner.start(context);
|
|
214
|
+
|
|
215
|
+
await waitUntil(() => finished.length === 1);
|
|
216
|
+
expect(finished).toEqual(["fast"]);
|
|
217
|
+
slow.release();
|
|
218
|
+
await waitUntil(() => finished.length === 2);
|
|
219
|
+
await runner.stop();
|
|
220
|
+
expect(finished).toEqual(["fast", "slow"]);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("never runs more than concurrency tasks and claims again as soon as a slot frees", async () => {
|
|
224
|
+
const queue = ["a", "b", "c", "d", "e"].map(fakeTask);
|
|
225
|
+
const ran: string[] = [];
|
|
226
|
+
let running = 0;
|
|
227
|
+
let peak = 0;
|
|
228
|
+
const context = fakeContext({
|
|
229
|
+
claimNext: async () => queue.shift() ?? null,
|
|
230
|
+
runTask: async (task) => {
|
|
231
|
+
running += 1;
|
|
232
|
+
peak = Math.max(peak, running);
|
|
233
|
+
await new Promise((resolve) => setTimeout(resolve, 15));
|
|
234
|
+
ran.push(String(task._id));
|
|
235
|
+
running -= 1;
|
|
236
|
+
},
|
|
237
|
+
});
|
|
238
|
+
// Only a freed slot (not the 10-minute poll) can pick up the rest.
|
|
239
|
+
const runner = new InProcessRunner({concurrency: 2, pollInterval: {minutes: 10}});
|
|
240
|
+
await runner.start(context);
|
|
241
|
+
await waitUntil(() => ran.length === 5);
|
|
242
|
+
await runner.stop();
|
|
243
|
+
|
|
244
|
+
expect(peak).toBe(2);
|
|
245
|
+
expect([...ran].sort()).toEqual(["a", "b", "c", "d", "e"]);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("waits for every task in flight before stop resolves", async () => {
|
|
249
|
+
const gates = [blockingTask(), blockingTask()];
|
|
250
|
+
const queue = [fakeTask("0"), fakeTask("1")];
|
|
251
|
+
let started = 0;
|
|
252
|
+
let isStopped = false;
|
|
253
|
+
const context = fakeContext({
|
|
254
|
+
claimNext: async () => queue.shift() ?? null,
|
|
255
|
+
runTask: async (task) => {
|
|
256
|
+
started += 1;
|
|
257
|
+
await gates[Number(task._id)].done;
|
|
258
|
+
},
|
|
259
|
+
});
|
|
260
|
+
const runner = new InProcessRunner({concurrency: 2, pollInterval: {milliseconds: 5}});
|
|
261
|
+
await runner.start(context);
|
|
262
|
+
await waitUntil(() => started === 2);
|
|
263
|
+
|
|
264
|
+
const stopping = runner.stop().then(() => {
|
|
265
|
+
isStopped = true;
|
|
266
|
+
});
|
|
267
|
+
gates[0].release();
|
|
268
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
269
|
+
expect(isStopped).toBe(false);
|
|
270
|
+
gates[1].release();
|
|
271
|
+
await stopping;
|
|
272
|
+
expect(isStopped).toBe(true);
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
it("rejects a concurrency that is not a positive integer", () => {
|
|
276
|
+
for (const concurrency of [0, -1, 1.5]) {
|
|
277
|
+
expect(() => new InProcessRunner({concurrency})).toThrow(
|
|
278
|
+
harnessErrorMatching(
|
|
279
|
+
"configInvalid",
|
|
280
|
+
"InProcessRunner concurrency must be a positive integer"
|
|
281
|
+
)
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
});
|
|
285
|
+
});
|
|
286
|
+
|
|
159
287
|
it("rejects a second start while running", async () => {
|
|
160
288
|
const context = fakeContext({claimNext: async () => null, runTask: async () => {}});
|
|
161
289
|
const runner = new InProcessRunner({pollInterval: {milliseconds: 5}});
|
|
@@ -3,10 +3,17 @@ import {hostname} from "node:os";
|
|
|
3
3
|
import {logger} from "@terreno/api";
|
|
4
4
|
import {Duration, type DurationLike} from "luxon";
|
|
5
5
|
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
HarnessLeaseSettings,
|
|
8
|
+
HarnessRunner,
|
|
9
|
+
HarnessRunnerContext,
|
|
10
|
+
HarnessTaskDocument,
|
|
11
|
+
} from "../../types/harness";
|
|
7
12
|
import {errorMessage, harnessError} from "../errors";
|
|
8
13
|
|
|
9
14
|
export interface InProcessRunnerOptions {
|
|
15
|
+
/** Most claimed tasks this runner executes at once. Default 8. */
|
|
16
|
+
concurrency?: number;
|
|
10
17
|
/** How often the runner renews its owner lease and its running task's lease. */
|
|
11
18
|
heartbeatInterval?: DurationLike;
|
|
12
19
|
/** How long the owner lease and each task lease live without a heartbeat. */
|
|
@@ -28,16 +35,19 @@ export type InProcessRunnerRole =
|
|
|
28
35
|
(typeof IN_PROCESS_RUNNER_ROLES)[keyof typeof IN_PROCESS_RUNNER_ROLES];
|
|
29
36
|
|
|
30
37
|
/**
|
|
31
|
-
* Executes runnable tasks inside this process,
|
|
38
|
+
* Executes runnable tasks inside this process, up to `concurrency` at a time, each until it
|
|
39
|
+
* stops (terminal, retry, waiting, or aborted).
|
|
32
40
|
* Only the process holding the singleton `HarnessOwner` lease drains; every other
|
|
33
41
|
* process stays on standby and takes over once that lease expires. On becoming owner,
|
|
34
42
|
* and on every heartbeat while owner, it recovers tasks whose lease expired.
|
|
35
43
|
*/
|
|
36
44
|
export class InProcessRunner implements HarnessRunner {
|
|
45
|
+
private readonly concurrency: number;
|
|
37
46
|
private context: HarnessRunnerContext | undefined;
|
|
38
47
|
private heartbeatInFlight: Promise<void> = Promise.resolve();
|
|
39
48
|
private heartbeatTimer: ReturnType<typeof setTimeout> | undefined;
|
|
40
49
|
private hasPendingWake = false;
|
|
50
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
41
51
|
private isHeartbeatStopped = false;
|
|
42
52
|
private isStopping = false;
|
|
43
53
|
private readonly lease: HarnessLeaseSettings;
|
|
@@ -55,6 +65,14 @@ export class InProcessRunner implements HarnessRunner {
|
|
|
55
65
|
kind: "configInvalid",
|
|
56
66
|
});
|
|
57
67
|
}
|
|
68
|
+
const concurrency = options.concurrency ?? 8;
|
|
69
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
70
|
+
throw harnessError({
|
|
71
|
+
detail: "InProcessRunner concurrency must be a positive integer",
|
|
72
|
+
kind: "configInvalid",
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
this.concurrency = concurrency;
|
|
58
76
|
this.lease = {
|
|
59
77
|
duration,
|
|
60
78
|
heartbeat,
|
|
@@ -87,14 +105,14 @@ export class InProcessRunner implements HarnessRunner {
|
|
|
87
105
|
this.scheduleHeartbeat();
|
|
88
106
|
}
|
|
89
107
|
|
|
90
|
-
/** Stop claiming new work, wait for
|
|
108
|
+
/** Stop claiming new work, wait for every task in flight, then release the owner lease. */
|
|
91
109
|
async stop(): Promise<void> {
|
|
92
110
|
if (this.loop === undefined) {
|
|
93
111
|
return;
|
|
94
112
|
}
|
|
95
113
|
this.isStopping = true;
|
|
96
114
|
this.wake();
|
|
97
|
-
// Keep renewing the owner lease until the
|
|
115
|
+
// Keep renewing the owner lease until the tasks in flight settle, so a standby does
|
|
98
116
|
// not take over while this process is still working.
|
|
99
117
|
await this.loop;
|
|
100
118
|
this.isHeartbeatStopped = true;
|
|
@@ -132,11 +150,13 @@ export class InProcessRunner implements HarnessRunner {
|
|
|
132
150
|
continue;
|
|
133
151
|
}
|
|
134
152
|
}
|
|
135
|
-
|
|
136
|
-
if (
|
|
153
|
+
await this.fillSlots(this.context);
|
|
154
|
+
if (!this.isStopping) {
|
|
155
|
+
// Woken early by new work or by a task that frees a slot.
|
|
137
156
|
await this.sleep(this.pollIntervalMs);
|
|
138
157
|
}
|
|
139
158
|
}
|
|
159
|
+
await Promise.allSettled([...this.inFlight]);
|
|
140
160
|
}
|
|
141
161
|
|
|
142
162
|
/** Become owner when the lease is free or expired, then recover abandoned tasks. */
|
|
@@ -155,18 +175,38 @@ export class InProcessRunner implements HarnessRunner {
|
|
|
155
175
|
}
|
|
156
176
|
}
|
|
157
177
|
|
|
158
|
-
/**
|
|
159
|
-
private async
|
|
160
|
-
|
|
161
|
-
|
|
178
|
+
/** Claim and start tasks until every slot is busy or nothing is runnable. */
|
|
179
|
+
private async fillSlots(context: HarnessRunnerContext): Promise<void> {
|
|
180
|
+
while (
|
|
181
|
+
this.inFlight.size < this.concurrency &&
|
|
182
|
+
!this.isStopping &&
|
|
183
|
+
this.currentRole === IN_PROCESS_RUNNER_ROLES.owner
|
|
184
|
+
) {
|
|
185
|
+
let task: HarnessTaskDocument | null;
|
|
186
|
+
try {
|
|
187
|
+
task = await context.claimNext(this.lease);
|
|
188
|
+
} catch (error: unknown) {
|
|
189
|
+
logger.error(`InProcessRunner iteration failed: ${errorMessage(error)}`);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
162
192
|
if (!task) {
|
|
163
|
-
return
|
|
193
|
+
return;
|
|
164
194
|
}
|
|
195
|
+
const run = this.runOne(context, task);
|
|
196
|
+
this.inFlight.add(run);
|
|
197
|
+
void run.finally(() => {
|
|
198
|
+
this.inFlight.delete(run);
|
|
199
|
+
this.wake();
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Run one claimed task to a stop; its own task lease and heartbeat guard its commits. */
|
|
205
|
+
private async runOne(context: HarnessRunnerContext, task: HarnessTaskDocument): Promise<void> {
|
|
206
|
+
try {
|
|
165
207
|
await context.runTask(task, this.lease);
|
|
166
|
-
return false;
|
|
167
208
|
} catch (error: unknown) {
|
|
168
|
-
logger.error(`InProcessRunner
|
|
169
|
-
return true;
|
|
209
|
+
logger.error(`InProcessRunner task ${task._id} failed to run: ${errorMessage(error)}`);
|
|
170
210
|
}
|
|
171
211
|
}
|
|
172
212
|
|