@terreno/ai 57.10.1 → 57.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/aiApp.d.ts +5 -0
  2. package/dist/aiApp.js +4 -2
  3. package/dist/harness/commit.d.ts +17 -1
  4. package/dist/harness/commit.js +86 -2
  5. package/dist/harness/errors.d.ts +6 -0
  6. package/dist/harness/errors.js +6 -0
  7. package/dist/harness/harness.d.ts +2 -2
  8. package/dist/harness/harness.js +4 -2
  9. package/dist/harness/models/harnessTask.js +5 -0
  10. package/dist/harness/routes/approvals.js +2 -2
  11. package/dist/harness/routes/conversations.js +1 -1
  12. package/dist/harness/routes/tasks.js +85 -3
  13. package/dist/harness/runners/inProcessRunner.d.ts +11 -4
  14. package/dist/harness/runners/inProcessRunner.js +44 -13
  15. package/dist/harness/runners/jobsRunner.d.ts +62 -0
  16. package/dist/harness/runners/jobsRunner.js +212 -0
  17. package/dist/harness/runtime.d.ts +20 -2
  18. package/dist/harness/runtime.js +91 -32
  19. package/dist/harness/subagent.d.ts +17 -2
  20. package/dist/harness/subagent.js +26 -2
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.js +3 -2
  23. package/dist/models/project.d.ts +13 -0
  24. package/dist/models/project.js +32 -2
  25. package/dist/observability/local/models/obsTrace.js +16 -2
  26. package/dist/observability/local/traceStore.d.ts +5 -0
  27. package/dist/observability/local/traceStore.js +5 -0
  28. package/dist/observability/routes/traces.js +2 -0
  29. package/dist/routes/projects.js +3 -3
  30. package/dist/service/chatTurn.js +1 -1
  31. package/dist/types/harness.d.ts +47 -3
  32. package/dist/types/observability.d.ts +3 -0
  33. package/package.json +65 -6
  34. package/src/aiApp.test.ts +15 -0
  35. package/src/aiApp.ts +9 -1
  36. package/src/harness/commit.ts +103 -1
  37. package/src/harness/concurrency.test.ts +155 -0
  38. package/src/harness/errors.ts +6 -0
  39. package/src/harness/events.test.ts +17 -5
  40. package/src/harness/harness.ts +14 -3
  41. package/src/harness/models/harnessTask.ts +5 -0
  42. package/src/harness/ownership.test.ts +9 -9
  43. package/src/harness/recovery.test.ts +2 -7
  44. package/src/harness/routes/approvals.ts +2 -2
  45. package/src/harness/routes/conversations.ts +1 -1
  46. package/src/harness/routes/tasks.test.ts +34 -0
  47. package/src/harness/routes/tasks.ts +98 -3
  48. package/src/harness/runners/inProcessRunner.test.ts +128 -0
  49. package/src/harness/runners/inProcessRunner.ts +54 -14
  50. package/src/harness/runners/jobsRunner.test.ts +709 -0
  51. package/src/harness/runners/jobsRunner.ts +252 -0
  52. package/src/harness/runtime.ts +125 -30
  53. package/src/harness/subagent.test.ts +124 -3
  54. package/src/harness/subagent.ts +43 -1
  55. package/src/index.ts +1 -1
  56. package/src/isolated/lazyProjectModel.isolated.ts +25 -0
  57. package/src/models/project.ts +31 -1
  58. package/src/observability/local/models/obsTrace.ts +22 -5
  59. package/src/observability/local/traceStore.test.ts +7 -0
  60. package/src/observability/local/traceStore.ts +12 -0
  61. package/src/observability/routes/traces.test.ts +15 -0
  62. package/src/observability/routes/traces.ts +2 -0
  63. package/src/routes/projects.ts +4 -4
  64. package/src/service/chatTurn.ts +2 -2
  65. package/src/types/harness.ts +56 -3
  66. package/src/types/observability.ts +3 -0
@@ -12,6 +12,8 @@ const fakeTask = (id: string): HarnessTaskDocument => ({_id: id}) as unknown as
12
12
  const fakeContext = (overrides: Partial<HarnessRunnerContext>): HarnessRunnerContext => ({
13
13
  acquireOwnerLease: async () => true,
14
14
  claimNext: async () => null,
15
+ claimTask: async () => null,
16
+ listRunnable: async () => [],
15
17
  recoverExpired: async () => 0,
16
18
  releaseOwnerLease: async () => {},
17
19
  runTask: async () => {},
@@ -156,6 +158,132 @@ describe("InProcessRunner", () => {
156
158
  expect(isStopped).toBe(true);
157
159
  });
158
160
 
161
+ describe("concurrency", () => {
162
+ const blockingTask = (): {done: Promise<void>; release: () => void} => {
163
+ let release: () => void = () => {};
164
+ const done = new Promise<void>((resolve) => {
165
+ release = resolve;
166
+ });
167
+ return {done, release};
168
+ };
169
+
170
+ it("defaults to 8 tasks at a time", async () => {
171
+ const gates = Array.from({length: 10}, () => blockingTask());
172
+ const queue = gates.map((_gate, index) => fakeTask(String(index)));
173
+ let running = 0;
174
+ let peak = 0;
175
+ const context = fakeContext({
176
+ claimNext: async () => queue.shift() ?? null,
177
+ runTask: async (task) => {
178
+ running += 1;
179
+ peak = Math.max(peak, running);
180
+ await gates[Number(task._id)].done;
181
+ running -= 1;
182
+ },
183
+ });
184
+ const runner = new InProcessRunner({pollInterval: {milliseconds: 5}});
185
+ await runner.start(context);
186
+ await waitUntil(() => running === 8);
187
+ await new Promise((resolve) => setTimeout(resolve, 20));
188
+
189
+ expect(running).toBe(8);
190
+ expect(queue.length).toBe(2);
191
+ for (const gate of gates) {
192
+ gate.release();
193
+ }
194
+ await waitUntil(() => queue.length === 0 && running === 0);
195
+ await runner.stop();
196
+ expect(peak).toBe(8);
197
+ });
198
+
199
+ it("lets a second task complete while the first is blocked on a gate", async () => {
200
+ const slow = blockingTask();
201
+ const finished: string[] = [];
202
+ const queue = [fakeTask("slow"), fakeTask("fast")];
203
+ const context = fakeContext({
204
+ claimNext: async () => queue.shift() ?? null,
205
+ runTask: async (task) => {
206
+ if (String(task._id) === "slow") {
207
+ await slow.done;
208
+ }
209
+ finished.push(String(task._id));
210
+ },
211
+ });
212
+ const runner = new InProcessRunner({concurrency: 2, pollInterval: {minutes: 10}});
213
+ await runner.start(context);
214
+
215
+ await waitUntil(() => finished.length === 1);
216
+ expect(finished).toEqual(["fast"]);
217
+ slow.release();
218
+ await waitUntil(() => finished.length === 2);
219
+ await runner.stop();
220
+ expect(finished).toEqual(["fast", "slow"]);
221
+ });
222
+
223
+ it("never runs more than concurrency tasks and claims again as soon as a slot frees", async () => {
224
+ const queue = ["a", "b", "c", "d", "e"].map(fakeTask);
225
+ const ran: string[] = [];
226
+ let running = 0;
227
+ let peak = 0;
228
+ const context = fakeContext({
229
+ claimNext: async () => queue.shift() ?? null,
230
+ runTask: async (task) => {
231
+ running += 1;
232
+ peak = Math.max(peak, running);
233
+ await new Promise((resolve) => setTimeout(resolve, 15));
234
+ ran.push(String(task._id));
235
+ running -= 1;
236
+ },
237
+ });
238
+ // Only a freed slot (not the 10-minute poll) can pick up the rest.
239
+ const runner = new InProcessRunner({concurrency: 2, pollInterval: {minutes: 10}});
240
+ await runner.start(context);
241
+ await waitUntil(() => ran.length === 5);
242
+ await runner.stop();
243
+
244
+ expect(peak).toBe(2);
245
+ expect([...ran].sort()).toEqual(["a", "b", "c", "d", "e"]);
246
+ });
247
+
248
+ it("waits for every task in flight before stop resolves", async () => {
249
+ const gates = [blockingTask(), blockingTask()];
250
+ const queue = [fakeTask("0"), fakeTask("1")];
251
+ let started = 0;
252
+ let isStopped = false;
253
+ const context = fakeContext({
254
+ claimNext: async () => queue.shift() ?? null,
255
+ runTask: async (task) => {
256
+ started += 1;
257
+ await gates[Number(task._id)].done;
258
+ },
259
+ });
260
+ const runner = new InProcessRunner({concurrency: 2, pollInterval: {milliseconds: 5}});
261
+ await runner.start(context);
262
+ await waitUntil(() => started === 2);
263
+
264
+ const stopping = runner.stop().then(() => {
265
+ isStopped = true;
266
+ });
267
+ gates[0].release();
268
+ await new Promise((resolve) => setTimeout(resolve, 20));
269
+ expect(isStopped).toBe(false);
270
+ gates[1].release();
271
+ await stopping;
272
+ expect(isStopped).toBe(true);
273
+ });
274
+
275
+ it("rejects a concurrency that is not a positive integer", () => {
276
+ for (const concurrency of [0, -1, 1.5]) {
277
+ expect(() => new InProcessRunner({concurrency})).toThrow(
278
+ harnessErrorMatching(
279
+ "configInvalid",
280
+ "InProcessRunner concurrency must be a positive integer"
281
+ )
282
+ );
283
+ }
284
+ });
285
+ });
286
+
159
287
  it("rejects a second start while running", async () => {
160
288
  const context = fakeContext({claimNext: async () => null, runTask: async () => {}});
161
289
  const runner = new InProcessRunner({pollInterval: {milliseconds: 5}});
@@ -3,10 +3,17 @@ import {hostname} from "node:os";
3
3
  import {logger} from "@terreno/api";
4
4
  import {Duration, type DurationLike} from "luxon";
5
5
 
6
- import type {HarnessLeaseSettings, HarnessRunner, HarnessRunnerContext} from "../../types/harness";
6
+ import type {
7
+ HarnessLeaseSettings,
8
+ HarnessRunner,
9
+ HarnessRunnerContext,
10
+ HarnessTaskDocument,
11
+ } from "../../types/harness";
7
12
  import {errorMessage, harnessError} from "../errors";
8
13
 
9
14
  export interface InProcessRunnerOptions {
15
+ /** Most claimed tasks this runner executes at once. Default 8. */
16
+ concurrency?: number;
10
17
  /** How often the runner renews its owner lease and its running task's lease. */
11
18
  heartbeatInterval?: DurationLike;
12
19
  /** How long the owner lease and each task lease live without a heartbeat. */
@@ -28,16 +35,19 @@ export type InProcessRunnerRole =
28
35
  (typeof IN_PROCESS_RUNNER_ROLES)[keyof typeof IN_PROCESS_RUNNER_ROLES];
29
36
 
30
37
  /**
31
- * Executes runnable tasks inside this process, one at a time, until each is terminal.
38
+ * Executes runnable tasks inside this process, up to `concurrency` at a time, each until it
39
+ * stops (terminal, retry, waiting, or aborted).
32
40
  * Only the process holding the singleton `HarnessOwner` lease drains; every other
33
41
  * process stays on standby and takes over once that lease expires. On becoming owner,
34
42
  * and on every heartbeat while owner, it recovers tasks whose lease expired.
35
43
  */
36
44
  export class InProcessRunner implements HarnessRunner {
45
+ private readonly concurrency: number;
37
46
  private context: HarnessRunnerContext | undefined;
38
47
  private heartbeatInFlight: Promise<void> = Promise.resolve();
39
48
  private heartbeatTimer: ReturnType<typeof setTimeout> | undefined;
40
49
  private hasPendingWake = false;
50
+ private readonly inFlight = new Set<Promise<void>>();
41
51
  private isHeartbeatStopped = false;
42
52
  private isStopping = false;
43
53
  private readonly lease: HarnessLeaseSettings;
@@ -55,6 +65,14 @@ export class InProcessRunner implements HarnessRunner {
55
65
  kind: "configInvalid",
56
66
  });
57
67
  }
68
+ const concurrency = options.concurrency ?? 8;
69
+ if (!Number.isInteger(concurrency) || concurrency < 1) {
70
+ throw harnessError({
71
+ detail: "InProcessRunner concurrency must be a positive integer",
72
+ kind: "configInvalid",
73
+ });
74
+ }
75
+ this.concurrency = concurrency;
58
76
  this.lease = {
59
77
  duration,
60
78
  heartbeat,
@@ -87,14 +105,14 @@ export class InProcessRunner implements HarnessRunner {
87
105
  this.scheduleHeartbeat();
88
106
  }
89
107
 
90
- /** Stop claiming new work, wait for the task in flight, then release the owner lease. */
108
+ /** Stop claiming new work, wait for every task in flight, then release the owner lease. */
91
109
  async stop(): Promise<void> {
92
110
  if (this.loop === undefined) {
93
111
  return;
94
112
  }
95
113
  this.isStopping = true;
96
114
  this.wake();
97
- // Keep renewing the owner lease until the task in flight settles, so a standby does
115
+ // Keep renewing the owner lease until the tasks in flight settle, so a standby does
98
116
  // not take over while this process is still working.
99
117
  await this.loop;
100
118
  this.isHeartbeatStopped = true;
@@ -132,11 +150,13 @@ export class InProcessRunner implements HarnessRunner {
132
150
  continue;
133
151
  }
134
152
  }
135
- const isIdle = await this.runNext(this.context);
136
- if (isIdle && !this.isStopping) {
153
+ await this.fillSlots(this.context);
154
+ if (!this.isStopping) {
155
+ // Woken early by new work or by a task that frees a slot.
137
156
  await this.sleep(this.pollIntervalMs);
138
157
  }
139
158
  }
159
+ await Promise.allSettled([...this.inFlight]);
140
160
  }
141
161
 
142
162
  /** Become owner when the lease is free or expired, then recover abandoned tasks. */
@@ -155,18 +175,38 @@ export class InProcessRunner implements HarnessRunner {
155
175
  }
156
176
  }
157
177
 
158
- /** Run one claimed task; returns true when nothing was runnable. */
159
- private async runNext(context: HarnessRunnerContext): Promise<boolean> {
160
- try {
161
- const task = await context.claimNext(this.lease);
178
+ /** Claim and start tasks until every slot is busy or nothing is runnable. */
179
+ private async fillSlots(context: HarnessRunnerContext): Promise<void> {
180
+ while (
181
+ this.inFlight.size < this.concurrency &&
182
+ !this.isStopping &&
183
+ this.currentRole === IN_PROCESS_RUNNER_ROLES.owner
184
+ ) {
185
+ let task: HarnessTaskDocument | null;
186
+ try {
187
+ task = await context.claimNext(this.lease);
188
+ } catch (error: unknown) {
189
+ logger.error(`InProcessRunner iteration failed: ${errorMessage(error)}`);
190
+ return;
191
+ }
162
192
  if (!task) {
163
- return true;
193
+ return;
164
194
  }
195
+ const run = this.runOne(context, task);
196
+ this.inFlight.add(run);
197
+ void run.finally(() => {
198
+ this.inFlight.delete(run);
199
+ this.wake();
200
+ });
201
+ }
202
+ }
203
+
204
+ /** Run one claimed task to a stop; its own task lease and heartbeat guard its commits. */
205
+ private async runOne(context: HarnessRunnerContext, task: HarnessTaskDocument): Promise<void> {
206
+ try {
165
207
  await context.runTask(task, this.lease);
166
- return false;
167
208
  } catch (error: unknown) {
168
- logger.error(`InProcessRunner iteration failed: ${errorMessage(error)}`);
169
- return true;
209
+ logger.error(`InProcessRunner task ${task._id} failed to run: ${errorMessage(error)}`);
170
210
  }
171
211
  }
172
212