@nylorun/runtime 0.49.2-beta → 0.51.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/api/components.d.ts +108 -0
- package/dist/gates/meter.js +5 -1
- package/dist/harness/main.js +13 -1
- package/dist/host/stack-config.d.ts +6 -0
- package/dist/host/stack-config.js +1 -0
- package/dist/management-openapi.json +1 -1
- package/dist/model/pi-model.js +27 -8
- package/dist/model/retry.d.ts +14 -0
- package/dist/model/retry.js +34 -0
- package/dist/openapi.json +88 -1
- package/dist/sandbox/pods/client.d.ts +5 -0
- package/dist/sandbox/pods/lifecycle.d.ts +9 -0
- package/dist/sandbox/pods/lifecycle.js +16 -0
- package/dist/store/postgres/drizzle/0047_model_usage_attempts.sql +2 -0
- package/dist/store/postgres/drizzle/0048_sandbox_pod_drained_at.sql +1 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +14 -0
- package/dist/store/postgres/schema.d.ts +59 -2
- package/dist/store/postgres/schema.js +12 -1
- package/dist/store/postgres/store.js +2 -0
- package/dist/store/types.d.ts +5 -0
- package/dist/tenant/transcript.d.ts +12 -8
- package/dist/tenant/transcript.js +4 -0
- package/dist/traces/map.js +2 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +16 -16
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.51.0-beta
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 874cc83: Pods on Agent Substrate (preview): suspend before drain, following Substrate's eviction contract. A worker pod that stops (a drain, a GKE node upgrade or scale-down, a WorkerPool rollout, a deleted pod) sends each of its actors SIGTERM and gives them 30 minutes to be suspended; until now the engine exited on that signal, so the actor crashed and went back to its last suspend, losing everything since.
|
|
8
|
+
|
|
9
|
+
- `@nylorun/runtime`: on Substrate (`NYLORUN_SANDBOX_SUBSTRATE=1`, which the sandboxes service's template sets), the engine logs `harness_worker_stopping` on SIGTERM and keeps running, with its runs. The sandboxes service watches worker pods (label `ate.dev/worker-pool`) and, when one turns Terminating, suspends each of the Tenant's actors on it synchronously (eight at a time, trying again for up to 29 minutes), records `drainedAt`, and resumes a sandbox whose request is `Running` on another worker. While it moves, its status reason is `Moving`, and the pod lifecycle waits instead of failing it at the ready deadline. Each drain is recorded once, as `agent.suspended` with reason `drain` (the pod row's `drained_at`, store migration 0048, nullable, run at start). Not covered: a spot reclaim or a node that dies, which still crash and revert. An actor created by an earlier release keeps its engine (#643) until it is reset.
|
|
10
|
+
- `@nylorun/core`: `agent.suspended`'s `reason` may be `drain`.
|
|
11
|
+
- `nylorun`: `local sandbox enable --backend substrate` lets the sandboxes service read pods cluster-wide (`get`, `list`, `watch`), for the worker pods it watches.
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- Pin core to the tested release.
|
|
16
|
+
- Pin harness to the tested release.
|
|
17
|
+
- Updated dependencies [874cc83]
|
|
18
|
+
- Updated dependencies
|
|
19
|
+
- @nylorun/core@0.44.0-beta
|
|
20
|
+
- @nylorun/harness@0.36.4-beta
|
|
21
|
+
|
|
22
|
+
## 0.50.0-beta
|
|
23
|
+
|
|
24
|
+
### Minor Changes
|
|
25
|
+
|
|
26
|
+
- 745ef22: A model call's retries are in its event. `message.assistant` and `model.failed` carry `attempts`, the provider calls the gate made for the call (the first included), and `retries`, one `{ attempt, delayMs, error }` for each retry (the error as the provider worded it, cut to 300 characters, with the provider's credential scrubbed). Before, a call that answered after a retry was one `message.assistant` that showed no sign of it, and a call that failed after its attempts showed only its last error; the retries were only in the gates service's `model_retry` log lines.
|
|
27
|
+
|
|
28
|
+
- `@nylorun/core`: `ModelRetrySchema`, and the optional `attempts` and `retries` on the facts `message.assistant`, `model.failed` and `context.compacted` share. `ModelFailureOutcome` takes optional `attempts` and `retries`.
|
|
29
|
+
- `@nylorun/runtime`: the gate's answer reports them in `evidence.extras`, the usage ledger keeps them (`model_usage.attempts` and `retries`, store migration 0047, nullable, run at start), and the call's event takes them from its row; a failed call's from its failure outcome. A trace's chat span has `nylorun.model.attempts`. A retried call is still metered once, as one call. `/calls/model` is unchanged.
|
|
30
|
+
|
|
31
|
+
### Patch Changes
|
|
32
|
+
|
|
33
|
+
- Pin core to the tested release.
|
|
34
|
+
- Pin harness to the tested release.
|
|
35
|
+
- Updated dependencies [745ef22]
|
|
36
|
+
- Updated dependencies
|
|
37
|
+
- @nylorun/core@0.43.0-beta
|
|
38
|
+
- @nylorun/harness@0.36.3-beta
|
|
39
|
+
|
|
3
40
|
## 0.49.2-beta
|
|
4
41
|
|
|
5
42
|
### Patch Changes
|
package/dist/api/components.d.ts
CHANGED
|
@@ -3502,6 +3502,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3502
3502
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
3503
3503
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
3504
3504
|
}, z.core.$loose>>;
|
|
3505
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
3506
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
3507
|
+
attempt: z.ZodNumber;
|
|
3508
|
+
delayMs: z.ZodNumber;
|
|
3509
|
+
error: z.ZodString;
|
|
3510
|
+
}, z.core.$loose>>>;
|
|
3505
3511
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
3506
3512
|
trigger: z.ZodEnum<{
|
|
3507
3513
|
overflow: "overflow";
|
|
@@ -4363,6 +4369,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
4363
4369
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
4364
4370
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
4365
4371
|
}, z.core.$loose>>;
|
|
4372
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
4373
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4374
|
+
attempt: z.ZodNumber;
|
|
4375
|
+
delayMs: z.ZodNumber;
|
|
4376
|
+
error: z.ZodString;
|
|
4377
|
+
}, z.core.$loose>>>;
|
|
4366
4378
|
invocationId: z.ZodString;
|
|
4367
4379
|
text: z.ZodString;
|
|
4368
4380
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -4540,6 +4552,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
4540
4552
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
4541
4553
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
4542
4554
|
}, z.core.$loose>>;
|
|
4555
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
4556
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4557
|
+
attempt: z.ZodNumber;
|
|
4558
|
+
delayMs: z.ZodNumber;
|
|
4559
|
+
error: z.ZodString;
|
|
4560
|
+
}, z.core.$loose>>>;
|
|
4543
4561
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
4544
4562
|
code: z.ZodString;
|
|
4545
4563
|
message: z.ZodString;
|
|
@@ -6735,6 +6753,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
6735
6753
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
6736
6754
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
6737
6755
|
}, z.core.$loose>>;
|
|
6756
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
6757
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
6758
|
+
attempt: z.ZodNumber;
|
|
6759
|
+
delayMs: z.ZodNumber;
|
|
6760
|
+
error: z.ZodString;
|
|
6761
|
+
}, z.core.$loose>>>;
|
|
6738
6762
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
6739
6763
|
trigger: z.ZodEnum<{
|
|
6740
6764
|
overflow: "overflow";
|
|
@@ -7596,6 +7620,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7596
7620
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
7597
7621
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
7598
7622
|
}, z.core.$loose>>;
|
|
7623
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
7624
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
7625
|
+
attempt: z.ZodNumber;
|
|
7626
|
+
delayMs: z.ZodNumber;
|
|
7627
|
+
error: z.ZodString;
|
|
7628
|
+
}, z.core.$loose>>>;
|
|
7599
7629
|
invocationId: z.ZodString;
|
|
7600
7630
|
text: z.ZodString;
|
|
7601
7631
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -7773,6 +7803,12 @@ export declare const SessionEvent: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7773
7803
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
7774
7804
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
7775
7805
|
}, z.core.$loose>>;
|
|
7806
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
7807
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
7808
|
+
attempt: z.ZodNumber;
|
|
7809
|
+
delayMs: z.ZodNumber;
|
|
7810
|
+
error: z.ZodString;
|
|
7811
|
+
}, z.core.$loose>>>;
|
|
7776
7812
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
7777
7813
|
code: z.ZodString;
|
|
7778
7814
|
message: z.ZodString;
|
|
@@ -9971,6 +10007,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
9971
10007
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
9972
10008
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
9973
10009
|
}, z.core.$loose>>;
|
|
10010
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
10011
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
10012
|
+
attempt: z.ZodNumber;
|
|
10013
|
+
delayMs: z.ZodNumber;
|
|
10014
|
+
error: z.ZodString;
|
|
10015
|
+
}, z.core.$loose>>>;
|
|
9974
10016
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
9975
10017
|
trigger: z.ZodEnum<{
|
|
9976
10018
|
overflow: "overflow";
|
|
@@ -10832,6 +10874,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
10832
10874
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
10833
10875
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
10834
10876
|
}, z.core.$loose>>;
|
|
10877
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
10878
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
10879
|
+
attempt: z.ZodNumber;
|
|
10880
|
+
delayMs: z.ZodNumber;
|
|
10881
|
+
error: z.ZodString;
|
|
10882
|
+
}, z.core.$loose>>>;
|
|
10835
10883
|
invocationId: z.ZodString;
|
|
10836
10884
|
text: z.ZodString;
|
|
10837
10885
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -11009,6 +11057,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
11009
11057
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
11010
11058
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
11011
11059
|
}, z.core.$loose>>;
|
|
11060
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
11061
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
11062
|
+
attempt: z.ZodNumber;
|
|
11063
|
+
delayMs: z.ZodNumber;
|
|
11064
|
+
error: z.ZodString;
|
|
11065
|
+
}, z.core.$loose>>>;
|
|
11012
11066
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
11013
11067
|
code: z.ZodString;
|
|
11014
11068
|
message: z.ZodString;
|
|
@@ -13204,6 +13258,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
13204
13258
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
13205
13259
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
13206
13260
|
}, z.core.$loose>>;
|
|
13261
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
13262
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
13263
|
+
attempt: z.ZodNumber;
|
|
13264
|
+
delayMs: z.ZodNumber;
|
|
13265
|
+
error: z.ZodString;
|
|
13266
|
+
}, z.core.$loose>>>;
|
|
13207
13267
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
13208
13268
|
trigger: z.ZodEnum<{
|
|
13209
13269
|
overflow: "overflow";
|
|
@@ -14065,6 +14125,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
14065
14125
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
14066
14126
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
14067
14127
|
}, z.core.$loose>>;
|
|
14128
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
14129
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
14130
|
+
attempt: z.ZodNumber;
|
|
14131
|
+
delayMs: z.ZodNumber;
|
|
14132
|
+
error: z.ZodString;
|
|
14133
|
+
}, z.core.$loose>>>;
|
|
14068
14134
|
invocationId: z.ZodString;
|
|
14069
14135
|
text: z.ZodString;
|
|
14070
14136
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -14242,6 +14308,12 @@ export declare const SessionItemsResponse: z.ZodObject<{
|
|
|
14242
14308
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
14243
14309
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
14244
14310
|
}, z.core.$loose>>;
|
|
14311
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
14312
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
14313
|
+
attempt: z.ZodNumber;
|
|
14314
|
+
delayMs: z.ZodNumber;
|
|
14315
|
+
error: z.ZodString;
|
|
14316
|
+
}, z.core.$loose>>>;
|
|
14245
14317
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
14246
14318
|
code: z.ZodString;
|
|
14247
14319
|
message: z.ZodString;
|
|
@@ -17390,6 +17462,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
17390
17462
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
17391
17463
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
17392
17464
|
}, z.core.$loose>>;
|
|
17465
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
17466
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
17467
|
+
attempt: z.ZodNumber;
|
|
17468
|
+
delayMs: z.ZodNumber;
|
|
17469
|
+
error: z.ZodString;
|
|
17470
|
+
}, z.core.$loose>>>;
|
|
17393
17471
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
17394
17472
|
trigger: z.ZodEnum<{
|
|
17395
17473
|
overflow: "overflow";
|
|
@@ -18251,6 +18329,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
18251
18329
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
18252
18330
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
18253
18331
|
}, z.core.$loose>>;
|
|
18332
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
18333
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
18334
|
+
attempt: z.ZodNumber;
|
|
18335
|
+
delayMs: z.ZodNumber;
|
|
18336
|
+
error: z.ZodString;
|
|
18337
|
+
}, z.core.$loose>>>;
|
|
18254
18338
|
invocationId: z.ZodString;
|
|
18255
18339
|
text: z.ZodString;
|
|
18256
18340
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -18428,6 +18512,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
18428
18512
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
18429
18513
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
18430
18514
|
}, z.core.$loose>>;
|
|
18515
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
18516
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
18517
|
+
attempt: z.ZodNumber;
|
|
18518
|
+
delayMs: z.ZodNumber;
|
|
18519
|
+
error: z.ZodString;
|
|
18520
|
+
}, z.core.$loose>>>;
|
|
18431
18521
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
18432
18522
|
code: z.ZodString;
|
|
18433
18523
|
message: z.ZodString;
|
|
@@ -20623,6 +20713,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
20623
20713
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
20624
20714
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
20625
20715
|
}, z.core.$loose>>;
|
|
20716
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
20717
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
20718
|
+
attempt: z.ZodNumber;
|
|
20719
|
+
delayMs: z.ZodNumber;
|
|
20720
|
+
error: z.ZodString;
|
|
20721
|
+
}, z.core.$loose>>>;
|
|
20626
20722
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
20627
20723
|
trigger: z.ZodEnum<{
|
|
20628
20724
|
overflow: "overflow";
|
|
@@ -21484,6 +21580,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
21484
21580
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
21485
21581
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
21486
21582
|
}, z.core.$loose>>;
|
|
21583
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
21584
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
21585
|
+
attempt: z.ZodNumber;
|
|
21586
|
+
delayMs: z.ZodNumber;
|
|
21587
|
+
error: z.ZodString;
|
|
21588
|
+
}, z.core.$loose>>>;
|
|
21487
21589
|
invocationId: z.ZodString;
|
|
21488
21590
|
text: z.ZodString;
|
|
21489
21591
|
model: z.ZodOptional<z.ZodObject<{
|
|
@@ -21661,6 +21763,12 @@ export declare const HistoryPage: z.ZodObject<{
|
|
|
21661
21763
|
cachedTokens: z.ZodOptional<z.ZodNumber>;
|
|
21662
21764
|
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
21663
21765
|
}, z.core.$loose>>;
|
|
21766
|
+
attempts: z.ZodOptional<z.ZodNumber>;
|
|
21767
|
+
retries: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
21768
|
+
attempt: z.ZodNumber;
|
|
21769
|
+
delayMs: z.ZodNumber;
|
|
21770
|
+
error: z.ZodString;
|
|
21771
|
+
}, z.core.$loose>>>;
|
|
21664
21772
|
invocationId: z.ZodOptional<z.ZodString>;
|
|
21665
21773
|
code: z.ZodString;
|
|
21666
21774
|
message: z.ZodString;
|
package/dist/gates/meter.js
CHANGED
|
@@ -11,10 +11,12 @@
|
|
|
11
11
|
*
|
|
12
12
|
* The row also keeps the gate's own timing of the call (R7.1): `now()` before the call and
|
|
13
13
|
* after it, and the adapter's `firstTokenMs` (`evidence.extras`), so a model call's span is
|
|
14
|
-
* what the gate measured
|
|
14
|
+
* what the gate measured; and the adapter's `attempts` and `retries`, so the call's event says
|
|
15
|
+
* how many times the provider was called for it, and why again.
|
|
15
16
|
*/
|
|
16
17
|
import { randomUUID } from "node:crypto";
|
|
17
18
|
import { failure } from "../model/classify.js";
|
|
19
|
+
import { attemptFacts } from "../model/retry.js";
|
|
18
20
|
export function createMeter(options) {
|
|
19
21
|
const now = options.now ?? (() => new Date());
|
|
20
22
|
/** Calls running through this gate, per Tenant and budget scope. */
|
|
@@ -143,6 +145,8 @@ export function usageRow(request, outcome, at, timing) {
|
|
|
143
145
|
? { firstTokenMs: Math.round(firstTokenMs) }
|
|
144
146
|
: {}),
|
|
145
147
|
...costSplitColumns(outcome.evidence?.extras?.costSplit),
|
|
148
|
+
// How many times the gate called the provider for this answer, and why again.
|
|
149
|
+
...attemptFacts(outcome.evidence?.extras?.attempts, outcome.evidence?.extras?.retries),
|
|
146
150
|
};
|
|
147
151
|
}
|
|
148
152
|
/** The ledger's cost-by-kind columns from the adapter's `costSplit`; none unless all four are costs. */
|
package/dist/harness/main.js
CHANGED
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
* pod's join token instead of a harness token (`pod.ts`), runs its disk's setup on a new volume
|
|
15
15
|
* (`setup.ts`), and runs its sandbox's tools in the pod itself (the `local` backend), with egress
|
|
16
16
|
* through egress-gate. SIGTERM (tini forwards
|
|
17
|
-
* it; the pod's grace is 10 s by default) gives its runs back and exits.
|
|
17
|
+
* it; the pod's grace is 10 s by default) gives its runs back and exits. On Agent Substrate
|
|
18
|
+
* (`NYLORUN_SANDBOX_SUBSTRATE=1`), SIGTERM means the worker pod is stopping: the engine logs it and
|
|
19
|
+
* keeps running, so the sandboxes service can suspend the actor within Substrate's 30 minutes.
|
|
18
20
|
*/
|
|
19
21
|
import { existsSync, mkdirSync } from "node:fs";
|
|
20
22
|
import { createServer } from "node:http";
|
|
@@ -106,8 +108,18 @@ export async function runHarness(stack, baseline) {
|
|
|
106
108
|
});
|
|
107
109
|
logger.info("harness_ready", { health: `http://127.0.0.1:${config.healthPort}/health` });
|
|
108
110
|
let stopping = false;
|
|
111
|
+
let workerStopping = false;
|
|
109
112
|
for (const signal of ["SIGINT", "SIGTERM"])
|
|
110
113
|
process.on(signal, () => {
|
|
114
|
+
// On Agent Substrate, SIGTERM means the worker pod is stopping: the actor has 30 minutes to
|
|
115
|
+
// be suspended, and the sandboxes service suspends it. A process that exits can't be, so the
|
|
116
|
+
// engine keeps running, and keeps its runs (Substrate's eviction contract).
|
|
117
|
+
if (signal === "SIGTERM" && config.pod?.substrate) {
|
|
118
|
+
if (!workerStopping)
|
|
119
|
+
logger.info("harness_worker_stopping", { sandboxId: config.pod.sandboxId });
|
|
120
|
+
workerStopping = true;
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
111
123
|
if (stopping)
|
|
112
124
|
return;
|
|
113
125
|
stopping = true;
|
|
@@ -169,6 +169,12 @@ export interface PodHostConfig {
|
|
|
169
169
|
* for the hosts whose keys Substrate's egress adds.
|
|
170
170
|
*/
|
|
171
171
|
egressCaFile?: string;
|
|
172
|
+
/**
|
|
173
|
+
* `NYLORUN_SANDBOX_SUBSTRATE=1`: the pod is an Agent Substrate actor. A stopping worker sends
|
|
174
|
+
* it SIGTERM and gives it 30 minutes to be suspended, which the sandboxes service does; the
|
|
175
|
+
* engine stays up instead of exiting (suspend before drain).
|
|
176
|
+
*/
|
|
177
|
+
substrate?: boolean;
|
|
172
178
|
/** Addresses the pod's NetworkPolicy must block, probed before the engine starts. */
|
|
173
179
|
blocked: BlockedAddress[];
|
|
174
180
|
}
|
|
@@ -647,6 +647,7 @@ function parsePodHost(env, url) {
|
|
|
647
647
|
httpUrl: origin,
|
|
648
648
|
...(egress ? { egressProxy: egress.replace(/\/+$/, "") } : {}),
|
|
649
649
|
...(egressCa ? { egressCaFile: egressCa } : {}),
|
|
650
|
+
...(read(env, "NYLORUN_SANDBOX_SUBSTRATE") === "1" ? { substrate: true } : {}),
|
|
650
651
|
blocked: blockedAddresses(env),
|
|
651
652
|
},
|
|
652
653
|
};
|
package/dist/model/pi-model.js
CHANGED
|
@@ -25,6 +25,8 @@ export function modelCallSettingsOf(limits) {
|
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
27
|
const OPENAI_COMPATIBLE_REASONING = /^reasoning(_content|_text)?$/;
|
|
28
|
+
/** How much of a retried attempt's error the outcome keeps. */
|
|
29
|
+
const RETRY_ERROR_CHARS = 300;
|
|
28
30
|
const emptyUsage = () => ({
|
|
29
31
|
input: 0,
|
|
30
32
|
output: 0,
|
|
@@ -200,21 +202,35 @@ export function piModel(options = {}) {
|
|
|
200
202
|
clearTimeout(timer);
|
|
201
203
|
}
|
|
202
204
|
};
|
|
203
|
-
|
|
205
|
+
// Each retry, for the outcome (Model Calls §6): the gate's ledger row and the call's event
|
|
206
|
+
// carry how many times the provider was called, and why again.
|
|
207
|
+
const retries = [];
|
|
208
|
+
const response = await retryModelCall(attempt, { attempts: settings.attempts, baseDelayMs: settings.retryBaseDelayMs }, context.signal, (retry) => {
|
|
209
|
+
retries.push({
|
|
210
|
+
attempt: retry.attempt,
|
|
211
|
+
delayMs: retry.delayMs,
|
|
212
|
+
error: String(scrub(retry.errorMessage, secrets)).slice(0, RETRY_ERROR_CHARS),
|
|
213
|
+
});
|
|
214
|
+
options.onRetry?.(retry);
|
|
215
|
+
});
|
|
216
|
+
const tried = { attempts: retries.length + 1, ...(retries.length ? { retries } : {}) };
|
|
204
217
|
// An abort is never a failure outcome: the host decides what it means.
|
|
205
218
|
context.signal.throwIfAborted();
|
|
206
219
|
if (natives.error)
|
|
207
220
|
return redacted(failure("invalid_request", natives.error, false));
|
|
208
221
|
if (response.stopReason === "aborted")
|
|
209
|
-
return redacted(failure("transient", "The provider stream was aborted.", true));
|
|
222
|
+
return { ...redacted(failure("transient", "The provider stream was aborted.", true)), ...tried };
|
|
210
223
|
if (response.stopReason === "error") {
|
|
211
224
|
const outcome = classifyAssistantError(response, selected.contextWindow);
|
|
212
|
-
return
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
225
|
+
return {
|
|
226
|
+
...redacted(outcome.code === "auth"
|
|
227
|
+
? {
|
|
228
|
+
...outcome,
|
|
229
|
+
message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
|
|
230
|
+
}
|
|
231
|
+
: outcome),
|
|
232
|
+
...tried,
|
|
233
|
+
};
|
|
218
234
|
}
|
|
219
235
|
if (response.stopReason === "deferred")
|
|
220
236
|
return failure("invalid_request", "The provider deferred the response, which the Runtime does not request.", false);
|
|
@@ -272,6 +288,9 @@ export function piModel(options = {}) {
|
|
|
272
288
|
maxOutputTokens: selected.maxTokens,
|
|
273
289
|
// From the call to the provider's first stream event (R7.1); absent when none came.
|
|
274
290
|
...(firstEventAt !== undefined ? { firstTokenMs: firstEventAt - calledAt } : {}),
|
|
291
|
+
// The provider calls this answer took, and each retry before it: the ledger keeps them.
|
|
292
|
+
attempts: tried.attempts,
|
|
293
|
+
...(retries.length ? { retries: retries } : {}),
|
|
275
294
|
},
|
|
276
295
|
},
|
|
277
296
|
};
|
package/dist/model/retry.d.ts
CHANGED
|
@@ -34,3 +34,17 @@ export interface ModelRetryPolicy {
|
|
|
34
34
|
* delay comes back as the last failure marked `aborted`, as pi-ai's `retryAssistantCall` does.
|
|
35
35
|
*/
|
|
36
36
|
export declare function retryModelCall(produce: () => Promise<AssistantMessage>, policy: ModelRetryPolicy, signal: AbortSignal, onRetry?: (retry: ModelRetry) => void): Promise<AssistantMessage>;
|
|
37
|
+
/** A retry as the ledger and the call's event keep it (Model Calls §6). */
|
|
38
|
+
export interface RecordedRetry {
|
|
39
|
+
readonly attempt: number;
|
|
40
|
+
readonly delayMs: number;
|
|
41
|
+
readonly error: string;
|
|
42
|
+
}
|
|
43
|
+
/** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
|
|
44
|
+
export declare function modelRetries(value: unknown): RecordedRetry[];
|
|
45
|
+
/**
|
|
46
|
+
* `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
|
|
47
|
+
* called the provider, and each retry: the ledger's columns and the event's fields alike. None
|
|
48
|
+
* unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
|
|
49
|
+
*/
|
|
50
|
+
export declare function attemptFacts(attempts: unknown, retries: unknown): {};
|
package/dist/model/retry.js
CHANGED
|
@@ -61,3 +61,37 @@ function wait(ms, signal) {
|
|
|
61
61
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
62
62
|
});
|
|
63
63
|
}
|
|
64
|
+
/** The most retries a row keeps; `modelAttempts` is 3 unless a Tenant raised it. */
|
|
65
|
+
const MAX_RETRIES = 10;
|
|
66
|
+
/** A retry list as the ledger and the events keep it: well-formed entries only, at most 10. */
|
|
67
|
+
export function modelRetries(value) {
|
|
68
|
+
if (!Array.isArray(value))
|
|
69
|
+
return [];
|
|
70
|
+
const out = [];
|
|
71
|
+
for (const entry of value) {
|
|
72
|
+
if (!entry || typeof entry !== "object")
|
|
73
|
+
continue;
|
|
74
|
+
const { attempt, delayMs, error } = entry;
|
|
75
|
+
if (typeof attempt !== "number" || !Number.isInteger(attempt) || attempt < 2)
|
|
76
|
+
continue;
|
|
77
|
+
out.push({
|
|
78
|
+
attempt,
|
|
79
|
+
delayMs: typeof delayMs === "number" && Number.isFinite(delayMs) && delayMs >= 0 ? Math.round(delayMs) : 0,
|
|
80
|
+
error: typeof error === "string" ? error.slice(0, 300) : "",
|
|
81
|
+
});
|
|
82
|
+
if (out.length === MAX_RETRIES)
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* `attempts` and `retries` for a model call's event (Model Calls §6): how many times the gate
|
|
89
|
+
* called the provider, and each retry: the ledger's columns and the event's fields alike. None
|
|
90
|
+
* unless `attempts` is a positive whole number (a call a Runtime before 0.50 served).
|
|
91
|
+
*/
|
|
92
|
+
export function attemptFacts(attempts, retries) {
|
|
93
|
+
if (typeof attempts !== "number" || !Number.isInteger(attempts) || attempts < 1)
|
|
94
|
+
return {};
|
|
95
|
+
const kept = modelRetries(retries);
|
|
96
|
+
return { attempts, ...(kept.length ? { retries: kept } : {}) };
|
|
97
|
+
}
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.2.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "Nylorun Runtime API",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.51.0-beta",
|
|
6
6
|
"x-nylorun-protocol": {
|
|
7
7
|
"min": 16,
|
|
8
8
|
"max": 18,
|
|
@@ -13430,6 +13430,35 @@
|
|
|
13430
13430
|
],
|
|
13431
13431
|
"additionalProperties": {}
|
|
13432
13432
|
},
|
|
13433
|
+
"attempts": {
|
|
13434
|
+
"type": "integer",
|
|
13435
|
+
"exclusiveMinimum": 0
|
|
13436
|
+
},
|
|
13437
|
+
"retries": {
|
|
13438
|
+
"type": "array",
|
|
13439
|
+
"items": {
|
|
13440
|
+
"type": "object",
|
|
13441
|
+
"properties": {
|
|
13442
|
+
"attempt": {
|
|
13443
|
+
"type": "integer",
|
|
13444
|
+
"minimum": 2
|
|
13445
|
+
},
|
|
13446
|
+
"delayMs": {
|
|
13447
|
+
"type": "integer",
|
|
13448
|
+
"minimum": 0
|
|
13449
|
+
},
|
|
13450
|
+
"error": {
|
|
13451
|
+
"type": "string"
|
|
13452
|
+
}
|
|
13453
|
+
},
|
|
13454
|
+
"required": [
|
|
13455
|
+
"attempt",
|
|
13456
|
+
"delayMs",
|
|
13457
|
+
"error"
|
|
13458
|
+
],
|
|
13459
|
+
"additionalProperties": {}
|
|
13460
|
+
}
|
|
13461
|
+
},
|
|
13433
13462
|
"agent": {
|
|
13434
13463
|
"type": "object",
|
|
13435
13464
|
"properties": {
|
|
@@ -13716,6 +13745,35 @@
|
|
|
13716
13745
|
],
|
|
13717
13746
|
"additionalProperties": {}
|
|
13718
13747
|
},
|
|
13748
|
+
"attempts": {
|
|
13749
|
+
"type": "integer",
|
|
13750
|
+
"exclusiveMinimum": 0
|
|
13751
|
+
},
|
|
13752
|
+
"retries": {
|
|
13753
|
+
"type": "array",
|
|
13754
|
+
"items": {
|
|
13755
|
+
"type": "object",
|
|
13756
|
+
"properties": {
|
|
13757
|
+
"attempt": {
|
|
13758
|
+
"type": "integer",
|
|
13759
|
+
"minimum": 2
|
|
13760
|
+
},
|
|
13761
|
+
"delayMs": {
|
|
13762
|
+
"type": "integer",
|
|
13763
|
+
"minimum": 0
|
|
13764
|
+
},
|
|
13765
|
+
"error": {
|
|
13766
|
+
"type": "string"
|
|
13767
|
+
}
|
|
13768
|
+
},
|
|
13769
|
+
"required": [
|
|
13770
|
+
"attempt",
|
|
13771
|
+
"delayMs",
|
|
13772
|
+
"error"
|
|
13773
|
+
],
|
|
13774
|
+
"additionalProperties": {}
|
|
13775
|
+
}
|
|
13776
|
+
},
|
|
13719
13777
|
"agent": {
|
|
13720
13778
|
"type": "object",
|
|
13721
13779
|
"properties": {
|
|
@@ -14007,6 +14065,35 @@
|
|
|
14007
14065
|
],
|
|
14008
14066
|
"additionalProperties": {}
|
|
14009
14067
|
},
|
|
14068
|
+
"attempts": {
|
|
14069
|
+
"type": "integer",
|
|
14070
|
+
"exclusiveMinimum": 0
|
|
14071
|
+
},
|
|
14072
|
+
"retries": {
|
|
14073
|
+
"type": "array",
|
|
14074
|
+
"items": {
|
|
14075
|
+
"type": "object",
|
|
14076
|
+
"properties": {
|
|
14077
|
+
"attempt": {
|
|
14078
|
+
"type": "integer",
|
|
14079
|
+
"minimum": 2
|
|
14080
|
+
},
|
|
14081
|
+
"delayMs": {
|
|
14082
|
+
"type": "integer",
|
|
14083
|
+
"minimum": 0
|
|
14084
|
+
},
|
|
14085
|
+
"error": {
|
|
14086
|
+
"type": "string"
|
|
14087
|
+
}
|
|
14088
|
+
},
|
|
14089
|
+
"required": [
|
|
14090
|
+
"attempt",
|
|
14091
|
+
"delayMs",
|
|
14092
|
+
"error"
|
|
14093
|
+
],
|
|
14094
|
+
"additionalProperties": {}
|
|
14095
|
+
}
|
|
14096
|
+
},
|
|
14010
14097
|
"agent": {
|
|
14011
14098
|
"type": "object",
|
|
14012
14099
|
"properties": {
|
|
@@ -63,6 +63,11 @@ export interface PodStatus {
|
|
|
63
63
|
readonly shutdownTime?: string;
|
|
64
64
|
readonly reason?: string;
|
|
65
65
|
readonly message?: string;
|
|
66
|
+
/**
|
|
67
|
+
* Agent Substrate: when the sandbox was last suspended because its worker stopped. While it
|
|
68
|
+
* moves to another worker, `reason` is `Moving`.
|
|
69
|
+
*/
|
|
70
|
+
readonly drainedAt?: string;
|
|
66
71
|
/** With `?wait=`: whether the state was reached before the timeout. */
|
|
67
72
|
readonly met?: boolean;
|
|
68
73
|
}
|
|
@@ -25,6 +25,10 @@
|
|
|
25
25
|
* (`agent.failed`, saying what the Agent lost); a crash an idle suspend finds is recorded the
|
|
26
26
|
* same way. The next turn start asks for `Running`, and the sandboxes service reverts the
|
|
27
27
|
* sandbox to its last suspend before resuming it.
|
|
28
|
+
* - **Drained** (Agent Substrate): a worker that stops gives its actors 30 minutes to be
|
|
29
|
+
* suspended. The sandboxes service suspends each, records `drainedAt` and resumes it on
|
|
30
|
+
* another worker; a new `drainedAt` is `agent.suspended` (reason `drain`), once, and a pod
|
|
31
|
+
* that is `Moving` waits rather than fail at the ready deadline.
|
|
28
32
|
* - **Lost**: an incarnation that has run (a pod joined) whose Sandbox is gone or whose volume
|
|
29
33
|
* is missing is `lost` (`agent.lost`): its host epoch moves, its Sandbox is deleted, and
|
|
30
34
|
* only a reset brings the Environment back. agent-sandbox recreates a deleted claim empty, so
|
|
@@ -60,6 +64,11 @@ export declare function repositoryOf(reference: string): string;
|
|
|
60
64
|
* the node made up for an image loaded onto it, which no registry serves.
|
|
61
65
|
*/
|
|
62
66
|
export declare function imageDigestOf(imageID: string | undefined, requested: string): string | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
|
|
69
|
+
* stopped, until it runs on another one.
|
|
70
|
+
*/
|
|
71
|
+
export declare const MOVING = "Moving";
|
|
63
72
|
/** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
|
|
64
73
|
export declare const SETUP_FAILED = "setup:";
|
|
65
74
|
/** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */
|
|
@@ -34,6 +34,11 @@ export function imageDigestOf(imageID, requested) {
|
|
|
34
34
|
return undefined;
|
|
35
35
|
return repositoryOf(id) === repositoryOf(requested) ? id : undefined;
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* The sandboxes service's reason for a sandbox suspended because its Agent Substrate worker
|
|
39
|
+
* stopped, until it runs on another one.
|
|
40
|
+
*/
|
|
41
|
+
export const MOVING = "Moving";
|
|
37
42
|
/** A setup failure's `reason` starts with this (`sandbox/setup.ts`); it is sticky. */
|
|
38
43
|
export const SETUP_FAILED = "setup:";
|
|
39
44
|
/** A crash's `reason` (see `crashReasonOf`) starts with this; it stays until a turn starts the pod. */
|
|
@@ -158,6 +163,12 @@ export function decide(facts) {
|
|
|
158
163
|
again(pollMs);
|
|
159
164
|
return done();
|
|
160
165
|
}
|
|
166
|
+
// Agent Substrate: its worker stopped, and the sandboxes service suspended it with everything it
|
|
167
|
+
// held and resumes it on another worker (suspend before drain). Recorded once per drain.
|
|
168
|
+
if (status.drainedAt !== undefined && status.drainedAt !== pod.drainedAt) {
|
|
169
|
+
patch.drainedAt = status.drainedAt;
|
|
170
|
+
events.push(event("agent.suspended", { reason: "drain" }));
|
|
171
|
+
}
|
|
161
172
|
const opId = opIdOf({ k8sName: pod.k8sName, rev });
|
|
162
173
|
if (desired === "suspended" && !status.exists) {
|
|
163
174
|
if (pod.observed !== "suspended")
|
|
@@ -217,6 +228,11 @@ export function decide(facts) {
|
|
|
217
228
|
events.push(event("agent.failed", { reason: crash }));
|
|
218
229
|
return done();
|
|
219
230
|
}
|
|
231
|
+
// On its way to another worker after a drain: neither failed nor starting.
|
|
232
|
+
if (status.reason === MOVING) {
|
|
233
|
+
again(pollMs);
|
|
234
|
+
return done();
|
|
235
|
+
}
|
|
220
236
|
if (now - startedAt >= readyDeadlineMs) {
|
|
221
237
|
const reason = crash ??
|
|
222
238
|
([status.reason, status.message].filter(Boolean).join(": ") ||
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ALTER TABLE "nylorun"."agents" ADD COLUMN "drained_at" text;
|
|
@@ -330,6 +330,20 @@
|
|
|
330
330
|
"when": 1794240246972,
|
|
331
331
|
"tag": "0046_artifacts_audience",
|
|
332
332
|
"breakpoints": true
|
|
333
|
+
},
|
|
334
|
+
{
|
|
335
|
+
"idx": 47,
|
|
336
|
+
"version": "7",
|
|
337
|
+
"when": 1794630124577,
|
|
338
|
+
"tag": "0047_model_usage_attempts",
|
|
339
|
+
"breakpoints": true
|
|
340
|
+
},
|
|
341
|
+
{
|
|
342
|
+
"idx": 48,
|
|
343
|
+
"version": "7",
|
|
344
|
+
"when": 1794635246579,
|
|
345
|
+
"tag": "0048_sandbox_pod_drained_at",
|
|
346
|
+
"breakpoints": true
|
|
333
347
|
}
|
|
334
348
|
]
|
|
335
349
|
}
|
|
@@ -1321,6 +1321,23 @@ export declare const agents: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
1321
1321
|
}, {}, {
|
|
1322
1322
|
pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
|
|
1323
1323
|
}>;
|
|
1324
|
+
drainedAt: import("drizzle-orm/pg-core").PgColumn<{
|
|
1325
|
+
name: "drainedAt";
|
|
1326
|
+
tableName: "agents";
|
|
1327
|
+
dataType: "string";
|
|
1328
|
+
columnType: "PgText";
|
|
1329
|
+
data: string;
|
|
1330
|
+
driverParam: string;
|
|
1331
|
+
notNull: false;
|
|
1332
|
+
hasDefault: false;
|
|
1333
|
+
isPrimaryKey: false;
|
|
1334
|
+
isAutoincrement: false;
|
|
1335
|
+
hasRuntimeDefault: false;
|
|
1336
|
+
enumValues: [string, ...string[]];
|
|
1337
|
+
baseColumn: never;
|
|
1338
|
+
identity: undefined;
|
|
1339
|
+
generated: undefined;
|
|
1340
|
+
}, {}, {}>;
|
|
1324
1341
|
diskOp: import("drizzle-orm/pg-core").PgColumn<{
|
|
1325
1342
|
name: "diskOp";
|
|
1326
1343
|
tableName: "agents";
|
|
@@ -3587,7 +3604,9 @@ export declare const signingKeys: import("drizzle-orm/pg-core").PgTableWithColum
|
|
|
3587
3604
|
* when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
|
|
3588
3605
|
* timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
|
|
3589
3606
|
* when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
|
|
3590
|
-
* prices the adapter applied (migration 0037); null on rows written before.
|
|
3607
|
+
* prices the adapter applied (migration 0037); null on rows written before. `attempts` and
|
|
3608
|
+
* `retries` say how many times the gate called the provider for the answer, and why again
|
|
3609
|
+
* (migration 0047).
|
|
3591
3610
|
*/
|
|
3592
3611
|
export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
3593
3612
|
name: "model_usage";
|
|
@@ -4047,6 +4066,42 @@ export declare const modelUsage: import("drizzle-orm/pg-core").PgTableWithColumn
|
|
|
4047
4066
|
identity: undefined;
|
|
4048
4067
|
generated: undefined;
|
|
4049
4068
|
}, {}, {}>;
|
|
4069
|
+
attempts: import("drizzle-orm/pg-core").PgColumn<{
|
|
4070
|
+
name: "attempts";
|
|
4071
|
+
tableName: "model_usage";
|
|
4072
|
+
dataType: "number";
|
|
4073
|
+
columnType: "PgInteger";
|
|
4074
|
+
data: number;
|
|
4075
|
+
driverParam: string | number;
|
|
4076
|
+
notNull: false;
|
|
4077
|
+
hasDefault: false;
|
|
4078
|
+
isPrimaryKey: false;
|
|
4079
|
+
isAutoincrement: false;
|
|
4080
|
+
hasRuntimeDefault: false;
|
|
4081
|
+
enumValues: undefined;
|
|
4082
|
+
baseColumn: never;
|
|
4083
|
+
identity: undefined;
|
|
4084
|
+
generated: undefined;
|
|
4085
|
+
}, {}, {}>;
|
|
4086
|
+
retries: import("drizzle-orm/pg-core").PgColumn<{
|
|
4087
|
+
name: "retries";
|
|
4088
|
+
tableName: "model_usage";
|
|
4089
|
+
dataType: "custom";
|
|
4090
|
+
columnType: "PgCustomColumn";
|
|
4091
|
+
data: unknown;
|
|
4092
|
+
driverParam: unknown;
|
|
4093
|
+
notNull: false;
|
|
4094
|
+
hasDefault: false;
|
|
4095
|
+
isPrimaryKey: false;
|
|
4096
|
+
isAutoincrement: false;
|
|
4097
|
+
hasRuntimeDefault: false;
|
|
4098
|
+
enumValues: undefined;
|
|
4099
|
+
baseColumn: never;
|
|
4100
|
+
identity: undefined;
|
|
4101
|
+
generated: undefined;
|
|
4102
|
+
}, {}, {
|
|
4103
|
+
pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
|
|
4104
|
+
}>;
|
|
4050
4105
|
};
|
|
4051
4106
|
dialect: 'pg';
|
|
4052
4107
|
}>;
|
|
@@ -6707,7 +6762,7 @@ export type EnvironmentRowColumns = typeof agents.$inferSelect;
|
|
|
6707
6762
|
export type ArtifactRow = typeof artifacts.$inferSelect;
|
|
6708
6763
|
export type ArtifactVersionRow = typeof artifactVersions.$inferSelect;
|
|
6709
6764
|
export type DefinitionFileRow = typeof definitionFiles.$inferSelect;
|
|
6710
|
-
export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd"> & {
|
|
6765
|
+
export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokensReported" | "costKnown" | "startedAt" | "endedAt" | "firstTokenMs" | "costInputUsd" | "costCacheReadUsd" | "costCacheWriteUsd" | "costOutputUsd" | "attempts" | "retries"> & {
|
|
6711
6766
|
tokensReported?: boolean | null;
|
|
6712
6767
|
costKnown?: boolean | null;
|
|
6713
6768
|
startedAt?: string | null;
|
|
@@ -6717,4 +6772,6 @@ export type ModelUsageWrite = Omit<ModelUsageRow, "duplicate" | "txid" | "tokens
|
|
|
6717
6772
|
costCacheReadUsd?: number | null;
|
|
6718
6773
|
costCacheWriteUsd?: number | null;
|
|
6719
6774
|
costOutputUsd?: number | null;
|
|
6775
|
+
attempts?: number | null;
|
|
6776
|
+
retries?: unknown;
|
|
6720
6777
|
};
|
|
@@ -173,6 +173,8 @@ export const agents = nylorun.table("agents", {
|
|
|
173
173
|
imageDigest: textC(),
|
|
174
174
|
/** The engine the image carries (ADR-99): `{ version, digest? }`, from a join of its engine. */
|
|
175
175
|
engineImage: jsonText(),
|
|
176
|
+
/** When its Agent Substrate worker last stopped under it (the sandboxes service's `drainedAt`). */
|
|
177
|
+
drainedAt: text(),
|
|
176
178
|
/**
|
|
177
179
|
* The disk lease (R2d step 3): `{ kind, version?, startedAt, until }` while a snapshot, a
|
|
178
180
|
* restore or a seed copies the disk; turns are refused until it ends or `until` passes.
|
|
@@ -455,7 +457,9 @@ export const signingKeys = nylorun.table("signing_keys", {
|
|
|
455
457
|
* when the provider sent none. `startedAt`, `endedAt` and `firstTokenMs` are the gate's own
|
|
456
458
|
* timing of the call (R7.1, migration 0022); null on rows written before, and `firstTokenMs`
|
|
457
459
|
* when the provider streamed nothing. The `cost*Usd` columns split `costUsd` by token kind, at the
|
|
458
|
-
* prices the adapter applied (migration 0037); null on rows written before.
|
|
460
|
+
* prices the adapter applied (migration 0037); null on rows written before. `attempts` and
|
|
461
|
+
* `retries` say how many times the gate called the provider for the answer, and why again
|
|
462
|
+
* (migration 0047).
|
|
459
463
|
*/
|
|
460
464
|
export const modelUsage = nylorun.table("model_usage", {
|
|
461
465
|
id: textC().primaryKey(),
|
|
@@ -497,6 +501,13 @@ export const modelUsage = nylorun.table("model_usage", {
|
|
|
497
501
|
costCacheWriteUsd: doublePrecision(),
|
|
498
502
|
/** What the output cost. */
|
|
499
503
|
costOutputUsd: doublePrecision(),
|
|
504
|
+
/**
|
|
505
|
+
* The provider calls the gate made for this answer, the first included (migration 0047);
|
|
506
|
+
* null on rows written before.
|
|
507
|
+
*/
|
|
508
|
+
attempts: integer(),
|
|
509
|
+
/** Each retry before the answer: `{ attempt, delayMs, error }` (migration 0047); null when none. */
|
|
510
|
+
retries: jsonText(),
|
|
500
511
|
}, (t) => [
|
|
501
512
|
index("model_usage_session").on(t.sessionId, t.createdAt, t.id),
|
|
502
513
|
index("model_usage_export").on(t.txid, t.id),
|
|
@@ -284,6 +284,7 @@ function environmentOf(row) {
|
|
|
284
284
|
...(row.volumeId !== null ? { volumeId: row.volumeId } : {}),
|
|
285
285
|
...(row.imageDigest !== null ? { imageDigest: row.imageDigest } : {}),
|
|
286
286
|
...(row.engineImage ? { engineImage: row.engineImage } : {}),
|
|
287
|
+
...(row.drainedAt !== null ? { drainedAt: row.drainedAt } : {}),
|
|
287
288
|
},
|
|
288
289
|
}
|
|
289
290
|
: {}),
|
|
@@ -319,6 +320,7 @@ function podColumns(patch) {
|
|
|
319
320
|
volumeId: patch.volumeId,
|
|
320
321
|
imageDigest: patch.imageDigest,
|
|
321
322
|
engineImage: patch.engineImage,
|
|
323
|
+
drainedAt: patch.drainedAt,
|
|
322
324
|
};
|
|
323
325
|
return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
|
|
324
326
|
}
|
package/dist/store/types.d.ts
CHANGED
|
@@ -269,6 +269,11 @@ export interface SandboxPodState {
|
|
|
269
269
|
version: string;
|
|
270
270
|
digest?: string;
|
|
271
271
|
};
|
|
272
|
+
/**
|
|
273
|
+
* When the sandbox was last suspended because its Agent Substrate worker stopped (the sandboxes
|
|
274
|
+
* service's `drainedAt`), so each drain is recorded once (`agent.suspended`, reason `drain`).
|
|
275
|
+
*/
|
|
276
|
+
drainedAt?: string;
|
|
272
277
|
}
|
|
273
278
|
/** A change to a pod's columns: `null` clears an optional one. */
|
|
274
279
|
export type SandboxPodPatch = {
|
|
@@ -128,6 +128,10 @@ export interface MeteredCall {
|
|
|
128
128
|
readonly totalTokens: number;
|
|
129
129
|
readonly cachedTokens: number;
|
|
130
130
|
readonly costUsd: number;
|
|
131
|
+
/** The provider calls the gate made for the answer (migration 0047); null on older rows. */
|
|
132
|
+
readonly attempts?: number | null;
|
|
133
|
+
/** Each retry before the answer (migration 0047); null when none. */
|
|
134
|
+
readonly retries?: unknown;
|
|
131
135
|
}
|
|
132
136
|
/** From `startedAt` to `endedAt` in whole milliseconds; undefined unless both are times. */
|
|
133
137
|
export declare function durationBetween(startedAt: string | undefined, endedAt: string | undefined): number | undefined;
|
|
@@ -227,6 +231,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
227
231
|
gate?: GateTimes | undefined;
|
|
228
232
|
policy?: ToolPolicyFact | undefined;
|
|
229
233
|
} | {
|
|
234
|
+
redacted?: undefined;
|
|
230
235
|
agent: import("@nylorun/core/define").AgentRef;
|
|
231
236
|
error: {
|
|
232
237
|
code: string;
|
|
@@ -236,7 +241,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
236
241
|
retryable?: boolean | undefined;
|
|
237
242
|
};
|
|
238
243
|
output?: undefined;
|
|
239
|
-
redacted?: undefined;
|
|
240
244
|
toolName: string;
|
|
241
245
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
242
246
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -250,6 +254,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
250
254
|
gate?: GateTimes | undefined;
|
|
251
255
|
policy?: ToolPolicyFact | undefined;
|
|
252
256
|
} | {
|
|
257
|
+
redacted?: undefined;
|
|
253
258
|
agent?: undefined;
|
|
254
259
|
error: {
|
|
255
260
|
code: string;
|
|
@@ -259,7 +264,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
259
264
|
retryable?: boolean | undefined;
|
|
260
265
|
};
|
|
261
266
|
output?: undefined;
|
|
262
|
-
redacted?: undefined;
|
|
263
267
|
toolName: string;
|
|
264
268
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
265
269
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -307,10 +311,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
307
311
|
gate?: GateTimes | undefined;
|
|
308
312
|
policy?: ToolPolicyFact | undefined;
|
|
309
313
|
} | {
|
|
314
|
+
redacted?: undefined;
|
|
310
315
|
agent: import("@nylorun/core/define").AgentRef;
|
|
311
316
|
error?: undefined;
|
|
312
317
|
output: unknown;
|
|
313
|
-
redacted?: undefined;
|
|
314
318
|
toolName: string;
|
|
315
319
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
316
320
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -324,10 +328,10 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
324
328
|
gate?: GateTimes | undefined;
|
|
325
329
|
policy?: ToolPolicyFact | undefined;
|
|
326
330
|
} | {
|
|
331
|
+
redacted?: undefined;
|
|
327
332
|
agent?: undefined;
|
|
328
333
|
error?: undefined;
|
|
329
334
|
output: unknown;
|
|
330
|
-
redacted?: undefined;
|
|
331
335
|
toolName: string;
|
|
332
336
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
333
337
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -391,6 +395,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
391
395
|
gate?: GateTimes | undefined;
|
|
392
396
|
policy?: ToolPolicyFact | undefined;
|
|
393
397
|
} | {
|
|
398
|
+
redacted?: undefined;
|
|
394
399
|
callId: string;
|
|
395
400
|
invocationId: string;
|
|
396
401
|
agent: import("@nylorun/core/define").AgentRef;
|
|
@@ -402,7 +407,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
402
407
|
retryable?: boolean | undefined;
|
|
403
408
|
};
|
|
404
409
|
output?: undefined;
|
|
405
|
-
redacted?: undefined;
|
|
406
410
|
toolName: string;
|
|
407
411
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
408
412
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -416,6 +420,7 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
416
420
|
gate?: GateTimes | undefined;
|
|
417
421
|
policy?: ToolPolicyFact | undefined;
|
|
418
422
|
} | {
|
|
423
|
+
redacted?: undefined;
|
|
419
424
|
callId: string;
|
|
420
425
|
invocationId: string;
|
|
421
426
|
agent?: undefined;
|
|
@@ -427,7 +432,6 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
427
432
|
retryable?: boolean | undefined;
|
|
428
433
|
};
|
|
429
434
|
output?: undefined;
|
|
430
|
-
redacted?: undefined;
|
|
431
435
|
toolName: string;
|
|
432
436
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
433
437
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -479,12 +483,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
479
483
|
gate?: GateTimes | undefined;
|
|
480
484
|
policy?: ToolPolicyFact | undefined;
|
|
481
485
|
} | {
|
|
486
|
+
redacted?: undefined;
|
|
482
487
|
callId: string;
|
|
483
488
|
invocationId: string;
|
|
484
489
|
agent: import("@nylorun/core/define").AgentRef;
|
|
485
490
|
error?: undefined;
|
|
486
491
|
output: unknown;
|
|
487
|
-
redacted?: undefined;
|
|
488
492
|
toolName: string;
|
|
489
493
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
490
494
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -498,12 +502,12 @@ export declare function toolCompleted(request: HostEffect, value: unknown, facts
|
|
|
498
502
|
gate?: GateTimes | undefined;
|
|
499
503
|
policy?: ToolPolicyFact | undefined;
|
|
500
504
|
} | {
|
|
505
|
+
redacted?: undefined;
|
|
501
506
|
callId: string;
|
|
502
507
|
invocationId: string;
|
|
503
508
|
agent?: undefined;
|
|
504
509
|
error?: undefined;
|
|
505
510
|
output: unknown;
|
|
506
|
-
redacted?: undefined;
|
|
507
511
|
toolName: string;
|
|
508
512
|
status?: "cancelled" | "completed" | "failed" | undefined;
|
|
509
513
|
kind?: "harness" | "http" | "mcp" | "runtime" | "sandbox" | undefined;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isModelFailureOutcome } from "@nylorun/core/define";
|
|
2
2
|
import { HarnessApiError } from "@nylorun/core/harness-api";
|
|
3
|
+
import { attemptFacts } from "../model/retry.js";
|
|
3
4
|
/** The model's tool call id and the harness invocation, for tool effects. */
|
|
4
5
|
export function toolIds(context) {
|
|
5
6
|
const c = context;
|
|
@@ -95,6 +96,8 @@ export function modelFailed(request, value) {
|
|
|
95
96
|
code: value.code,
|
|
96
97
|
message: value.message,
|
|
97
98
|
retryable: value.retryable,
|
|
99
|
+
// How many times the gate called the provider before it gave up, and why again.
|
|
100
|
+
...attemptFacts(value.attempts, value.retries),
|
|
98
101
|
...agentOf(request),
|
|
99
102
|
};
|
|
100
103
|
}
|
|
@@ -133,6 +136,7 @@ export function modelOutcomeFacts(effectId, window, metered) {
|
|
|
133
136
|
},
|
|
134
137
|
}
|
|
135
138
|
: {}),
|
|
139
|
+
...attemptFacts(metered?.attempts, metered?.retries),
|
|
136
140
|
};
|
|
137
141
|
}
|
|
138
142
|
/**
|
package/dist/traces/map.js
CHANGED
|
@@ -135,6 +135,8 @@ export function spanOf(row, turn, settings) {
|
|
|
135
135
|
.str("gen_ai.agent.id", p.agent?.id)
|
|
136
136
|
.str("nylorun.effect.id", p.effectId)
|
|
137
137
|
.int("nylorun.effect.window_ms", p.durationMs)
|
|
138
|
+
// The gate's provider calls for this one (Model Calls §6): more than 1 after a retry.
|
|
139
|
+
.int("nylorun.model.attempts", typeof p.attempts === "number" ? p.attempts : undefined)
|
|
138
140
|
.str("nylorun.compaction.trigger", row.type === "context.compacted" ? p.trigger : undefined)
|
|
139
141
|
.str("error.type", row.type === "model.failed" ? p.code : undefined)
|
|
140
142
|
.str("gen_ai.output.messages", output);
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const RUNTIME_VERSION = "0.
|
|
1
|
+
export declare const RUNTIME_VERSION = "0.51.0-beta";
|
package/dist/version.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
// Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
|
|
2
2
|
// release:prepare also rewrites this when it bumps @nylorun/runtime.
|
|
3
|
-
export const RUNTIME_VERSION = "0.
|
|
3
|
+
export const RUNTIME_VERSION = "0.51.0-beta";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nylorun/runtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.51.0-beta",
|
|
4
4
|
"description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -61,39 +61,39 @@
|
|
|
61
61
|
"dependencies": {
|
|
62
62
|
"@ag-ui/core": "~1.0.4",
|
|
63
63
|
"@asteasolutions/zod-to-openapi": "^9.1.0",
|
|
64
|
-
"@earendil-works/pi-ai": "1.
|
|
65
|
-
"@hono/node-server": "^2.1.
|
|
64
|
+
"@earendil-works/pi-ai": "1.5.2",
|
|
65
|
+
"@hono/node-server": "^2.1.8",
|
|
66
66
|
"@hono/zod-openapi": "^1.6.3",
|
|
67
|
-
"@modelcontextprotocol/sdk": "^1.32.
|
|
68
|
-
"@nylorun/core": "0.
|
|
69
|
-
"@nylorun/harness": "0.36.
|
|
70
|
-
"@restatedev/restate-sdk": "1.
|
|
71
|
-
"@restatedev/restate-sdk-clients": "1.
|
|
72
|
-
"@s2-dev/streamstore": "0.
|
|
67
|
+
"@modelcontextprotocol/sdk": "^1.32.2",
|
|
68
|
+
"@nylorun/core": "0.44.0-beta",
|
|
69
|
+
"@nylorun/harness": "0.36.4-beta",
|
|
70
|
+
"@restatedev/restate-sdk": "1.18.3",
|
|
71
|
+
"@restatedev/restate-sdk-clients": "1.18.3",
|
|
72
|
+
"@s2-dev/streamstore": "0.30.0",
|
|
73
73
|
"aws4fetch": "1.0.20",
|
|
74
74
|
"croner": "^9.1.0",
|
|
75
75
|
"drizzle-orm": "^0.45.4",
|
|
76
|
-
"hono": "^4.13.
|
|
77
|
-
"jose": "^6.2.
|
|
76
|
+
"hono": "^4.13.18",
|
|
77
|
+
"jose": "^6.2.17",
|
|
78
78
|
"just-bash": "3.6.0",
|
|
79
79
|
"pg": "8.23.1",
|
|
80
|
-
"pg-logical-replication": "2.
|
|
80
|
+
"pg-logical-replication": "2.6.0",
|
|
81
81
|
"postgres": "^3.4.9",
|
|
82
|
-
"ws": "^8.22.
|
|
82
|
+
"ws": "^8.22.2",
|
|
83
83
|
"yaml": "^2.9.1",
|
|
84
84
|
"zod": "^4.6.7"
|
|
85
85
|
},
|
|
86
86
|
"devDependencies": {
|
|
87
87
|
"@a2a-js/sdk": "~1.3.0",
|
|
88
88
|
"@ag-ui/client": "1.0.4",
|
|
89
|
-
"@scalar/openapi-parser": "^0.29.
|
|
90
|
-
"@types/node": "^26.6.
|
|
89
|
+
"@scalar/openapi-parser": "^0.29.21",
|
|
90
|
+
"@types/node": "^26.6.6",
|
|
91
91
|
"@types/pg": "^8.23.1",
|
|
92
92
|
"@types/ws": "^8.18.2",
|
|
93
93
|
"@typescript/native": "npm:typescript@^7.0.2",
|
|
94
94
|
"drizzle-kit": "^0.31.11",
|
|
95
95
|
"esbuild": "^0.28.2",
|
|
96
96
|
"typescript": "npm:@typescript/typescript6@^6.0.2",
|
|
97
|
-
"vitest": "^5.0
|
|
97
|
+
"vitest": "^5.1.0"
|
|
98
98
|
}
|
|
99
99
|
}
|