@runuai/host 0.9.161 → 0.9.163
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agent-cli.ts +28 -15
- package/lib/agent.ts +9 -0
- package/lib/agents/claude.ts +110 -4
- package/lib/agents/codex.ts +175 -8
- package/lib/agents/opencode.ts +35 -12
- package/lib/agents/rate-limit.ts +299 -3
- package/lib/agents/types.ts +29 -1
- package/lib/agents/uai.ts +3 -4
- package/lib/engine-accounts.ts +82 -4
- package/lib/orchestrator.ts +189 -17
- package/lib/task-credential-cleanup.ts +12 -2
- package/lib/task-environment/machine-task-up.ts +6 -2
- package/lib/task-environment/machine.ts +2 -1
- package/lib/task-environment/types.ts +4 -0
- package/package.json +1 -1
- package/scripts/agent/task-down.sh +13 -0
- package/scripts/agent/task-up.sh +30 -0
- package/src/main.ts +8 -0
- package/src/protocol.ts +55 -0
package/lib/orchestrator.ts
CHANGED
|
@@ -31,6 +31,7 @@ import { dirname, join } from "node:path";
|
|
|
31
31
|
import { inArray } from "drizzle-orm";
|
|
32
32
|
|
|
33
33
|
import { getDb, schema } from "./db";
|
|
34
|
+
import { newId } from "./ulid";
|
|
34
35
|
import { mockAgentFactory } from "./agents/mock";
|
|
35
36
|
import { realAgentFactory } from "./agents/factory";
|
|
36
37
|
import { resolveAgentMode } from "./agents/mode";
|
|
@@ -152,7 +153,12 @@ import {
|
|
|
152
153
|
codexHomeIsolatable,
|
|
153
154
|
codexHomeSeedCommand,
|
|
154
155
|
codexPerAgentHome,
|
|
156
|
+
clearEngineAccountCooldowns,
|
|
157
|
+
clearEngineAccountCooldownWindow,
|
|
155
158
|
cooldownEngineAccount,
|
|
159
|
+
cooldownEngineAccountUntil,
|
|
160
|
+
cooldownEngineAccountWindow,
|
|
161
|
+
cooldownEngineAccountWindowUntil,
|
|
156
162
|
noteEngineAccountUsed,
|
|
157
163
|
pickEngineAccount,
|
|
158
164
|
provisionEngineAccounts,
|
|
@@ -227,6 +233,7 @@ import {
|
|
|
227
233
|
import { env } from "./env";
|
|
228
234
|
import type {
|
|
229
235
|
ChannelEnsureInput,
|
|
236
|
+
EngineUsageFrame,
|
|
230
237
|
ChannelHuman,
|
|
231
238
|
HostEvent,
|
|
232
239
|
} from "../src/protocol";
|
|
@@ -247,6 +254,17 @@ import {
|
|
|
247
254
|
import { forgetTaskEnvSnapshot } from "./task-env-source";
|
|
248
255
|
|
|
249
256
|
export type HostEventSubscriber = (event: HostEvent) => void;
|
|
257
|
+
/** True means the frame reached the current cloud transport generation. */
|
|
258
|
+
export type EngineUsageSubscriber = (frame: EngineUsageFrame) => boolean;
|
|
259
|
+
|
|
260
|
+
export const ENGINE_USAGE_REFRESH_MS = 15 * 60_000;
|
|
261
|
+
|
|
262
|
+
interface DeliveredEngineUsage {
|
|
263
|
+
payload: string;
|
|
264
|
+
sentAt: number;
|
|
265
|
+
taskId: string;
|
|
266
|
+
agentId: string;
|
|
267
|
+
}
|
|
250
268
|
|
|
251
269
|
// Rolling upgrade fence: v0 Secretary runners were launched under the
|
|
252
270
|
// communicator tool-denial profile and may survive a host restart. Durable
|
|
@@ -568,6 +586,18 @@ const LEASED_CREDENTIAL_BLANKS: Readonly<Record<string, string>> = Object.freeze
|
|
|
568
586
|
* again. */
|
|
569
587
|
const LEASED_SYNC_RETRY_MS = 30_000;
|
|
570
588
|
|
|
589
|
+
const ENGINE_USAGE_KINDS = new Set<EngineUsageFrame["engine"]>([
|
|
590
|
+
"claude",
|
|
591
|
+
"codex",
|
|
592
|
+
"grok",
|
|
593
|
+
"cursor",
|
|
594
|
+
"opencode",
|
|
595
|
+
]);
|
|
596
|
+
|
|
597
|
+
function isEngineUsageKind(kind: string): kind is EngineUsageFrame["engine"] {
|
|
598
|
+
return ENGINE_USAGE_KINDS.has(kind as EngineUsageFrame["engine"]);
|
|
599
|
+
}
|
|
600
|
+
|
|
571
601
|
/** ADR-154 §6: the exact lease material a spawn starts on: an AI account's,
|
|
572
602
|
* or, for the Managed Engine, the task's open run alone (its agent's task
|
|
573
603
|
* token is its credential). */
|
|
@@ -762,6 +792,10 @@ export class Orchestrator {
|
|
|
762
792
|
* later command cannot overtake an earlier one after channel close. */
|
|
763
793
|
private readonly taskLifecycleTails = new Map<string, Promise<void>>();
|
|
764
794
|
private readonly hostSubscribers = new Set<HostEventSubscriber>();
|
|
795
|
+
private readonly engineUsageSubscribers = new Set<EngineUsageSubscriber>();
|
|
796
|
+
/** Last payload (excluding report id/time) for each leased account + agent.
|
|
797
|
+
* Structured Claude events can repeat without any state change. */
|
|
798
|
+
private readonly lastEngineUsage = new Map<string, DeliveredEngineUsage>();
|
|
765
799
|
/** ADR-154 §6: launch slots retired whose retirement has not yet reached
|
|
766
800
|
* their runtime, per task, whatever channel they were in: an agent's next
|
|
767
801
|
* spawn launches there only once they have (`retireLeasedLaunch`). */
|
|
@@ -779,6 +813,11 @@ export class Orchestrator {
|
|
|
779
813
|
return () => this.hostSubscribers.delete(fn);
|
|
780
814
|
}
|
|
781
815
|
|
|
816
|
+
subscribeEngineUsageFrames(fn: EngineUsageSubscriber): () => void {
|
|
817
|
+
this.engineUsageSubscribers.add(fn);
|
|
818
|
+
return () => this.engineUsageSubscribers.delete(fn);
|
|
819
|
+
}
|
|
820
|
+
|
|
782
821
|
/**
|
|
783
822
|
* The cloud credential composer changed the MCP view it serves for a task.
|
|
784
823
|
* Its committed lease view, not the independently delivered channel spec,
|
|
@@ -896,6 +935,30 @@ export class Orchestrator {
|
|
|
896
935
|
}
|
|
897
936
|
}
|
|
898
937
|
|
|
938
|
+
private emitEngineUsage(frame: EngineUsageFrame): boolean {
|
|
939
|
+
let delivered = false;
|
|
940
|
+
for (const fn of this.engineUsageSubscribers) {
|
|
941
|
+
try {
|
|
942
|
+
delivered = fn(frame) || delivered;
|
|
943
|
+
} catch {
|
|
944
|
+
// Usage is best-effort; a broken bridge subscriber cannot affect a
|
|
945
|
+
// running agent turn.
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
return delivered;
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
private forgetEngineUsage(taskId: string, agentId?: string): void {
|
|
952
|
+
for (const [key, delivered] of this.lastEngineUsage) {
|
|
953
|
+
if (
|
|
954
|
+
delivered.taskId === taskId &&
|
|
955
|
+
(agentId === undefined || delivered.agentId === agentId)
|
|
956
|
+
) {
|
|
957
|
+
this.lastEngineUsage.delete(key);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
|
|
899
962
|
// -- channel lifecycle ----------------------------------------------------
|
|
900
963
|
|
|
901
964
|
registerChannelSpec(spec: ChannelEnsureInput): void {
|
|
@@ -2595,7 +2658,6 @@ export class Orchestrator {
|
|
|
2595
2658
|
task.ownerUserId,
|
|
2596
2659
|
apiUrl,
|
|
2597
2660
|
cliSecret,
|
|
2598
|
-
this.isPromptLevelSecretary(channel, agent.id),
|
|
2599
2661
|
),
|
|
2600
2662
|
task,
|
|
2601
2663
|
),
|
|
@@ -4371,7 +4433,6 @@ export class Orchestrator {
|
|
|
4371
4433
|
task.ownerUserId,
|
|
4372
4434
|
apiUrl,
|
|
4373
4435
|
cliSecret,
|
|
4374
|
-
this.isPromptLevelSecretary(channel, agent.id),
|
|
4375
4436
|
),
|
|
4376
4437
|
task,
|
|
4377
4438
|
),
|
|
@@ -4588,6 +4649,7 @@ export class Orchestrator {
|
|
|
4588
4649
|
* could silently restart canceled work on the new generation.
|
|
4589
4650
|
*/
|
|
4590
4651
|
private retireRunnerGeneration(channel: Channel, agentId: string): void {
|
|
4652
|
+
this.forgetEngineUsage(channel.taskId, agentId);
|
|
4591
4653
|
const aborted = channel.interrupted.delete(agentId);
|
|
4592
4654
|
channel.activeTurns.delete(agentId);
|
|
4593
4655
|
channel.acceptedPrompts.delete(agentId);
|
|
@@ -4931,6 +4993,105 @@ export class Orchestrator {
|
|
|
4931
4993
|
): Promise<void> {
|
|
4932
4994
|
if (!this.isActiveChannel(channel)) return;
|
|
4933
4995
|
switch (event.type) {
|
|
4996
|
+
case "account_usage": {
|
|
4997
|
+
// Only the exact live session's cloud-leased ACCOUNT may be named on
|
|
4998
|
+
// the wire. A runner default slot and the Managed Engine's task-token
|
|
4999
|
+
// identity are deliberately unreportable.
|
|
5000
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
5001
|
+
const identity = channel.leasedIdentityByAgent.get(agentId);
|
|
5002
|
+
if (!identity || identity.subject !== "account") break;
|
|
5003
|
+
const agent = channel.roster.find((candidate) => candidate.id === agentId);
|
|
5004
|
+
if (!agent || !isEngineUsageKind(agent.kind)) break;
|
|
5005
|
+
|
|
5006
|
+
const coverage = event.usage.coverage;
|
|
5007
|
+
if (event.usage.status === "limited") {
|
|
5008
|
+
const limitedWindows =
|
|
5009
|
+
coverage?.kind === "all-windows"
|
|
5010
|
+
? event.usage.windows
|
|
5011
|
+
.filter((window) => (window.usedPercent ?? 0) >= 100)
|
|
5012
|
+
.map((window) => window.window)
|
|
5013
|
+
: coverage?.kind === "window"
|
|
5014
|
+
? [coverage.window]
|
|
5015
|
+
: [];
|
|
5016
|
+
const causes = limitedWindows.length > 0 ? limitedWindows : [null];
|
|
5017
|
+
for (const window of causes) {
|
|
5018
|
+
if (window === null) {
|
|
5019
|
+
if (event.usage.resetsAt === undefined) {
|
|
5020
|
+
cooldownEngineAccount(identity.credentialId);
|
|
5021
|
+
} else {
|
|
5022
|
+
cooldownEngineAccountUntil(
|
|
5023
|
+
identity.credentialId,
|
|
5024
|
+
event.usage.resetsAt,
|
|
5025
|
+
);
|
|
5026
|
+
}
|
|
5027
|
+
} else if (event.usage.resetsAt === undefined) {
|
|
5028
|
+
cooldownEngineAccountWindow(identity.credentialId, window);
|
|
5029
|
+
} else {
|
|
5030
|
+
cooldownEngineAccountWindowUntil(
|
|
5031
|
+
identity.credentialId,
|
|
5032
|
+
window,
|
|
5033
|
+
event.usage.resetsAt,
|
|
5034
|
+
);
|
|
5035
|
+
}
|
|
5036
|
+
}
|
|
5037
|
+
} else if (coverage?.kind === "window") {
|
|
5038
|
+
clearEngineAccountCooldownWindow(
|
|
5039
|
+
identity.credentialId,
|
|
5040
|
+
coverage.window,
|
|
5041
|
+
);
|
|
5042
|
+
} else if (coverage?.kind === "all-windows") {
|
|
5043
|
+
clearEngineAccountCooldowns(identity.credentialId);
|
|
5044
|
+
}
|
|
5045
|
+
|
|
5046
|
+
const key = `${identity.credentialId}\0${agentId}`;
|
|
5047
|
+
const payload = JSON.stringify({
|
|
5048
|
+
taskId: channel.taskId,
|
|
5049
|
+
engine: agent.kind,
|
|
5050
|
+
credentialId: identity.credentialId,
|
|
5051
|
+
status: event.usage.status,
|
|
5052
|
+
resetsAt: event.usage.resetsAt,
|
|
5053
|
+
usingOverage: event.usage.usingOverage,
|
|
5054
|
+
windows: event.usage.windows,
|
|
5055
|
+
source: event.usage.source,
|
|
5056
|
+
});
|
|
5057
|
+
const now = Date.now();
|
|
5058
|
+
const previous = this.lastEngineUsage.get(key);
|
|
5059
|
+
if (
|
|
5060
|
+
previous?.payload === payload &&
|
|
5061
|
+
now - previous.sentAt < ENGINE_USAGE_REFRESH_MS
|
|
5062
|
+
) {
|
|
5063
|
+
break;
|
|
5064
|
+
}
|
|
5065
|
+
const delivered = this.emitEngineUsage({
|
|
5066
|
+
kind: "engine.usage",
|
|
5067
|
+
reportId: newId(),
|
|
5068
|
+
taskId: channel.taskId,
|
|
5069
|
+
agentId,
|
|
5070
|
+
engine: agent.kind,
|
|
5071
|
+
credentialId: identity.credentialId,
|
|
5072
|
+
status: event.usage.status,
|
|
5073
|
+
...(event.usage.resetsAt === undefined
|
|
5074
|
+
? {}
|
|
5075
|
+
: { resetsAt: event.usage.resetsAt }),
|
|
5076
|
+
...(event.usage.usingOverage === undefined
|
|
5077
|
+
? {}
|
|
5078
|
+
: { usingOverage: event.usage.usingOverage }),
|
|
5079
|
+
windows: event.usage.windows,
|
|
5080
|
+
source: event.usage.source,
|
|
5081
|
+
observedAt: event.usage.observedAt,
|
|
5082
|
+
});
|
|
5083
|
+
// A disconnected bridge returns false. Do not suppress the next
|
|
5084
|
+
// identical observation unless this one actually reached the socket.
|
|
5085
|
+
if (delivered) {
|
|
5086
|
+
this.lastEngineUsage.set(key, {
|
|
5087
|
+
payload,
|
|
5088
|
+
sentAt: now,
|
|
5089
|
+
taskId: channel.taskId,
|
|
5090
|
+
agentId,
|
|
5091
|
+
});
|
|
5092
|
+
}
|
|
5093
|
+
break;
|
|
5094
|
+
}
|
|
4934
5095
|
case "message_delta": {
|
|
4935
5096
|
this.markActiveTurn(channel, agentId);
|
|
4936
5097
|
channel.openTurns.add(agentId);
|
|
@@ -5065,6 +5226,7 @@ export class Orchestrator {
|
|
|
5065
5226
|
agentId,
|
|
5066
5227
|
agent,
|
|
5067
5228
|
abortBoundaryOnRosterChange,
|
|
5229
|
+
event.resetsAt,
|
|
5068
5230
|
))
|
|
5069
5231
|
) {
|
|
5070
5232
|
break;
|
|
@@ -5488,7 +5650,6 @@ export class Orchestrator {
|
|
|
5488
5650
|
task.ownerUserId,
|
|
5489
5651
|
apiUrl,
|
|
5490
5652
|
cliSecret,
|
|
5491
|
-
this.isPromptLevelSecretary(channel, agent.id),
|
|
5492
5653
|
);
|
|
5493
5654
|
const boundAccountId = channel.accountByAgent.get(agentId);
|
|
5494
5655
|
const boundAccount = boundAccountId
|
|
@@ -5586,6 +5747,7 @@ export class Orchestrator {
|
|
|
5586
5747
|
agentId: string,
|
|
5587
5748
|
agent: RosterAgent,
|
|
5588
5749
|
abortBoundaryOnRosterChange = false,
|
|
5750
|
+
resetsAt?: number,
|
|
5589
5751
|
): Promise<boolean> {
|
|
5590
5752
|
if (!this.isActiveChannel(channel)) return true;
|
|
5591
5753
|
if (channel.rosterSessionChanges.has(agentId)) {
|
|
@@ -5609,7 +5771,10 @@ export class Orchestrator {
|
|
|
5609
5771
|
if (budget > MAX_ROTATIONS_PER_AGENT) return false;
|
|
5610
5772
|
|
|
5611
5773
|
const current = channel.accountByAgent.get(agentId);
|
|
5612
|
-
if (current)
|
|
5774
|
+
if (current) {
|
|
5775
|
+
if (resetsAt === undefined) cooldownEngineAccount(current);
|
|
5776
|
+
else cooldownEngineAccountUntil(current, resetsAt);
|
|
5777
|
+
}
|
|
5613
5778
|
const next = this.pickAccount(
|
|
5614
5779
|
channel,
|
|
5615
5780
|
agent.kind,
|
|
@@ -5698,7 +5863,6 @@ export class Orchestrator {
|
|
|
5698
5863
|
task.ownerUserId,
|
|
5699
5864
|
apiUrl,
|
|
5700
5865
|
cliSecret,
|
|
5701
|
-
this.isPromptLevelSecretary(channel, agent.id),
|
|
5702
5866
|
);
|
|
5703
5867
|
channel.accountByAgent.set(agentId, next.id);
|
|
5704
5868
|
noteEngineAccountUsed(next.id);
|
|
@@ -5908,7 +6072,6 @@ export class Orchestrator {
|
|
|
5908
6072
|
task.ownerUserId,
|
|
5909
6073
|
apiUrl,
|
|
5910
6074
|
cliSecret,
|
|
5911
|
-
this.isPromptLevelSecretary(channel, agent.id),
|
|
5912
6075
|
);
|
|
5913
6076
|
channel.accountByAgent.set(agentId, account.id);
|
|
5914
6077
|
noteEngineAccountUsed(account.id);
|
|
@@ -6161,6 +6324,7 @@ export class Orchestrator {
|
|
|
6161
6324
|
this.channelSpecs.delete(taskId);
|
|
6162
6325
|
const ch = this.channels.get(taskId);
|
|
6163
6326
|
if (!ch) return Promise.resolve();
|
|
6327
|
+
this.forgetEngineUsage(taskId);
|
|
6164
6328
|
// Invalidate the generation BEFORE the first await. Browser setup and the
|
|
6165
6329
|
// start/reconcile factory continuations all check this identity, so none
|
|
6166
6330
|
// can repopulate an orphan channel while its sessions close.
|
|
@@ -6893,8 +7057,8 @@ export function buildSystemPreamble(
|
|
|
6893
7057
|
" (at least one of the three; `default` as the model or effort clears it to",
|
|
6894
7058
|
" the engine's own default)",
|
|
6895
7059
|
"- `crew remove @agent --why \"…\"`",
|
|
6896
|
-
"Use the smallest crew the job needs, often none
|
|
6897
|
-
"
|
|
7060
|
+
"Use the smallest crew the job needs, often none, and hire only when",
|
|
7061
|
+
"parallel work or a different skill or model pays for itself.",
|
|
6898
7062
|
"A new agent is idle until you `dispatch` to it, so brief it then. Remove an",
|
|
6899
7063
|
"agent when its part is done; its handle is not reused, so hiring that",
|
|
6900
7064
|
"persona again gives it a new one. A model change restarts that agent and drops",
|
|
@@ -6919,18 +7083,24 @@ export function buildSystemPreamble(
|
|
|
6919
7083
|
"connections and persona permissions of any task agent, except native",
|
|
6920
7084
|
"subagents: helpers your engine starts on its own would work outside the",
|
|
6921
7085
|
"crew, unpriced and unseen by the humans, so they are switched off. Work",
|
|
6922
|
-
"for another agent goes to a crew agent through `dispatch`.
|
|
6923
|
-
|
|
7086
|
+
"for another agent goes to a crew agent through `dispatch`.",
|
|
7087
|
+
// One rule for who does the work, the plan's, so this section and
|
|
7088
|
+
// "Your crew" cannot disagree: the crew may be nobody.
|
|
7089
|
+
"Who does each step is your plan's call: do it yourself when one agent",
|
|
7090
|
+
"is enough, hand it to the crew when parallel work or another skill or",
|
|
7091
|
+
"model pays for itself. A human's request for you to do something",
|
|
7092
|
+
"yourself overrides the plan.",
|
|
7093
|
+
"",
|
|
6924
7094
|
]
|
|
6925
7095
|
: [
|
|
6926
7096
|
"Your Secretary role is prompt-guided, not tool-restricted. You have the",
|
|
6927
7097
|
"same normal tools, approved MCP connections, model choice, and persona",
|
|
6928
7098
|
"permissions as any other task agent. Your default job is to communicate,",
|
|
7099
|
+
"coordinate, and dispatch implementation work rather than doing it yourself.",
|
|
7100
|
+
"If the human explicitly asks YOU to make a change, you may use those tools",
|
|
7101
|
+
"and do it directly; otherwise hand mutations to a crew agent.",
|
|
7102
|
+
"",
|
|
6929
7103
|
]),
|
|
6930
|
-
"coordinate, and dispatch implementation work rather than doing it yourself.",
|
|
6931
|
-
"If the human explicitly asks YOU to make a change, you may use those tools",
|
|
6932
|
-
"and do it directly; otherwise hand mutations to a crew agent.",
|
|
6933
|
-
"",
|
|
6934
7104
|
]
|
|
6935
7105
|
: [];
|
|
6936
7106
|
const groupMessageBrief = isSecretary
|
|
@@ -7280,7 +7450,7 @@ export function buildSystemPreamble(
|
|
|
7280
7450
|
? ", `todo list|add|edit|remove|claim|done`"
|
|
7281
7451
|
: "") +
|
|
7282
7452
|
(agent.permissions?.includes("previews.write")
|
|
7283
|
-
? ", `preview add <name
|
|
7453
|
+
? ", `preview add|enable|public <name>`"
|
|
7284
7454
|
: "") +
|
|
7285
7455
|
(agent.permissions?.includes("projects.write")
|
|
7286
7456
|
? ", `project create --name --repo`"
|
|
@@ -7355,8 +7525,10 @@ export function buildSystemPreamble(
|
|
|
7355
7525
|
...(agent.permissions?.includes("previews.write")
|
|
7356
7526
|
? [
|
|
7357
7527
|
"**Previews:** started a dev server? `preview add <name>",
|
|
7358
|
-
"<containerPort>`
|
|
7359
|
-
"
|
|
7528
|
+
"<containerPort>` registers it, then `preview enable <name>` turns",
|
|
7529
|
+
"it on and prints its URL: give that URL to the humans. It needs",
|
|
7530
|
+
"a sign-in; `preview public <name>` makes the task's previews",
|
|
7531
|
+
"a link anyone can open — only when a human asks for that.",
|
|
7360
7532
|
]
|
|
7361
7533
|
: []),
|
|
7362
7534
|
"",
|
|
@@ -64,7 +64,8 @@ export interface TaskCredentialCleanupDeps {
|
|
|
64
64
|
db?: Db;
|
|
65
65
|
hooks: TaskCredentialCleanupHooks;
|
|
66
66
|
removePath?: (row: HostCredentialCleanupPath) => Promise<void>;
|
|
67
|
-
/** Full cleanup
|
|
67
|
+
/** Full cleanup under the cloud's taskDown (`reason: "teardown"`) only:
|
|
68
|
+
* destroy one exact runtime and prove it absent. */
|
|
68
69
|
destroyRuntime?: (request: {
|
|
69
70
|
taskId: string;
|
|
70
71
|
runEpoch: number;
|
|
@@ -464,9 +465,18 @@ export class TaskCredentialCleanupCoordinator {
|
|
|
464
465
|
}
|
|
465
466
|
|
|
466
467
|
// A whole-task cleanup may prove absence by destroying the exact writable
|
|
467
|
-
// runtime instead
|
|
468
|
+
// runtime instead, but ONLY inside the cloud's taskDown of this run (the
|
|
469
|
+
// durable reason `teardown` has no other source). Every other cleanup —
|
|
470
|
+
// boot recovery, a fence escalation, stop, park, assignment — defers: the
|
|
471
|
+
// gate stays closed, the pending witness and inventory rows stay, and the
|
|
472
|
+
// registry's retry tries the exact paths again later. A failed exec is
|
|
473
|
+
// not proof of absence; Docker briefly gone (an OrbStack update,
|
|
474
|
+
// 2026-10-20) once turned this fallback into task-down.sh on every live
|
|
475
|
+
// task, deleting their worktrees. Selective removal must never take this
|
|
476
|
+
// broader action either.
|
|
468
477
|
if (
|
|
469
478
|
!consumersFailed &&
|
|
479
|
+
request.reason === "teardown" &&
|
|
470
480
|
request.credentialIds === undefined &&
|
|
471
481
|
failed.length > 0
|
|
472
482
|
) {
|
|
@@ -1015,7 +1015,8 @@ export function createMachineHostTaskEnvironmentProvider(
|
|
|
1015
1015
|
// timeouts bound the wait). A teardown left for later could take a
|
|
1016
1016
|
// replacement, which reuses the task's machine name.
|
|
1017
1017
|
const late = await provisioning.catch(() => null);
|
|
1018
|
-
|
|
1018
|
+
// A reused machine holds the task's work: stopped, never destroyed.
|
|
1019
|
+
if (late) await (late.reused ? late.handle.stop() : late.handle.teardown()).catch(() => {});
|
|
1019
1020
|
}
|
|
1020
1021
|
throw error;
|
|
1021
1022
|
}
|
|
@@ -1043,7 +1044,10 @@ export function createMachineHostTaskEnvironmentProvider(
|
|
|
1043
1044
|
const abandon = (): Promise<void> => {
|
|
1044
1045
|
over = true;
|
|
1045
1046
|
abandoning ??= (async () => {
|
|
1046
|
-
|
|
1047
|
+
// Only a machine this start launched is destroyed. One it reused (a
|
|
1048
|
+
// Resume of a stopped machine) holds the task's work on its disk, so
|
|
1049
|
+
// a failed start stops it again and the next Resume tries afresh.
|
|
1050
|
+
await (provisioned.reused ? handle.stop() : handle.teardown()).catch(() => {});
|
|
1047
1051
|
await joinBounded(begun, deps.abandonJoinMs ?? ABANDON_JOIN_MS);
|
|
1048
1052
|
})();
|
|
1049
1053
|
return abandoning;
|
|
@@ -862,6 +862,7 @@ export function createMachineTaskEnvironmentProvider(
|
|
|
862
862
|
`machine ${machineId} state is unknown: ${existing.detail ?? "no detail"}`,
|
|
863
863
|
);
|
|
864
864
|
}
|
|
865
|
+
const reused = existing.state !== "absent";
|
|
865
866
|
if (existing.state === "absent") {
|
|
866
867
|
const launched = await deps.machines.launch({
|
|
867
868
|
taskId,
|
|
@@ -935,7 +936,7 @@ export function createMachineTaskEnvironmentProvider(
|
|
|
935
936
|
);
|
|
936
937
|
}
|
|
937
938
|
}
|
|
938
|
-
return { handle, result: handle.descriptor };
|
|
939
|
+
return { handle, result: handle.descriptor, reused };
|
|
939
940
|
},
|
|
940
941
|
|
|
941
942
|
async reconstruct(
|
|
@@ -349,6 +349,10 @@ export interface TaskEnvironmentProvisioned<TResult, TTeardownResult = void> {
|
|
|
349
349
|
handle: TaskEnvironmentHandle<TTeardownResult>;
|
|
350
350
|
/** Provider launch result retained for rolling wire compatibility. */
|
|
351
351
|
result: TResult;
|
|
352
|
+
/** The provider reused an environment that already existed — a Resume's
|
|
353
|
+
* stopped machine, whose disk holds the task's work — rather than creating
|
|
354
|
+
* it. A failed start stops such an environment; it never destroys it. */
|
|
355
|
+
reused?: boolean;
|
|
352
356
|
}
|
|
353
357
|
|
|
354
358
|
export interface TaskEnvironmentProvider<
|
package/package.json
CHANGED
|
@@ -5,6 +5,13 @@
|
|
|
5
5
|
# 2. rm -rf the whole task dir — workspace + task-private Git repositories.
|
|
6
6
|
# Host caches under projects/<id>/host-cache.git persist.
|
|
7
7
|
# 3. Persist final state: status=killed (unless already error/shipped).
|
|
8
|
+
#
|
|
9
|
+
# Destruction needs explicit terminal intent: the runner sets
|
|
10
|
+
# UAI_TASK_DOWN_TERMINAL=1 only on the cloud taskDown path (a task close or
|
|
11
|
+
# orphan GC). Without it the script refuses before touching Docker or disk —
|
|
12
|
+
# a stray caller (recovery, a credential-cleanup fallback, an operator
|
|
13
|
+
# running the script by hand) must never delete a live task's worktree
|
|
14
|
+
# (2026-10-20: a cleanup fallback did, while Docker was briefly gone).
|
|
8
15
|
|
|
9
16
|
set -euo pipefail
|
|
10
17
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
@@ -20,6 +27,12 @@ if [ "${#task_id}" -gt 128 ] ||
|
|
|
20
27
|
[[ ! "$task_id" =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then
|
|
21
28
|
emit_err "BAD_ARGS" "invalid task id" "task identity validation"
|
|
22
29
|
fi
|
|
30
|
+
# Before UAI_TASK_ID is set: a refusal must not mark the live task errored.
|
|
31
|
+
if [ "${UAI_TASK_DOWN_TERMINAL:-}" != "1" ]; then
|
|
32
|
+
emit_err "BAD_ARGS" \
|
|
33
|
+
"task-down requires terminal intent (UAI_TASK_DOWN_TERMINAL=1, set only by the cloud taskDown); refusing to destroy task $task_id" \
|
|
34
|
+
"terminal intent"
|
|
35
|
+
fi
|
|
23
36
|
UAI_TASK_ID="$task_id"
|
|
24
37
|
setup_err_trap
|
|
25
38
|
validate_task_runtime_selection
|
package/scripts/agent/task-up.sh
CHANGED
|
@@ -1453,9 +1453,39 @@ while IFS= read -r project_obj; do
|
|
|
1453
1453
|
git_host_control -C "$worktree_repo_dir" worktree repair \
|
|
1454
1454
|
"$worktree_target" 2>/dev/null || true
|
|
1455
1455
|
fi
|
|
1456
|
+
elif [ -n "$task_branch" ] && [ "$task_branch" != "null" ] \
|
|
1457
|
+
&& [ "$task_branch" != "$project_default" ] \
|
|
1458
|
+
&& ! git_host_control -C "$worktree_repo_dir" rev-parse --verify --quiet \
|
|
1459
|
+
"refs/heads/$task_branch" >/dev/null 2>&1 \
|
|
1460
|
+
&& git_host_control -C "$worktree_repo_dir" rev-parse --verify --quiet \
|
|
1461
|
+
"refs/remotes/origin/$task_branch^{commit}" >/dev/null 2>&1; then
|
|
1462
|
+
# The worktree is gone but the task's branch is on the remote (a lost
|
|
1463
|
+
# task directory, 2026-10-20: Resume rebuilt from the base, putting the
|
|
1464
|
+
# agents on the wrong code with the pushed commits out of sight). The
|
|
1465
|
+
# fetch above is fresh — a missing worktree makes a failed fetch fatal —
|
|
1466
|
+
# so continue from the pushed branch, never the base.
|
|
1467
|
+
step "WORKTREE_FAILED" "git worktree add from origin/$task_branch ($project_id)"
|
|
1468
|
+
log "worktree for $project_slug missing — restoring it from origin/$task_branch"
|
|
1469
|
+
# The base this task branched from, as nearly as the remote can tell:
|
|
1470
|
+
# where the task branch leaves the default branch.
|
|
1471
|
+
env_schema_oid=$(git_host_control -C "$mirror_dir" merge-base \
|
|
1472
|
+
"refs/remotes/origin/$task_branch" "refs/remotes/origin/$project_default" \
|
|
1473
|
+
2>/dev/null || true)
|
|
1474
|
+
if [ -n "$env_schema_oid" ]; then
|
|
1475
|
+
record_env_schema_base "$project_id" "oid $env_schema_oid" \
|
|
1476
|
+
|| log "warning: could not record the .env.schema base for $project_id; its keys without a binding are withheld"
|
|
1477
|
+
fi
|
|
1478
|
+
# --no-track: the sanitizer above already wrote this branch's upstream.
|
|
1479
|
+
if ! git_host_control -C "$worktree_repo_dir" worktree add --no-track \
|
|
1480
|
+
"$worktree_target" -b "$task_branch" "refs/remotes/origin/$task_branch"; then
|
|
1481
|
+
emit_err "WORKTREE_FAILED" \
|
|
1482
|
+
"could not restore the task worktree from origin/$task_branch for $repo_url" \
|
|
1483
|
+
"git worktree add from origin/$task_branch ($project_id)"
|
|
1484
|
+
fi
|
|
1456
1485
|
elif git_host_control -C "$worktree_repo_dir" rev-parse --verify --quiet \
|
|
1457
1486
|
"refs/remotes/origin/$project_default" >/dev/null 2>&1; then
|
|
1458
1487
|
# Normal case: branch the task worktree off origin/<defaultBranch>.
|
|
1488
|
+
# Only when the remote has no branch of this task's yet.
|
|
1459
1489
|
step "WORKTREE_FAILED" "git worktree add ($project_id)"
|
|
1460
1490
|
# The same commit the task repository just copied, read where the task
|
|
1461
1491
|
# never writes: the host cache, under this project's lock.
|
package/src/main.ts
CHANGED
|
@@ -285,6 +285,7 @@ import { hostCommands, hostEvents, setManagedHostRestartHook } from "./index";
|
|
|
285
285
|
import { EventOutbox, transmitEventOutbox } from "./event-outbox";
|
|
286
286
|
import {
|
|
287
287
|
HostErrorCode,
|
|
288
|
+
ENGINE_USAGE_PROTOCOL_FEATURE,
|
|
288
289
|
EVENT_REPLAY_PROTOCOL_FEATURE,
|
|
289
290
|
GITHUB_CREDENTIAL_GENERATION_PROTOCOL_FEATURE,
|
|
290
291
|
GITHUB_INSTALLATION_VERIFICATION_PROTOCOL_FEATURE,
|
|
@@ -1486,6 +1487,7 @@ function buildCapabilities(): HostCapabilities {
|
|
|
1486
1487
|
HOST_ENGINE_ACCOUNT_SCOPES_PROTOCOL_FEATURE,
|
|
1487
1488
|
HOST_ENGINE_ACCOUNT_MANAGE_PROTOCOL_FEATURE,
|
|
1488
1489
|
HOST_ENGINE_DISCONNECT_PROTOCOL_FEATURE,
|
|
1490
|
+
ENGINE_USAGE_PROTOCOL_FEATURE,
|
|
1489
1491
|
LIVE_ROSTER_EDIT_PROTOCOL_FEATURE,
|
|
1490
1492
|
AGENT_TOKEN_GENERATION_PROTOCOL_FEATURE,
|
|
1491
1493
|
// The echo adapter cannot execute the in-task CLI. Advertising typed
|
|
@@ -2874,6 +2876,11 @@ async function connect(): Promise<void> {
|
|
|
2874
2876
|
return false;
|
|
2875
2877
|
}
|
|
2876
2878
|
};
|
|
2879
|
+
const stopEngineUsageSubscription = getOrchestrator().subscribeEngineUsageFrames(
|
|
2880
|
+
(frame) => {
|
|
2881
|
+
return sendRemoteOperationFrame(frame);
|
|
2882
|
+
},
|
|
2883
|
+
);
|
|
2877
2884
|
|
|
2878
2885
|
let projectEnvOperationTail = Promise.resolve();
|
|
2879
2886
|
const projectEnvGenerationIsCurrent = (): boolean =>
|
|
@@ -2916,6 +2923,7 @@ async function connect(): Promise<void> {
|
|
|
2916
2923
|
};
|
|
2917
2924
|
|
|
2918
2925
|
const cleanup = (): void => {
|
|
2926
|
+
stopEngineUsageSubscription();
|
|
2919
2927
|
// Connection loss revokes the serving gate before deferred transport
|
|
2920
2928
|
// cleanup yields. A late close from a superseded socket must not revoke a
|
|
2921
2929
|
// replacement generation that has already committed its fresh snapshot.
|
package/src/protocol.ts
CHANGED
|
@@ -156,6 +156,60 @@ export const HOST_ENGINE_ACCOUNT_MANAGE_PROTOCOL_FEATURE =
|
|
|
156
156
|
"host-engine-account-manage-v1";
|
|
157
157
|
export const HOST_ENGINE_ACCOUNT_SCOPES_PROTOCOL_FEATURE =
|
|
158
158
|
"host-engine-account-scopes-v1";
|
|
159
|
+
/** AI-account usage (ADR-157 slice 3, amended 2026-10-21): the host reports
|
|
160
|
+
* what an engine CLI said about the account an agent runs on — Claude Code's
|
|
161
|
+
* `rate_limit_event`, Codex's `account/rateLimits` — and the reset time of a
|
|
162
|
+
* cut-off, as `engine.usage` frames. Reports are best-effort and
|
|
163
|
+
* self-superseding: a lost one is replaced by the next. The cloud keeps the
|
|
164
|
+
* latest per account and never asks the provider itself (subscription
|
|
165
|
+
* sign-ins are for the engine CLIs only). A host advertises this once it
|
|
166
|
+
* sends the frames; the cloud needs nothing from a host without it. */
|
|
167
|
+
export const ENGINE_USAGE_PROTOCOL_FEATURE = "engine-usage-v1";
|
|
168
|
+
export const MAX_ENGINE_USAGE_WINDOWS = 8;
|
|
169
|
+
|
|
170
|
+
/** One usage window of an AI account, in the engine's own terms. Every
|
|
171
|
+
* measurement is optional: an engine reports what it reports. */
|
|
172
|
+
export interface EngineUsageWindow {
|
|
173
|
+
/** The engine's name for the window: Claude's `rateLimitType`
|
|
174
|
+
* (`five_hour`, `seven_day`, `seven_day_opus`, …) or Codex's `primary` /
|
|
175
|
+
* `secondary`. At most 64 characters. */
|
|
176
|
+
window: string;
|
|
177
|
+
/** Window length in minutes. */
|
|
178
|
+
windowMinutes?: number;
|
|
179
|
+
/** Share of the window used, 0–100. */
|
|
180
|
+
usedPercent?: number;
|
|
181
|
+
/** When the window resets, epoch ms. */
|
|
182
|
+
resetsAt?: number;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** One usage report for one leased AI account (ADR-154 `credentialId`).
|
|
186
|
+
* Accounts that are not cloud credentials (a host's own default slot) are
|
|
187
|
+
* never reported. */
|
|
188
|
+
export interface EngineUsageFrame {
|
|
189
|
+
kind: "engine.usage";
|
|
190
|
+
/** Host-minted, unique per report; a repeat is ignored. */
|
|
191
|
+
reportId: string;
|
|
192
|
+
taskId: string;
|
|
193
|
+
agentId: string;
|
|
194
|
+
engine: "claude" | "codex" | "grok" | "cursor" | "opencode";
|
|
195
|
+
/** The `uai_credentials` id the agent's session runs on. */
|
|
196
|
+
credentialId: string;
|
|
197
|
+
/** `allowed`: working; `warning`: the engine says it is close to a limit;
|
|
198
|
+
* `limited`: cut off until `resetsAt` (or an unknown time). */
|
|
199
|
+
status: "allowed" | "warning" | "limited";
|
|
200
|
+
/** When a `limited` account should work again, epoch ms, if known. */
|
|
201
|
+
resetsAt?: number;
|
|
202
|
+
/** Whether the engine says it is now billing past the plan (Claude
|
|
203
|
+
* "overage"), when it says. */
|
|
204
|
+
usingOverage?: boolean;
|
|
205
|
+
windows: EngineUsageWindow[];
|
|
206
|
+
/** What the report was read from. */
|
|
207
|
+
source: "claude-rate-limit-event" | "codex-rate-limits" | "error-text";
|
|
208
|
+
/** When the engine said it, epoch ms (host clock). The cloud keeps the
|
|
209
|
+
* newest per account, so a late report never overwrites a newer one. */
|
|
210
|
+
observedAt: number;
|
|
211
|
+
}
|
|
212
|
+
|
|
159
213
|
/** ADR-049 addendum: live roster edits. On channel.ensure the host diffs the
|
|
160
214
|
* live roster against the spec — an agent whose kind, model or effort changed
|
|
161
215
|
* has its session recycled so the reconciler respawns it on the new settings,
|
|
@@ -4806,6 +4860,7 @@ export type HostToCloud =
|
|
|
4806
4860
|
// cloud's replay watermark only applies within a matching boot. Optional
|
|
4807
4861
|
// for pre-103 hosts (which also never send seq'd events).
|
|
4808
4862
|
| { kind: "auth"; token: string; hostId: string; bootId?: string }
|
|
4863
|
+
| EngineUsageFrame
|
|
4809
4864
|
| { kind: "host.capabilities"; capabilities: HostCapabilities }
|
|
4810
4865
|
| {
|
|
4811
4866
|
kind: "host.inventory.page";
|