@jc_stack/ez-agents 0.1.0-beta.52 → 0.1.0-beta.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -1
- package/docs/architecture/ai-selection.md +6 -2
- package/docs/docker-runtime.md +48 -0
- package/docs/scheduling.md +124 -16
- package/docs/upgrades.md +9 -4
- package/package.json +1 -1
- package/src/ai-cli.ts +31 -5
- package/src/ai.ts +17 -1
- package/src/codex-session.ts +23 -2
- package/src/control-state.ts +173 -8
- package/src/delivery-socket.ts +18 -1
- package/src/executor.ts +11 -4
- package/src/host-executor.ts +17 -6
- package/src/index.ts +358 -269
- package/src/run-telemetry.ts +163 -0
- package/src/running-work.ts +123 -0
- package/src/runs.ts +7 -2
- package/src/schedule-cli.ts +55 -16
- package/src/scheduler.ts +126 -29
- package/src/setup-fallback.ts +57 -18
- package/src/updates/supervisor.mjs +23 -8
- package/test/chat-fallback.test.ts +358 -0
- package/test/codex-session.test.ts +1 -1
- package/test/control-state.test.ts +62 -0
- package/test/executor.test.ts +2 -1
- package/test/host-executor.test.ts +41 -0
- package/test/run-handoff.test.ts +54 -0
- package/test/running-work.test.ts +175 -0
- package/test/schedule-cli.test.ts +55 -6
- package/test/scheduler.test.ts +128 -1
- package/test/script-schedules.test.ts +3 -3
- package/test/setup-fallback.test.ts +211 -14
- package/test/updates.test.mjs +25 -4
- package/test/upgrade-pause.test.ts +65 -2
package/src/control-state.ts
CHANGED
|
@@ -2,8 +2,9 @@ import { assertEffort } from './model-policy.js'
|
|
|
2
2
|
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'
|
|
3
3
|
import { mkdir, open, readFile, rename, rm, writeFile } from 'node:fs/promises'
|
|
4
4
|
import path from 'node:path'
|
|
5
|
-
import { isPreset, persistedPreset, presetProvider, sameEngine, type AiPreset, type ExecutionChoice, type ModelChoice } from './ai.js'
|
|
5
|
+
import { isPreset, persistedPreset, presetProvider, resolveFallbacks, sameEngine, type AiPreset, type ExecutionChoice, type ModelChoice } from './ai.js'
|
|
6
6
|
import { isAuthProfile } from './auth-profile.js'
|
|
7
|
+
import { quotaScope, quotaBlockedUntil, quotaCategory, validSetupHealth, type SetupAttempt, type SetupHealth } from './setup-fallback.js'
|
|
7
8
|
|
|
8
9
|
export type Owner = {
|
|
9
10
|
id?: string
|
|
@@ -55,6 +56,10 @@ type ApplicationPairingRequest = {
|
|
|
55
56
|
|
|
56
57
|
export type PairingRequest = TelegramPairingRequest | ApplicationPairingRequest
|
|
57
58
|
|
|
59
|
+
// A fallback login's own native conversation beside a conversation's engine.
|
|
60
|
+
export type LinkedSession = { cli: string; authProfile?: string; sessionId: string; nativeSessionId?: string; hasStarted: boolean }
|
|
61
|
+
type Engine = { cli: string; authProfile?: string }
|
|
62
|
+
|
|
58
63
|
export type SessionState = {
|
|
59
64
|
sessionId: string
|
|
60
65
|
hasStarted: boolean
|
|
@@ -67,6 +72,8 @@ export type SessionState = {
|
|
|
67
72
|
preset?: AiPreset
|
|
68
73
|
applicationScope?: string
|
|
69
74
|
telegramShared?: boolean
|
|
75
|
+
// One native session per other engine that served this conversation.
|
|
76
|
+
linked?: LinkedSession[]
|
|
70
77
|
}
|
|
71
78
|
|
|
72
79
|
export type ControlGuard = { owner: Owner; authorize: () => Promise<unknown>; expectedSession?: string | null; applicationScope?: string }
|
|
@@ -76,8 +83,12 @@ type ControlState = {
|
|
|
76
83
|
owner: Owner | null
|
|
77
84
|
pending: PairingRequest[]
|
|
78
85
|
activeSession?: SessionState | null
|
|
79
|
-
|
|
86
|
+
// fallbacks: the owner's ordered chat/application chain after the selected AI.
|
|
87
|
+
ai?: { presets: AiPreset[]; defaultId: string; selectedId: string; recentIds?: string[]; fallbacks?: AiPreset[] }
|
|
80
88
|
sessions?: SessionState[]
|
|
89
|
+
setupHealth?: SetupHealth
|
|
90
|
+
// Owner opt-in for per-attempt run telemetry; absent means off.
|
|
91
|
+
runTelemetry?: true
|
|
81
92
|
}
|
|
82
93
|
|
|
83
94
|
type Clock = () => number
|
|
@@ -133,17 +144,26 @@ const isState = (value: unknown): value is ControlState => {
|
|
|
133
144
|
(s.applicationScope === undefined || /^[a-f0-9]{64}$/.test(s.applicationScope)) &&
|
|
134
145
|
(s.telegramShared === undefined || typeof s.telegramShared === 'boolean') &&
|
|
135
146
|
(s.authProfile === undefined || (isAuthProfile(s.authProfile) && s.cli !== undefined)) &&
|
|
136
|
-
(s.preset === undefined || (isPreset(s.preset) && sameEngine(s.preset, s)))
|
|
147
|
+
(s.preset === undefined || (isPreset(s.preset) && sameEngine(s.preset, s))) &&
|
|
148
|
+
(s.linked === undefined || (Array.isArray(s.linked) && s.linked.length <= 16 && s.linked.every((l, i) =>
|
|
149
|
+
l && typeof l.cli === 'string' && /^[a-z-]{1,20}$/.test(l.cli) && (l.authProfile === undefined || isAuthProfile(l.authProfile)) &&
|
|
150
|
+
/^[0-9a-f-]{36}$/i.test(l.sessionId) && typeof l.hasStarted === 'boolean' &&
|
|
151
|
+
(l.nativeSessionId === undefined || /^[a-zA-Z0-9_-]{1,160}$/.test(l.nativeSessionId)) &&
|
|
152
|
+
!sameEngine(l, s) && !s.linked!.slice(0, i).some(other => sameEngine(other, l)))))
|
|
137
153
|
return (
|
|
138
154
|
candidate.version === 1 &&
|
|
139
155
|
Array.isArray(candidate.pending) &&
|
|
140
156
|
candidate.pending.every(pairing) &&
|
|
141
157
|
(candidate.owner === null || validOwner(candidate.owner)) &&
|
|
158
|
+
(candidate.setupHealth === undefined || validSetupHealth(candidate.setupHealth)) &&
|
|
142
159
|
(candidate.ai === undefined || (Array.isArray(candidate.ai.presets) &&
|
|
143
160
|
candidate.ai.presets.every(isPreset) &&
|
|
144
161
|
candidate.ai.presets.some((p) => p.id === candidate.ai!.defaultId) &&
|
|
145
162
|
candidate.ai.presets.some((p) => p.id === candidate.ai!.selectedId) &&
|
|
146
|
-
(candidate.ai.recentIds === undefined || isRecentIds(candidate.ai.recentIds))
|
|
163
|
+
(candidate.ai.recentIds === undefined || isRecentIds(candidate.ai.recentIds)) &&
|
|
164
|
+
(candidate.ai.fallbacks === undefined || (Array.isArray(candidate.ai.fallbacks) && candidate.ai.fallbacks.length > 0 &&
|
|
165
|
+
candidate.ai.fallbacks.length <= 12 && candidate.ai.fallbacks.every(isPreset))))) &&
|
|
166
|
+
(candidate.runTelemetry === undefined || candidate.runTelemetry === true) &&
|
|
147
167
|
(candidate.sessions === undefined || (Array.isArray(candidate.sessions) && candidate.sessions.every(
|
|
148
168
|
session))) &&
|
|
149
169
|
(candidate.activeSession == null ||
|
|
@@ -156,6 +176,15 @@ export const sessionTitle = (session: SessionState): string =>
|
|
|
156
176
|
|
|
157
177
|
const engineOf = (preset: AiPreset) => ({ cli: preset.cli, ...(preset.authProfile ? { authProfile: preset.authProfile } : {}) })
|
|
158
178
|
|
|
179
|
+
// Queued work captures the chain with its primary choice; later edits do not change it.
|
|
180
|
+
const withChain = (choice: ExecutionChoice, chain?: AiPreset[]): ExecutionChoice => {
|
|
181
|
+
const fallbacks = resolveFallbacks(choice.preset, chain)
|
|
182
|
+
return fallbacks ? { ...choice, fallbacks } : choice
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Clients that report their own native session ID; others use the Ez-assigned ID.
|
|
186
|
+
const reportsNativeSession = (cli?: string) => ['codex', 'codex-gui', 'opencode'].includes(cli ?? '')
|
|
187
|
+
|
|
159
188
|
const rememberPreset = (state: ControlState) => {
|
|
160
189
|
const preset = state.ai?.presets.find(p => p.id === state.ai!.selectedId)
|
|
161
190
|
if (state.activeSession && preset && sameEngine(state.activeSession, preset))
|
|
@@ -438,6 +467,7 @@ export class ControlStore {
|
|
|
438
467
|
state.activeSession = null
|
|
439
468
|
state.ai = undefined
|
|
440
469
|
state.sessions = undefined
|
|
470
|
+
state.setupHealth = undefined
|
|
441
471
|
await this.writeState(state)
|
|
442
472
|
return hadOwner
|
|
443
473
|
})
|
|
@@ -450,6 +480,63 @@ export class ControlStore {
|
|
|
450
480
|
})
|
|
451
481
|
}
|
|
452
482
|
|
|
483
|
+
async scheduledSetupHealth(owner: Owner): Promise<SetupHealth> {
|
|
484
|
+
return this.withLock(async () => {
|
|
485
|
+
const state = await this.readState()
|
|
486
|
+
if (!sameOwner(owner, state.owner)) throw new Error('Scheduled setup authority revoked')
|
|
487
|
+
return structuredClone(state.setupHealth ?? {})
|
|
488
|
+
})
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// Native account outcomes belong to existing owner control state. They do not
|
|
492
|
+
// change chat defaults or task definitions. Unknown resets use bounded checks.
|
|
493
|
+
async recordSetupOutcome(owner: Owner, attempt: SetupAttempt): Promise<void> {
|
|
494
|
+
if (attempt.outcome !== 'completed' && !(quotaCategory(attempt.category) || attempt.category === 'login-unavailable')) return
|
|
495
|
+
if (!isPreset(attempt.preset)) throw new Error('Invalid scheduled setup')
|
|
496
|
+
await this.withLock(async () => {
|
|
497
|
+
const state = await this.readState()
|
|
498
|
+
if (!sameOwner(owner, state.owner)) throw new Error('Scheduled setup authority revoked')
|
|
499
|
+
state.setupHealth ??= {}
|
|
500
|
+
const scope = quotaScope(attempt.preset), health = state.setupHealth[scope] ??= {}, now = this.clock()
|
|
501
|
+
if (attempt.outcome === 'completed') {
|
|
502
|
+
health.lastSuccessAt = new Date(now).toISOString()
|
|
503
|
+
// An older concurrent run's success cannot erase a future quota reset.
|
|
504
|
+
if (!(quotaBlockedUntil(health) > now)) {
|
|
505
|
+
delete health.resetAt; delete health.retryAt; delete health.quotaFailures
|
|
506
|
+
}
|
|
507
|
+
} else {
|
|
508
|
+
delete health.lastSuccessAt
|
|
509
|
+
if (attempt.category !== 'login-unavailable') {
|
|
510
|
+
if (attempt.category?.startsWith('credits-required')) health.manualRetry = true
|
|
511
|
+
else if (Date.parse(attempt.resetAt ?? '') > now) {
|
|
512
|
+
if (Date.parse(attempt.resetAt!) > (Date.parse(health.resetAt ?? '') || 0)) health.resetAt = new Date(attempt.resetAt!).toISOString()
|
|
513
|
+
} else if (!(quotaBlockedUntil(health) > now)) {
|
|
514
|
+
// A reset may pass during shutdown: probe once promptly, then back off.
|
|
515
|
+
// Concurrent rejections share one check rather than accelerating it.
|
|
516
|
+
const elapsedReset = Number.isFinite(Date.parse(attempt.resetAt ?? '')) && health.quotaFailures === undefined
|
|
517
|
+
health.quotaFailures = elapsedReset ? 0 : Math.min(4, (health.quotaFailures ?? 0) + 1)
|
|
518
|
+
const delay = elapsedReset ? 1000 : [5, 15, 30, 60][health.quotaFailures - 1] * 60_000
|
|
519
|
+
health.retryAt = new Date(now + delay).toISOString()
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
if (!Object.keys(health).length) delete state.setupHealth[scope]
|
|
524
|
+
await this.writeState(state)
|
|
525
|
+
})
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
async clearQuotaWait(owner: Owner, scopes: string[]): Promise<void> {
|
|
529
|
+
await this.withLock(async () => {
|
|
530
|
+
const state = await this.readState()
|
|
531
|
+
if (!sameOwner(owner, state.owner)) throw new Error('Scheduled setup authority revoked')
|
|
532
|
+
for (const scope of scopes) {
|
|
533
|
+
const health = state.setupHealth?.[scope]
|
|
534
|
+
if (health) { delete health.resetAt; delete health.retryAt; delete health.quotaFailures; delete health.manualRetry }
|
|
535
|
+
}
|
|
536
|
+
await this.writeState(state)
|
|
537
|
+
})
|
|
538
|
+
}
|
|
539
|
+
|
|
453
540
|
async ensureActiveSession(): Promise<SessionState> {
|
|
454
541
|
return this.withLock(async () => {
|
|
455
542
|
const state = await this.readState()
|
|
@@ -635,7 +722,7 @@ export class ControlStore {
|
|
|
635
722
|
presetProvider(session.preset) !== presetProvider(settledPreset)) &&
|
|
636
723
|
(!sameEngine(session, settledPreset) || JSON.stringify(session.preset) === JSON.stringify(settledPreset)) &&
|
|
637
724
|
Boolean(session.title || session.hasStarted || !title?.trim()))
|
|
638
|
-
return { sessionId: session.sessionId, preset: settledPreset }
|
|
725
|
+
return withChain({ sessionId: session.sessionId, preset: settledPreset }, settledAi?.fallbacks)
|
|
639
726
|
return this.withLock(async () => {
|
|
640
727
|
const state = await this.readState()
|
|
641
728
|
state.ai ??= { presets: [persistedPreset(initial)], defaultId: initial.id, selectedId: initial.id, recentIds: [] }
|
|
@@ -657,7 +744,7 @@ export class ControlStore {
|
|
|
657
744
|
if (!state.activeSession.title && !state.activeSession.hasStarted && title?.trim())
|
|
658
745
|
state.activeSession.title = title.replace(/\s+/g, ' ').trim().slice(0, 80)
|
|
659
746
|
await this.writeState(state)
|
|
660
|
-
return { sessionId: state.activeSession.sessionId, preset }
|
|
747
|
+
return withChain({ sessionId: state.activeSession.sessionId, preset }, state.ai.fallbacks)
|
|
661
748
|
})
|
|
662
749
|
}
|
|
663
750
|
|
|
@@ -686,14 +773,14 @@ export class ControlStore {
|
|
|
686
773
|
if (requested) previous.preset = requested
|
|
687
774
|
activate(previous)
|
|
688
775
|
if (shareTelegram || requested) await this.writeState(state)
|
|
689
|
-
return { sessionId: previous.sessionId, preset: previous.preset }
|
|
776
|
+
return withChain({ sessionId: previous.sessionId, preset: previous.preset }, state.ai.fallbacks)
|
|
690
777
|
}
|
|
691
778
|
const preset = requested ?? state.ai.presets.find(item => item.id === state.ai!.selectedId)!
|
|
692
779
|
const session: SessionState = { sessionId: crypto.randomUUID(), hasStarted: false, ...engineOf(preset), preset, applicationScope: scope }
|
|
693
780
|
state.sessions.push(session)
|
|
694
781
|
activate(session)
|
|
695
782
|
await this.writeState(state)
|
|
696
|
-
return { sessionId: session.sessionId, preset }
|
|
783
|
+
return withChain({ sessionId: session.sessionId, preset }, state.ai.fallbacks)
|
|
697
784
|
})
|
|
698
785
|
}
|
|
699
786
|
|
|
@@ -755,6 +842,84 @@ export class ControlStore {
|
|
|
755
842
|
})
|
|
756
843
|
}
|
|
757
844
|
|
|
845
|
+
// The native session an engine resumes for this conversation: the
|
|
846
|
+
// conversation's own for its engine, otherwise that login's linked session,
|
|
847
|
+
// created on first use. A session never resumes under another credential home.
|
|
848
|
+
async engineSession(sessionId: string, engine: Engine): Promise<{ sessionId: string; nativeSessionId?: string; hasStarted: boolean }> {
|
|
849
|
+
return this.withLock(async () => {
|
|
850
|
+
const state = await this.readState()
|
|
851
|
+
const session = this.conversation(state, sessionId)
|
|
852
|
+
if (sameEngine(session, engine)) return session
|
|
853
|
+
let linked = session.linked?.find(item => sameEngine(item, engine))
|
|
854
|
+
if (!linked) {
|
|
855
|
+
linked = { cli: engine.cli, ...(engine.authProfile ? { authProfile: engine.authProfile } : {}), sessionId: crypto.randomUUID(), hasStarted: false }
|
|
856
|
+
session.linked = [...session.linked ?? [], linked].slice(-16)
|
|
857
|
+
await this.writeState(state)
|
|
858
|
+
}
|
|
859
|
+
return linked
|
|
860
|
+
})
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
async saveEngineSession(sessionId: string, engine: Engine, nativeSessionId: string): Promise<void> {
|
|
864
|
+
if (!/^[a-zA-Z0-9_-]{1,160}$/.test(nativeSessionId)) throw new Error('Invalid native session ID')
|
|
865
|
+
await this.withLock(async () => {
|
|
866
|
+
const state = await this.readState()
|
|
867
|
+
const session = this.conversation(state, sessionId)
|
|
868
|
+
const target = sameEngine(session, engine) ? session : session.linked?.find(item => sameEngine(item, engine))
|
|
869
|
+
if (!target) throw new Error('Unknown conversation engine')
|
|
870
|
+
if (target.nativeSessionId && target.nativeSessionId !== nativeSessionId) throw new Error('Native session changed unexpectedly')
|
|
871
|
+
target.nativeSessionId = nativeSessionId
|
|
872
|
+
target.hasStarted = true
|
|
873
|
+
await this.writeState(state)
|
|
874
|
+
})
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
// After an ordered attempt: a session that began work resumes next time. A
|
|
878
|
+
// client that rejected an unstarted session may already have claimed its ID,
|
|
879
|
+
// so a client using Ez-assigned IDs gets a fresh one for its next attempt.
|
|
880
|
+
async settleEngineSession(sessionId: string, engine: Engine, started: boolean): Promise<void> {
|
|
881
|
+
await this.withLock(async () => {
|
|
882
|
+
const state = await this.readState()
|
|
883
|
+
const session = state.activeSession?.sessionId === sessionId ? state.activeSession : state.sessions?.find(s => s.sessionId === sessionId)
|
|
884
|
+
const target = !session ? undefined : sameEngine(session, engine) ? session : session.linked?.find(item => sameEngine(item, engine))
|
|
885
|
+
// Reporting clients start only through saveEngineSession.
|
|
886
|
+
if (!target || target.hasStarted || reportsNativeSession(target.cli)) return
|
|
887
|
+
if (started) target.hasStarted = true
|
|
888
|
+
else if (target === session) session.nativeSessionId = crypto.randomUUID()
|
|
889
|
+
else target.sessionId = crypto.randomUUID()
|
|
890
|
+
await this.writeState(state)
|
|
891
|
+
})
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
private conversation(state: ControlState, sessionId: string): SessionState {
|
|
895
|
+
const session = state.activeSession?.sessionId === sessionId ? state.activeSession : state.sessions?.find(s => s.sessionId === sessionId)
|
|
896
|
+
if (!session?.cli) throw new Error('Unknown conversation')
|
|
897
|
+
return session
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Applies to work queued afterwards; queued work keeps its captured chain.
|
|
901
|
+
async setChatFallbacks(chain: AiPreset[] | undefined): Promise<void> {
|
|
902
|
+
if (chain !== undefined && (!chain.length || chain.length > 12 || !chain.every(isPreset) ||
|
|
903
|
+
new Set(chain.map(p => p.id)).size !== chain.length)) throw new Error('Invalid fallback chain')
|
|
904
|
+
for (const preset of chain ?? []) assertEffort(preset.effort, preset.model, preset.cli)
|
|
905
|
+
await this.withLock(async () => {
|
|
906
|
+
const state = await this.readState()
|
|
907
|
+
if (!state.ai) throw new Error('AI settings not initialized')
|
|
908
|
+
if (chain) state.ai.fallbacks = chain.map(persistedPreset)
|
|
909
|
+
else delete state.ai.fallbacks
|
|
910
|
+
await this.writeState(state)
|
|
911
|
+
})
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
async setRunTelemetry(enabled: boolean): Promise<void> {
|
|
915
|
+
await this.withLock(async () => {
|
|
916
|
+
const state = await this.readState()
|
|
917
|
+
if (enabled) state.runTelemetry = true
|
|
918
|
+
else delete state.runTelemetry
|
|
919
|
+
await this.writeState(state)
|
|
920
|
+
})
|
|
921
|
+
}
|
|
922
|
+
|
|
758
923
|
async savePreset(preset: AiPreset, guard?: ControlGuard): Promise<void> {
|
|
759
924
|
if (!isPreset(preset)) throw new Error('Invalid AI preset')
|
|
760
925
|
assertEffort(preset.effort, preset.model, preset.cli)
|
package/src/delivery-socket.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { telegramOwner } from './control-state.js'
|
|
|
13
13
|
import { Scheduler } from './scheduler.js'
|
|
14
14
|
import {Tasks} from './tasks.js'
|
|
15
15
|
import { runEvidence, type EvidenceOptions } from './run-evidence.js'
|
|
16
|
+
import { runningWork } from './running-work.js'
|
|
16
17
|
|
|
17
18
|
// Stateless pipe transport. The relay owns the runs/outbox ledger in process
|
|
18
19
|
// memory; engine children, plugin children and the host executor reach it
|
|
@@ -34,6 +35,7 @@ export type DeliverySocketOp =
|
|
|
34
35
|
| { op: 'status' }
|
|
35
36
|
| { op: 'schedulePreflight'; payload?: Record<string, unknown> }
|
|
36
37
|
| { op: 'triggerSchedule'; payload: { scheduleId: string; revision: string; key: string; callerRunId?: string } }
|
|
38
|
+
| { op: 'retryScheduleQuota'; payload: { runId?: string; callerRunId?: string } }
|
|
37
39
|
| { op: 'enqueue'; payload: Record<string, unknown> }
|
|
38
40
|
| { op: 'wait'; payload: { id: string; timeoutMs?: number } }
|
|
39
41
|
| { op: 'get'; payload: { runId: string } }
|
|
@@ -45,6 +47,7 @@ export type DeliverySocketOp =
|
|
|
45
47
|
| { op: 'history'; payload: { runId: string; limit?: number; messageId?: number } }
|
|
46
48
|
| { op: 'deliveryStatus'; payload?: Record<string, never> }
|
|
47
49
|
| { op: 'evidence'; payload: EvidenceOptions }
|
|
50
|
+
| { op: 'runningWork'; payload?: { callerRunId?: string } }
|
|
48
51
|
| { op: 'approvalCheck'; payload: { actionId: string } }
|
|
49
52
|
| { op: 'latestReply'; payload?: Record<string, never> }
|
|
50
53
|
|
|
@@ -255,6 +258,19 @@ export const createLedgerHandler = (controlDir: string, hooks: LedgerHooks): Del
|
|
|
255
258
|
hooks.wake()
|
|
256
259
|
return run
|
|
257
260
|
}
|
|
261
|
+
case 'retryScheduleQuota': {
|
|
262
|
+
const { runId, callerRunId } = op.payload ?? {}
|
|
263
|
+
if (runId !== undefined && !nonEmptyString(runId)) throw new Error('Run ID is invalid')
|
|
264
|
+
if (callerRunId !== undefined) {
|
|
265
|
+
const caller = await requireOwnerExecution(controlDir, callerRunId)
|
|
266
|
+
if (caller.replyOnly || caller.script) throw new Error('Quota retry requires an active owner-authorized run')
|
|
267
|
+
}
|
|
268
|
+
const owner = await readOnlyOwner(controlDir)
|
|
269
|
+
if (!owner) throw new Error('Pair an owner before retrying')
|
|
270
|
+
const result = await new Scheduler(controlDir).retryQuota(owner, runs, runId)
|
|
271
|
+
hooks.wake()
|
|
272
|
+
return result
|
|
273
|
+
}
|
|
258
274
|
case 'enqueue': {
|
|
259
275
|
const envelope = (op.payload ?? {}) as Record<string, unknown>
|
|
260
276
|
if (envelope.kind === 'owner') {
|
|
@@ -333,7 +349,7 @@ export const createLedgerHandler = (controlDir: string, hooks: LedgerHooks): Del
|
|
|
333
349
|
if (typeof payload.runId !== 'string' || !payload.runId) throw new Error('Run ID is required')
|
|
334
350
|
if (!payload.change || typeof payload.change !== 'object') throw new Error('Patch change is required')
|
|
335
351
|
// Setup attempts select what the host admits; only the relay records them.
|
|
336
|
-
if ('attempts' in payload.change) throw new Error('Setup attempts are relay-owned')
|
|
352
|
+
if (['attempts', 'attemptRound', 'quotaWait', 'telemetry'].some(key => key in (payload.change as Record<string, unknown>))) throw new Error('Setup attempts, telemetry and quota admission are relay-owned')
|
|
337
353
|
return runs.patch(payload.runId, payload.change as never)
|
|
338
354
|
}
|
|
339
355
|
case 'taskApplication': {
|
|
@@ -360,6 +376,7 @@ export const createLedgerHandler = (controlDir: string, hooks: LedgerHooks): Del
|
|
|
360
376
|
})
|
|
361
377
|
}
|
|
362
378
|
case 'evidence': return runEvidence(controlDir, op.payload ?? {})
|
|
379
|
+
case 'runningWork': return runningWork(controlDir, op.payload ?? {})
|
|
363
380
|
case 'deliveryStatus': return runs.deliveryStatus()
|
|
364
381
|
case 'approvalCheck': {
|
|
365
382
|
const actionId = (op.payload as { actionId?: unknown } | undefined)?.actionId
|
package/src/executor.ts
CHANGED
|
@@ -36,6 +36,10 @@ export type ExecutorOptions = {
|
|
|
36
36
|
// Caller-owned routing flags. The core only spawns and streams.
|
|
37
37
|
taskRun?: boolean
|
|
38
38
|
nativeSession?: boolean
|
|
39
|
+
// Ordered setups and telemetry read structured client events; stdout is never chat.
|
|
40
|
+
structuredOutput?: boolean
|
|
41
|
+
// Owner-enabled run telemetry; Codex native sessions forward usage summaries.
|
|
42
|
+
telemetry?: boolean
|
|
39
43
|
toolsHome?: string
|
|
40
44
|
sharedWorkspace?: string
|
|
41
45
|
additionalWorkspaces?: string[]
|
|
@@ -131,7 +135,7 @@ export type CliAdapter = {
|
|
|
131
135
|
command: string
|
|
132
136
|
description: string
|
|
133
137
|
buildArgs: (
|
|
134
|
-
options: Pick<ExecutorOptions, 'workspace' | 'sessionId' | 'isResume' | 'model' | 'effort' | 'toolsHome' | 'sharedWorkspace' | 'additionalWorkspaces' | 'codexAutoCompactTokens' | 'codexSandbox' | 'codexProvider' | 'nativeSession'> & { controlDir?: string },
|
|
138
|
+
options: Pick<ExecutorOptions, 'workspace' | 'sessionId' | 'isResume' | 'model' | 'effort' | 'toolsHome' | 'sharedWorkspace' | 'additionalWorkspaces' | 'codexAutoCompactTokens' | 'codexSandbox' | 'codexProvider' | 'nativeSession' | 'structuredOutput'> & { controlDir?: string },
|
|
135
139
|
promptFile: string,
|
|
136
140
|
promptText: string,
|
|
137
141
|
) => string[]
|
|
@@ -178,9 +182,9 @@ export const EXECUTOR_REGISTRY: Record<string, CliAdapter> = {
|
|
|
178
182
|
args.push('--session-id', opts.sessionId)
|
|
179
183
|
}
|
|
180
184
|
args.push('--print', '--dangerously-skip-permissions')
|
|
181
|
-
// Scheduled runs stream structured events so core
|
|
182
|
-
// rejection before work from a failure after work began.
|
|
183
|
-
if (opts.nativeSession) args.push('--output-format', 'stream-json', '--verbose')
|
|
185
|
+
// Scheduled runs and ordered chat setups stream structured events so core
|
|
186
|
+
// can tell a provider rejection before work from a failure after work began.
|
|
187
|
+
if (opts.nativeSession || opts.structuredOutput) args.push('--output-format', 'stream-json', '--verbose')
|
|
184
188
|
for (const directory of [opts.controlDir, opts.sharedWorkspace, ...(opts.additionalWorkspaces ?? []), opts.toolsHome])
|
|
185
189
|
if (directory) args.push('--add-dir', directory)
|
|
186
190
|
if (opts.model) args.push('--model', opts.model)
|
|
@@ -448,6 +452,9 @@ export const startExecutorJob = async (
|
|
|
448
452
|
// token (`claude setup-token`) never refreshes, so one file can be copied
|
|
449
453
|
// to every agent without them invalidating each other's OAuth session.
|
|
450
454
|
Object.assign(environment, await claudeAuthEnvironment(home, options.authProfile))
|
|
455
|
+
// Print mode otherwise kills unfinished background work after 10 idle minutes.
|
|
456
|
+
// Keep native background work alive until completion or cancellation.
|
|
457
|
+
environment.CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS = '0'
|
|
451
458
|
}
|
|
452
459
|
if (!host && !gui && key === 'opencode') {
|
|
453
460
|
// Point OpenCode at the agent-bound data home (cli/opencode/auth.json)
|
package/src/host-executor.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { fileURLToPath } from 'node:url'
|
|
|
10
10
|
import { CliUnavailableError, startExecutorJob, terminateJob, resolveExecutor, validateCodexProvider, type CodexProviderBinding, type ExecutorOptions } from './executor.js'
|
|
11
11
|
import { activeSetup } from './runs.js'
|
|
12
12
|
import { startupLine, startupObserver } from './setup-fallback.js'
|
|
13
|
+
import { telemetryObserver } from './run-telemetry.js'
|
|
13
14
|
import { parseIsolationClass, type IsolationClass } from './isolation.js'
|
|
14
15
|
import { readModels, sameEngine, validateSelection, validateOpencodeProviders, type ModelChoice } from './ai.js'
|
|
15
16
|
import type { ChildProcess } from 'node:child_process'
|
|
@@ -236,16 +237,22 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
|
|
|
236
237
|
job=await launch(request.texts,options)
|
|
237
238
|
} else {
|
|
238
239
|
const cli = opts.cli || installation.cli
|
|
240
|
+
// Ordered runs (scheduled, captured chat chain or telemetry) record
|
|
241
|
+
// attempts; only the relay's active attempt is admitted.
|
|
242
|
+
const ordered = Boolean(run?.scheduled || run?.attempts?.length)
|
|
239
243
|
if (run?.scheduled) {
|
|
240
244
|
const setup = activeSetup(run)
|
|
241
245
|
assertScheduledModel(setup?.model)
|
|
242
246
|
if (opts.model !== setup!.model || cli !== setup!.cli) throw new Error('Scheduled run model does not match its saved task model')
|
|
243
|
-
|
|
247
|
+
} else if (ordered) {
|
|
248
|
+
const setup = activeSetup(run!)
|
|
249
|
+
if (!setup || opts.model !== setup.model || cli !== setup.cli || opts.effort !== setup.effort) throw new Error('Run setup does not match its captured AI selection')
|
|
244
250
|
}
|
|
251
|
+
observeClaude = ordered && cli === 'claude'
|
|
245
252
|
resolveExecutor(cli)
|
|
246
253
|
// The credential home comes from the authorized run's captured
|
|
247
254
|
// choice; a request can never move it to another account.
|
|
248
|
-
const captured =
|
|
255
|
+
const captured = ordered ? activeSetup(run!) : run?.execution?.preset
|
|
249
256
|
if ((opts.authProfile !== undefined || (captured?.cli === cli && captured.authProfile !== undefined)) &&
|
|
250
257
|
!(captured && sameEngine(captured, {cli, authProfile: opts.authProfile})))
|
|
251
258
|
throw new Error('Run auth profile does not match its captured AI selection')
|
|
@@ -265,7 +272,8 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
|
|
|
265
272
|
const options:ExecutorOptions={catalogTransport:containerId?'local':'host',workspace:agent.workspace,controlDir:agent.controlDir,binDir:agent.binDir,toolsHome:agent.toolsHome,sharedWorkspace,additionalWorkspaces:additionalWorkspaces.get(agent),cli,
|
|
266
273
|
runId:path.basename(base),timeoutMs:0,repairEnabled:opts.repairEnabled,
|
|
267
274
|
sessionId:opts.sessionId,isResume:opts.isResume,model,effort:opts.effort,provider:opts.provider,authProfile:opts.authProfile,codexAutoCompactTokens:opts.codexAutoCompactTokens,codexProvider:provider,
|
|
268
|
-
|
|
275
|
+
codexSandbox:containerId && installation.isolation==='isolated' && cli==='codex' && !run?.taskId && process.env.EZ_CODEX_SANDBOX==='external' ? 'external' : undefined,
|
|
276
|
+
promptSuffix:runPromptSuffix(run),taskRun:Boolean(run?.taskId),nativeSession:Boolean(run?.scheduled),structuredOutput:ordered,telemetry:Boolean(run?.telemetry)}
|
|
269
277
|
job=await launch(request.texts,options)
|
|
270
278
|
}
|
|
271
279
|
active.set(base,job.child)
|
|
@@ -275,13 +283,16 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
|
|
|
275
283
|
? Promise.resolve(job.child.exitCode??1)
|
|
276
284
|
: new Promise<number>(resolve=>job!.child.once('close',code=>resolve(code??1)))
|
|
277
285
|
if(signal.aborted)terminateJob(job.child)
|
|
278
|
-
//
|
|
286
|
+
// Ordered Claude streams every event; forward only the startup and,
|
|
287
|
+
// when enabled, telemetry summaries (counts and categories only).
|
|
279
288
|
const startup = observeClaude ? startupObserver() : undefined
|
|
289
|
+
const telemetry = startup && run?.telemetry ? telemetryObserver() : undefined
|
|
280
290
|
let summary = ''
|
|
281
291
|
job.child.stdout?.setEncoding('utf8').on('data',(chunk:string)=>{
|
|
282
292
|
if (!startup) return emit({stream:'stdout',text:chunk})
|
|
283
|
-
startup.write(chunk)
|
|
284
|
-
|
|
293
|
+
startup.write(chunk); telemetry?.write(chunk)
|
|
294
|
+
const next = [startup.summary(), ...(telemetry ? [telemetry.summary()] : [])].join('\n')
|
|
295
|
+
if (next !== summary) { summary = next; emit({stream:'stdout',text:summary+'\n'}) }
|
|
285
296
|
})
|
|
286
297
|
job.child.stderr?.on('data',chunk=>emit({stream:'stderr',text:chunk.toString()}))
|
|
287
298
|
await writeFile(base+'.process.json',JSON.stringify({pid:job.child.pid,started:job.child.pid ? await processStart(job.child.pid) : undefined,hostPid:process.pid,hostStarted,...(containerId ? {containerId} : {})}),{mode:0o600})
|