@jc_stack/ez-agents 0.1.0-beta.52 → 0.1.0-beta.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -1
- package/docs/architecture/ai-selection.md +6 -2
- package/docs/docker-runtime.md +48 -0
- package/docs/scheduling.md +124 -16
- package/docs/upgrades.md +9 -4
- package/package.json +1 -1
- package/src/ai-cli.ts +31 -5
- package/src/ai.ts +17 -1
- package/src/codex-session.ts +23 -2
- package/src/control-state.ts +173 -8
- package/src/delivery-socket.ts +18 -1
- package/src/executor.ts +11 -4
- package/src/host-executor.ts +17 -6
- package/src/index.ts +358 -269
- package/src/run-telemetry.ts +163 -0
- package/src/running-work.ts +123 -0
- package/src/runs.ts +7 -2
- package/src/schedule-cli.ts +55 -16
- package/src/scheduler.ts +126 -29
- package/src/setup-fallback.ts +57 -18
- package/src/updates/supervisor.mjs +23 -8
- package/test/chat-fallback.test.ts +358 -0
- package/test/codex-session.test.ts +1 -1
- package/test/control-state.test.ts +62 -0
- package/test/executor.test.ts +2 -1
- package/test/host-executor.test.ts +41 -0
- package/test/run-handoff.test.ts +54 -0
- package/test/running-work.test.ts +175 -0
- package/test/schedule-cli.test.ts +55 -6
- package/test/scheduler.test.ts +128 -1
- package/test/script-schedules.test.ts +3 -3
- package/test/setup-fallback.test.ts +211 -14
- package/test/updates.test.mjs +25 -4
- package/test/upgrade-pause.test.ts +65 -2
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Opt-in per-attempt run telemetry read from structured native client events:
|
|
2
|
+
// reported model, token counts, tool counts/durations and error categories.
|
|
3
|
+
// Only counts, tool names and category codes are kept. Prompts, model text,
|
|
4
|
+
// tool arguments/results, credentials and reasoning are never read into it,
|
|
5
|
+
// and no cost is derived. Missing or partial coverage is labelled, not guessed.
|
|
6
|
+
export type TokenCounts = { input: number; output: number; cacheRead: number; cacheWrite: number }
|
|
7
|
+
export type ToolStats = { calls: number; completed: number; errors: number; durationMs: number }
|
|
8
|
+
export type AttemptTelemetry = {
|
|
9
|
+
coverage: 'complete' | 'partial' | 'none'
|
|
10
|
+
gaps?: string[]
|
|
11
|
+
model?: string
|
|
12
|
+
// native-total: the client's own cumulative total for this invocation;
|
|
13
|
+
// message-sum: deduplicated per-message usage when no total was reported.
|
|
14
|
+
tokens?: TokenCounts & { source: 'native-total' | 'message-sum' }
|
|
15
|
+
// Durations are observed stream intervals between a tool's start and result.
|
|
16
|
+
tools?: ToolStats & { byName: Record<string, ToolStats> }
|
|
17
|
+
errors?: string[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const MAX_NAMES = 32
|
|
21
|
+
const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0
|
|
22
|
+
const code = (value: unknown): string | undefined => typeof value === 'string' && /^[A-Za-z0-9_.:@/-]{1,80}$/.test(value) ? value : undefined
|
|
23
|
+
const toolName = (value: unknown): string => code(value) ?? 'unnamed'
|
|
24
|
+
const isCounts = (v: any): boolean => v && ['input', 'output', 'cacheRead', 'cacheWrite'].every(key => Number.isSafeInteger(v[key]) && v[key] >= 0)
|
|
25
|
+
const isStats = (v: any): boolean => v && ['calls', 'completed', 'errors', 'durationMs'].every(key => Number.isSafeInteger(v[key]) && v[key] >= 0)
|
|
26
|
+
|
|
27
|
+
// Revalidates a forwarded summary; anything outside the closed shape is dropped.
|
|
28
|
+
export const validTelemetry = (value: unknown): AttemptTelemetry | undefined => {
|
|
29
|
+
const v = value as AttemptTelemetry
|
|
30
|
+
if (!v || typeof v !== 'object' || !['complete', 'partial', 'none'].includes(v.coverage)) return undefined
|
|
31
|
+
if (v.gaps !== undefined && !(Array.isArray(v.gaps) && v.gaps.length <= 16 && v.gaps.every(g => code(g)))) return undefined
|
|
32
|
+
if (v.errors !== undefined && !(Array.isArray(v.errors) && v.errors.length <= 16 && v.errors.every(e => code(e)))) return undefined
|
|
33
|
+
if (v.model !== undefined && !code(v.model)) return undefined
|
|
34
|
+
if (v.tokens !== undefined && !(isCounts(v.tokens) && ['native-total', 'message-sum'].includes(v.tokens.source))) return undefined
|
|
35
|
+
if (v.tools !== undefined && !(isStats(v.tools) && v.tools.byName && typeof v.tools.byName === 'object' &&
|
|
36
|
+
Object.keys(v.tools.byName).length <= MAX_NAMES + 1 && Object.entries(v.tools.byName).every(([name, stats]) => code(name) && isStats(stats)))) return undefined
|
|
37
|
+
const { coverage, gaps, model, tokens, tools, errors } = v
|
|
38
|
+
return JSON.parse(JSON.stringify({ coverage, gaps, model, tokens, tools, errors }))
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export const telemetryLine = (telemetry: AttemptTelemetry): string => JSON.stringify({ type: 'ez.telemetry', telemetry })
|
|
42
|
+
|
|
43
|
+
// Codex reports input including cached input; Claude reports them separately.
|
|
44
|
+
// Both are normalized to uncached input plus cache reads/writes.
|
|
45
|
+
const codexCounts = (usage: any): TokenCounts => {
|
|
46
|
+
const cached = count(usage?.cached_input_tokens)
|
|
47
|
+
return { input: Math.max(0, count(usage?.input_tokens) - cached), output: count(usage?.output_tokens), cacheRead: cached, cacheWrite: 0 }
|
|
48
|
+
}
|
|
49
|
+
const claudeCounts = (usage: any): TokenCounts => ({ input: count(usage?.input_tokens), output: count(usage?.output_tokens),
|
|
50
|
+
cacheRead: count(usage?.cache_read_input_tokens), cacheWrite: count(usage?.cache_creation_input_tokens) })
|
|
51
|
+
const codexTools: Record<string, string> = { command_execution: 'command', file_change: 'file_change', web_search: 'web_search', mcp_tool_call: 'mcp' }
|
|
52
|
+
|
|
53
|
+
// Repeated and cumulative events are deduplicated: Claude repeats one
|
|
54
|
+
// message's usage per content block (keyed by message ID), native totals
|
|
55
|
+
// replace earlier totals, tools are keyed by their call ID, and a forwarded
|
|
56
|
+
// `ez.telemetry` summary from the producing side replaces local observation.
|
|
57
|
+
// `cumulativeSession` marks a resumed client whose total may span earlier turns.
|
|
58
|
+
export const telemetryObserver = (options: { cumulativeSession?: boolean } = {}, now = () => Date.now()) => {
|
|
59
|
+
let buffer = '', observed = false, model: string | undefined, total: TokenCounts | undefined
|
|
60
|
+
let forwarded: AttemptTelemetry | undefined
|
|
61
|
+
const messages = new Map<string, TokenCounts>()
|
|
62
|
+
const tools = new Map<string, { name: string; startedAt?: number; endedAt?: number; error: boolean }>()
|
|
63
|
+
const errors = new Set<string>()
|
|
64
|
+
const start = (id: unknown, name: string) => {
|
|
65
|
+
const key = code(id)
|
|
66
|
+
if (key && !tools.has(key)) tools.set(key, { name, startedAt: now(), error: false })
|
|
67
|
+
}
|
|
68
|
+
const end = (id: unknown, name: string, error: boolean) => {
|
|
69
|
+
const key = code(id)
|
|
70
|
+
if (!key) return
|
|
71
|
+
const tool = tools.get(key) ?? { name, error: false }
|
|
72
|
+
if (tool.endedAt !== undefined) return
|
|
73
|
+
tools.set(key, { ...tool, endedAt: now(), error })
|
|
74
|
+
}
|
|
75
|
+
const line = (text: string) => {
|
|
76
|
+
if (!text.trim()) return
|
|
77
|
+
let event: any
|
|
78
|
+
try { event = JSON.parse(text) } catch { return }
|
|
79
|
+
if (!event || typeof event !== 'object') return
|
|
80
|
+
if (event.type === 'ez.telemetry') { const summary = validTelemetry(event.telemetry); if (summary) forwarded = summary; return }
|
|
81
|
+
if (event.type === 'ez.startup') return
|
|
82
|
+
observed = true
|
|
83
|
+
// Claude Code stream-json.
|
|
84
|
+
if (event.type === 'system' && event.subtype === 'init') model = code(event.model) ?? model
|
|
85
|
+
else if (event.type === 'system' && ['api_error', 'api_retry'].includes(event.subtype)) errors.add(event.subtype === 'api_retry' ? 'api-retry' : 'api-error')
|
|
86
|
+
else if (event.type === 'rate_limit_event' && event.rate_limit_info?.status === 'rejected') errors.add('quota')
|
|
87
|
+
else if (event.type === 'assistant') {
|
|
88
|
+
const error = code(event.error)
|
|
89
|
+
if (error) errors.add(error)
|
|
90
|
+
const id = code(event.message?.id)
|
|
91
|
+
if (id && event.message?.usage && !event.isApiErrorMessage) messages.set(id, claudeCounts(event.message.usage))
|
|
92
|
+
for (const block of Array.isArray(event.message?.content) ? event.message.content : [])
|
|
93
|
+
if (block?.type === 'tool_use') start(block.id, toolName(block.name))
|
|
94
|
+
} else if (event.type === 'user') {
|
|
95
|
+
for (const block of Array.isArray(event.message?.content) ? event.message.content : [])
|
|
96
|
+
if (block?.type === 'tool_result') end(block.tool_use_id, tools.get(code(block.tool_use_id) ?? '')?.name ?? 'unnamed', block.is_error === true)
|
|
97
|
+
} else if (event.type === 'result') {
|
|
98
|
+
if (event.usage) total = claudeCounts(event.usage)
|
|
99
|
+
if (event.is_error === true) errors.add(code(event.subtype) ?? 'result-error')
|
|
100
|
+
}
|
|
101
|
+
// Codex exec --json (also the shape codex-session.ts forwards for app-server items).
|
|
102
|
+
else if ((event.type === 'item.started' || event.type === 'item.completed') && codexTools[event.item?.type]) {
|
|
103
|
+
const name = event.item.type === 'mcp_tool_call' ? toolName(`mcp:${event.item.server}/${event.item.tool}`) : codexTools[event.item.type]
|
|
104
|
+
if (event.type === 'item.started') start(event.item.id, name)
|
|
105
|
+
else end(event.item.id, name, event.item.status === 'failed')
|
|
106
|
+
} else if (event.type === 'turn.completed' && event.usage) total = codexCounts(event.usage)
|
|
107
|
+
else if (event.type === 'turn.failed') errors.add('turn-failed')
|
|
108
|
+
else if (event.type === 'error') errors.add('native-error')
|
|
109
|
+
}
|
|
110
|
+
const result = (): AttemptTelemetry => {
|
|
111
|
+
if (forwarded) return forwarded
|
|
112
|
+
const gaps: string[] = []
|
|
113
|
+
let tokens: AttemptTelemetry['tokens']
|
|
114
|
+
if (total) tokens = { ...total, source: 'native-total' }
|
|
115
|
+
else if (messages.size) {
|
|
116
|
+
const sum = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
|
|
117
|
+
for (const usage of messages.values()) for (const key of Object.keys(sum) as (keyof TokenCounts)[]) sum[key] += usage[key]
|
|
118
|
+
tokens = { ...sum, source: 'message-sum' }
|
|
119
|
+
gaps.push('tokens-without-native-total')
|
|
120
|
+
} else gaps.push('tokens')
|
|
121
|
+
if (tokens && options.cumulativeSession) gaps.push('tokens-may-include-earlier-session-turns')
|
|
122
|
+
if (!model) gaps.push('reported-model')
|
|
123
|
+
const byName: Record<string, ToolStats> = {}
|
|
124
|
+
const totals: ToolStats = { calls: 0, completed: 0, errors: 0, durationMs: 0 }
|
|
125
|
+
let unmeasured = false
|
|
126
|
+
for (const tool of tools.values()) {
|
|
127
|
+
const name = byName[tool.name] || Object.keys(byName).length < MAX_NAMES ? tool.name : 'other'
|
|
128
|
+
const stats = byName[name] ??= { calls: 0, completed: 0, errors: 0, durationMs: 0 }
|
|
129
|
+
const done = tool.endedAt !== undefined
|
|
130
|
+
const duration = done && tool.startedAt !== undefined ? Math.max(0, tool.endedAt! - tool.startedAt) : 0
|
|
131
|
+
if (done && tool.startedAt === undefined) unmeasured = true
|
|
132
|
+
for (const target of [stats, totals]) {
|
|
133
|
+
target.calls++
|
|
134
|
+
if (done) target.completed++
|
|
135
|
+
if (tool.error) target.errors++
|
|
136
|
+
target.durationMs += duration
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (totals.completed < totals.calls) gaps.push('tools-incomplete')
|
|
140
|
+
if (unmeasured) gaps.push('tool-durations-partial')
|
|
141
|
+
const telemetry: AttemptTelemetry = {
|
|
142
|
+
coverage: !observed ? 'none' : gaps.length ? 'partial' : 'complete',
|
|
143
|
+
...(gaps.length ? { gaps } : {}),
|
|
144
|
+
...(model ? { model } : {}),
|
|
145
|
+
...(tokens ? { tokens } : {}),
|
|
146
|
+
...(observed ? { tools: { ...totals, byName } } : {}),
|
|
147
|
+
...(errors.size ? { errors: [...errors].slice(0, 16) } : {}),
|
|
148
|
+
}
|
|
149
|
+
return telemetry
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
write: (chunk: string) => {
|
|
153
|
+
buffer += chunk
|
|
154
|
+
const lines = buffer.split('\n')
|
|
155
|
+
buffer = lines.pop() ?? ''
|
|
156
|
+
for (const text of lines) line(text)
|
|
157
|
+
},
|
|
158
|
+
// A client that reports its model outside the event stream (Codex app-server).
|
|
159
|
+
model: (name: unknown) => { model = code(name) ?? model },
|
|
160
|
+
result: () => { line(buffer); buffer = ''; return result() },
|
|
161
|
+
summary: () => telemetryLine(result()),
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import type { Owner } from './control-state.js'
|
|
2
|
+
import { sameOwner } from './control-state.js'
|
|
3
|
+
import { readOnlyOwner, requireOwnerExecution } from './execution-authority.js'
|
|
4
|
+
import { redactFailure } from './failure.js'
|
|
5
|
+
import { ownsRun } from './identity.js'
|
|
6
|
+
import { activeSetup, retainedOutbox, RunStore, type RunRecord, type StoredOutboxItem } from './runs.js'
|
|
7
|
+
import { Scheduler, occurrenceOf, type Schedule } from './scheduler.js'
|
|
8
|
+
|
|
9
|
+
const timestamp = (value?: string): string | null =>
|
|
10
|
+
value && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null
|
|
11
|
+
const secondsSince = (value: string | null, now: number): number | null =>
|
|
12
|
+
value && Date.parse(value) <= now ? Math.floor((now - Date.parse(value)) / 1000) : null
|
|
13
|
+
const label = (value: string): string => redactFailure(value).replace(/[\p{Cc}\p{Cf}]/gu, ' ').slice(0, 120)
|
|
14
|
+
|
|
15
|
+
// Current-owner relay records only, without application/channel request context.
|
|
16
|
+
// Nothing here queries native sessions, processes or provider state, or exports prompts.
|
|
17
|
+
const visible = (run: RunRecord, owner: Owner): boolean =>
|
|
18
|
+
ownsRun(owner, run) && Date.parse(run.createdAt) >= Date.parse(owner.pairedAt) &&
|
|
19
|
+
!run.external && !run.taskId &&
|
|
20
|
+
(!run.scheduled || run.scheduled.pairedAt === owner.pairedAt)
|
|
21
|
+
|
|
22
|
+
export function projectRunningWork(records: RunRecord[], outbox: StoredOutboxItem[], schedules: Schedule[], owner: Owner, now = Date.now()) {
|
|
23
|
+
const owned = records.filter(run => visible(run, owner))
|
|
24
|
+
const active = owned.filter(run => run.status === 'running' || run.status === 'queued')
|
|
25
|
+
.sort((a, b) => Number(b.status === 'running') - Number(a.status === 'running') ||
|
|
26
|
+
a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id))
|
|
27
|
+
const runs = active.map(run => {
|
|
28
|
+
const preset = activeSetup(run)
|
|
29
|
+
const attempt = [...run.attempts ?? []].reverse().find(item => item.outcome !== 'skipped')
|
|
30
|
+
const schedule = schedules.find(item => occurrenceOf(item, run) && sameOwner(item.owner, owner))
|
|
31
|
+
const originId = run.scheduled?.originRunId
|
|
32
|
+
const origin = owned.find(item => item.id === originId)
|
|
33
|
+
const events = [
|
|
34
|
+
{ at: timestamp(run.createdAt), kind: 'queued' },
|
|
35
|
+
{ at: timestamp(run.startedAt), kind: 'relay-started' },
|
|
36
|
+
...run.attempts?.flatMap(item => [
|
|
37
|
+
{ at: timestamp(item.startedAt), kind: item.outcome === 'skipped' ? 'setup-skipped' : 'setup-attempt-started' },
|
|
38
|
+
{ at: timestamp(item.endedAt), kind: item.outcome === 'skipped' ? 'setup-skipped' : 'setup-attempt-ended' },
|
|
39
|
+
]) ?? [],
|
|
40
|
+
...outbox.filter(item => item.runId === run.id && item.chatId === owner.telegramChatId &&
|
|
41
|
+
Date.parse(item.createdAt) >= Date.parse(owner.telegramLinkedAt ?? owner.pairedAt))
|
|
42
|
+
.map(item => ({ at: timestamp(item.createdAt), kind: 'delivery-enqueued' })),
|
|
43
|
+
].filter((item): item is { at: string; kind: string } => item.at !== null && Date.parse(item.at) <= now)
|
|
44
|
+
.sort((a, b) => b.at.localeCompare(a.at))
|
|
45
|
+
const latest = events[0]
|
|
46
|
+
return {
|
|
47
|
+
id: run.id, status: run.status,
|
|
48
|
+
...(run.status === 'queued' && run.quotaWait ? { quotaWait: { retryAt: timestamp(run.quotaWait.retryAt) } } : {}),
|
|
49
|
+
kind: run.script ? 'scheduled-script' : run.scheduled ? 'scheduled-agent' : run.sessionClose ? 'session-close' : run.application || run.telegramApplication ? 'application-foreground' : 'foreground',
|
|
50
|
+
schedule: run.scheduled ? { id: run.scheduled.id, revision: run.scheduled.revision,
|
|
51
|
+
name: schedule ? label(schedule.name) : null, dueAt: timestamp(run.scheduled.dueAt) } : null,
|
|
52
|
+
script: run.script ? { id: run.script.id, revision: run.script.revision ?? null } : null,
|
|
53
|
+
execution: run.script ? null : preset ? { cli: preset.cli, model: preset.model ?? null,
|
|
54
|
+
effort: preset.effort ?? null, provider: preset.provider ?? null,
|
|
55
|
+
source: attempt ? 'latest-attempt' : 'saved', attemptOutcome: attempt?.outcome ?? null } : null,
|
|
56
|
+
createdAt: timestamp(run.createdAt), startedAt: timestamp(run.startedAt),
|
|
57
|
+
elapsedSeconds: run.status === 'running' ? secondsSince(timestamp(run.startedAt), now) : null,
|
|
58
|
+
queuedSeconds: run.status === 'queued' ? secondsSince(timestamp(run.createdAt), now) : null,
|
|
59
|
+
lastObservedActivity: latest ? { ...latest, ageSeconds: secondsSince(latest.at, now) } : null,
|
|
60
|
+
// This is request provenance, not proof of native parent/child delegation.
|
|
61
|
+
origin: originId ? { runId: originId, relation: 'schedule-origin',
|
|
62
|
+
status: origin?.status ?? null, retained: Boolean(origin) } : null,
|
|
63
|
+
scheduledChildren: active.filter(item => item.scheduled?.originRunId === run.id)
|
|
64
|
+
.map(item => ({ id: item.id, status: item.status })),
|
|
65
|
+
nativeChildren: { state: 'unavailable' },
|
|
66
|
+
}
|
|
67
|
+
})
|
|
68
|
+
return { schemaVersion: 1, observedAt: new Date(now).toISOString(),
|
|
69
|
+
counts: { running: runs.filter(run => run.status === 'running').length, queued: runs.filter(run => run.status === 'queued').length },
|
|
70
|
+
coverage: {
|
|
71
|
+
scope: 'current-owner foreground and scheduled runs in this relay',
|
|
72
|
+
retention: 'relay memory; resets on restart except adopted host runs; older origin records may be absent',
|
|
73
|
+
activity: 'relay lifecycle, setup attempts and delivery enqueue times only; silence is not idle or completion',
|
|
74
|
+
nativeActivity: 'unavailable; native tool and delegation events are not retained',
|
|
75
|
+
relationships: 'schedule-origin links only; native child status is unknown',
|
|
76
|
+
excluded: ['standalone CLI/desktop work', 'other installations', 'external/isolated tasks', 'prompts, messages, application context and raw engine output'],
|
|
77
|
+
}, runs }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export type RunningWorkView = ReturnType<typeof projectRunningWork>
|
|
81
|
+
|
|
82
|
+
export async function runningWork(controlDir: string, options: { callerRunId?: string } = {}): Promise<RunningWorkView> {
|
|
83
|
+
if (options.callerRunId !== undefined) {
|
|
84
|
+
const caller = await new RunStore(controlDir).get(options.callerRunId)
|
|
85
|
+
if (caller?.application || caller?.delivery || caller?.replyOnly)
|
|
86
|
+
throw new Error('Running work requires Telegram owner execution or the bound operator')
|
|
87
|
+
await requireOwnerExecution(controlDir, options.callerRunId)
|
|
88
|
+
}
|
|
89
|
+
const owner = await readOnlyOwner(controlDir)
|
|
90
|
+
if (!owner) throw new Error('No authorized owner')
|
|
91
|
+
const schedules = await new Scheduler(controlDir).listReadOnly()
|
|
92
|
+
return projectRunningWork(await new RunStore(controlDir).list(), retainedOutbox(controlDir),
|
|
93
|
+
schedules, owner)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const duration = (seconds: number | null): string => {
|
|
97
|
+
if (seconds === null) return 'unknown'
|
|
98
|
+
if (seconds < 60) return `${seconds}s`
|
|
99
|
+
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`
|
|
100
|
+
return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds % 3600 / 60)}m`
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function formatRunningWork(view: RunningWorkView): string {
|
|
104
|
+
const lines = [`Queued and running work: ${view.counts.running} running, ${view.counts.queued} queued`, `Observed: ${view.observedAt}`]
|
|
105
|
+
if (!view.runs.length) lines.push('No queued or running work observed in the retained relay ledger.')
|
|
106
|
+
for (const run of view.runs) {
|
|
107
|
+
lines.push('', `${run.id} ${run.status} ${run.kind}`)
|
|
108
|
+
if (run.schedule) lines.push(` Schedule: ${run.schedule.name ?? '(name unavailable)'} (${run.schedule.id}, revision ${run.schedule.revision}); due ${run.schedule.dueAt ?? 'unknown'}`)
|
|
109
|
+
if (run.quotaWait) lines.push(` Waiting for quota: ${run.quotaWait.retryAt ? `next check ${run.quotaWait.retryAt}` : 'awaiting availability or manual retry'}; /retry_quota ${run.id}`)
|
|
110
|
+
if (run.script) lines.push(` Script: ${run.script.id}; engine/model: not applicable`)
|
|
111
|
+
else lines.push(` Engine/model: ${run.execution?.cli ?? 'unknown'} / ${run.execution?.model ?? 'unknown'}${run.execution?.effort ? `; effort ${run.execution.effort}` : ''}${run.execution ? ` (${run.execution.source}${run.execution.attemptOutcome ? `: ${run.execution.attemptOutcome}` : ''})` : ''}`)
|
|
112
|
+
lines.push(` Started: ${run.startedAt ?? (run.status === 'queued' ? 'not started' : 'unknown')}; elapsed: ${run.status === 'queued' ? 'not applicable' : duration(run.elapsedSeconds)}${run.status === 'queued' ? `; queued for ${duration(run.queuedSeconds)}` : ''}`)
|
|
113
|
+
const activity = run.lastObservedActivity
|
|
114
|
+
lines.push(` Last observed: ${activity ? `${activity.at} (${duration(activity.ageSeconds)} ago; ${activity.kind})` : 'unknown'}`)
|
|
115
|
+
if (run.origin) lines.push(` Schedule origin: ${run.origin.runId} (${run.origin.status ?? 'status unknown; record unavailable'})`)
|
|
116
|
+
if (run.scheduledChildren.length) lines.push(` Active scheduled children: ${run.scheduledChildren.map(item => `${item.id} (${item.status})`).join(', ')}`)
|
|
117
|
+
lines.push(' Native children: unavailable')
|
|
118
|
+
}
|
|
119
|
+
lines.push('', `Visibility: ${view.coverage.scope}.`, view.coverage.retention + '.', view.coverage.activity + '.',
|
|
120
|
+
'Native tool activity and child status are unavailable. Schedule origins are request provenance.',
|
|
121
|
+
'Standalone CLI/desktop work, other installations and external/isolated runs are outside this view.')
|
|
122
|
+
return lines.join('\n')
|
|
123
|
+
}
|
package/src/runs.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { validScheduledOrigin, type ScheduledOrigin } from './scheduler.js'
|
|
|
10
10
|
import type { IncomingItem } from './inbox.js'
|
|
11
11
|
import { assertId } from './identity.js'
|
|
12
12
|
import { isExecutionChoice, type AiPreset, type ExecutionChoice } from './ai.js'
|
|
13
|
-
import type { SetupAttempt } from './setup-fallback.js'
|
|
13
|
+
import type { SetupAttempt, QuotaWait } from './setup-fallback.js'
|
|
14
14
|
import type { ScriptRunRef } from './scripts.js'
|
|
15
15
|
import {authorizeDeliveryContext,currentDeliveryOwner,type DeliveryContext} from './delivery-context.mjs'
|
|
16
16
|
|
|
@@ -38,6 +38,11 @@ export type RunRecord = {
|
|
|
38
38
|
execution?: ExecutionChoice
|
|
39
39
|
// Ordered task setups tried for this occurrence; the last one is active.
|
|
40
40
|
attempts?: SetupAttempt[]
|
|
41
|
+
// Queued quota wait, also retained while its availability probe starts.
|
|
42
|
+
quotaWait?: QuotaWait
|
|
43
|
+
attemptRound?: number
|
|
44
|
+
// Owner-enabled run telemetry, fixed when the run starts.
|
|
45
|
+
telemetry?: true
|
|
41
46
|
replyOnly?: boolean
|
|
42
47
|
// Silent end-of-conversation turn in a replaced chat session (session-close.ts).
|
|
43
48
|
sessionClose?: true
|
|
@@ -273,7 +278,7 @@ export class RunStore {
|
|
|
273
278
|
|
|
274
279
|
async patch(
|
|
275
280
|
id: string,
|
|
276
|
-
change: Partial<Pick<RunRecord, 'status' | 'startedAt' | 'endedAt' | 'pid' | 'nativeSessionId' | 'interrupted' | 'blockReason' | 'backendSubmitted' | 'replyOnly' | 'exitCode' | 'failureReason' | 'failure' | 'failureReview' | 'externalReleased' | 'output' | 'timedOut' | 'attempts'>>,
|
|
281
|
+
change: Partial<Pick<RunRecord, 'status' | 'startedAt' | 'endedAt' | 'pid' | 'nativeSessionId' | 'interrupted' | 'blockReason' | 'backendSubmitted' | 'replyOnly' | 'exitCode' | 'failureReason' | 'failure' | 'failureReview' | 'externalReleased' | 'output' | 'timedOut' | 'attempts' | 'quotaWait' | 'attemptRound' | 'telemetry'>>,
|
|
277
282
|
): Promise<RunRecord> {
|
|
278
283
|
const store = runsFor(this.controlDir)
|
|
279
284
|
const run = store.get(id)
|
package/src/schedule-cli.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { needsFailureReview, failureStamp, redactFailure } from './failure.js'
|
|
2
2
|
import { parseArgs } from 'node:util'
|
|
3
3
|
import { readFile } from 'node:fs/promises'
|
|
4
|
+
import { statSync } from 'node:fs'
|
|
4
5
|
import { resolve } from 'node:path'
|
|
5
6
|
import { randomUUID } from 'node:crypto'
|
|
6
7
|
import { loadControlConfig } from './config.js'
|
|
@@ -16,6 +17,7 @@ import { executorEnvironment } from './executor.js'
|
|
|
16
17
|
import { provisionedHome } from './auth-profile.js'
|
|
17
18
|
import { ownsRun } from './identity.js'
|
|
18
19
|
import { type Trigger } from './schedule-time.js'
|
|
20
|
+
import { formatRunningWork, type RunningWorkView } from './running-work.js'
|
|
19
21
|
|
|
20
22
|
// Each fallback selects its own native engine, credential home and model.
|
|
21
23
|
const fallbackSetup = (spec: string, index: number): AiPreset => {
|
|
@@ -38,16 +40,20 @@ async function main() {
|
|
|
38
40
|
const { values:v, positionals:[action='list',id,scriptId] } = parseArgs({allowPositionals:true,options:{
|
|
39
41
|
script:{type:'string'}, arg:{type:'string',multiple:true}, file:{type:'string'}, interpreter:{type:'string'}, 'timeout-seconds':{type:'string'},
|
|
40
42
|
cli:{type:'string'}, 'auth-profile':{type:'string'}, model:{type:'string'}, effort:{type:'string'}, fallback:{type:'string',multiple:true}, 'clear-fallbacks':{type:'boolean'},
|
|
41
|
-
|
|
43
|
+
'expected-revision':{type:'string'}, stage:{type:'boolean'},
|
|
44
|
+
json:{type:'boolean'}, all:{type:'boolean'}, limit:{type:'string'}, offset:{type:'string'}, expected:{type:'string'}, key:{type:'string'}, when:{type:'string'}, status:{type:'string'}, diagnosis:{type:'string'}, recovery:{type:'string'}, outcome:{type:'string'}, 'failed-at':{type:'string'},
|
|
42
45
|
'preflight-file':{type:'string'},'clear-preflight':{type:'boolean'},name:{type:'string'}, text:{type:'string'}, 'text-file':{type:'string'}, at:{type:'string'}, now:{type:'boolean'},
|
|
43
46
|
cron:{type:'string'}, timezone:{type:'string'}, 'every-seconds':{type:'string'}, start:{type:'string'}, until:{type:'string'}, help:{type:'boolean'},
|
|
44
47
|
}})
|
|
45
48
|
if(v.help){console.log(`ezenciel-agents-schedule list | runs | show ID | pause ID | resume ID | remove ID | cancel RUN_ID
|
|
49
|
+
running [--json] Read queued/running foreground and scheduled work in this relay
|
|
50
|
+
retry [RUN_ID] — retry quota-blocked work after resetting a limit or adding credits
|
|
46
51
|
failures [--all] [--limit N] | run RUN_ID | context
|
|
47
52
|
evidence [RUN_ID] [--offset N --limit N --expected SNAPSHOT_SHA256]
|
|
48
53
|
trigger SCHEDULE_ID --key REQUEST_KEY
|
|
49
54
|
review RUN_ID --failed-at ISO --status resolved|attention --diagnosis TEXT --recovery TEXT --outcome TEXT
|
|
50
|
-
create [ID] | edit ID --
|
|
55
|
+
create [ID] | edit ID --expected-revision REVISION [--stage]
|
|
56
|
+
--name NAME (--text TEXT | --text-file FILE | --script SCRIPT_ID [--arg=VALUE ...])
|
|
51
57
|
--now | --at ISO_WITH_OFFSET | --every-seconds N | --cron 'MIN HOUR DAY MONTH WEEKDAY' --timezone IANA
|
|
52
58
|
[--cli EXECUTOR] [--auth-profile NAME] [--model MODEL] [--effort <native-effort>]
|
|
53
59
|
[--fallback cli=EXECUTOR,model=MODEL[,auth-profile=NAME][,effort=EFFORT][,provider=ID] ... | --clear-fallbacks]
|
|
@@ -64,6 +70,10 @@ Script runs receive EZ_RUN_ID, EZ_SCHEDULE_ID, EZ_SCHEDULE_REVISION, EZ_DUE_AT,
|
|
|
64
70
|
EZ_SCRIPT_SHA256 plus the normal executor allowlist. Stdout/stderr are bounded diagnostics in run, never owner messages.
|
|
65
71
|
Script runs have a timeout (default ${DEFAULT_SCRIPT_TIMEOUT_SECONDS}s) and are never retried automatically.
|
|
66
72
|
Context reads the current run only.
|
|
73
|
+
Running is read-only and defaults to readable output; --json exports sanitized metadata.
|
|
74
|
+
It includes current-owner foreground turns (Telegram/application) and schedules, with recorded setup, times and schedule-origin links.
|
|
75
|
+
Activity covers relay lifecycle, setup attempts and delivery enqueue times; native tools and child status are unavailable.
|
|
76
|
+
Missing or old observations never mean idle or complete. Standalone CLI/desktop and external/isolated work are outside this view.
|
|
67
77
|
Evidence exports sanitized current-owner Telegram operational metadata only.
|
|
68
78
|
Follow nextOffset with --expected; a changed snapshot requires restarting.
|
|
69
79
|
Relay retention is incomplete. Content hashes identify submitted text, not recipient content.
|
|
@@ -72,7 +82,9 @@ Failures default to unreviewed owner runs. Review records a diagnosis; it never
|
|
|
72
82
|
A conditional review schedule consumes no model run when there are no unreviewed failures.
|
|
73
83
|
New tasks capture the selected engine settings; edit preserves existing settings unless overridden.
|
|
74
84
|
Every task requires a concrete saved model. Supply --model if the selected settings have none; existing tasks never fall back to chat or client defaults.
|
|
75
|
-
|
|
85
|
+
Tasks reuse the last successful login among the leading saved setups with the same engine, provider,
|
|
86
|
+
model and effort, and skip logins until any reported quota reset. Other setup priority stays as saved.
|
|
87
|
+
Fallbacks continue only when the previous setup's CLI is unavailable or its provider
|
|
76
88
|
rejected the turn before any work began, including an unavailable login. A Claude usage limit the run ended on
|
|
77
89
|
after work began continues on the next setup in a fresh session, told the stopped session and that its work may be
|
|
78
90
|
partly applied. Account/organization access denial, unknown errors, cancellation and other failures after work
|
|
@@ -83,16 +95,32 @@ The run records each attempt, setup, failure category and any reported reset tim
|
|
|
83
95
|
Trigger runs an existing task once with its saved instructions, AI and delivery binding in a fresh session. Reuse the request key after an uncertain result; the regular schedule is unchanged.
|
|
84
96
|
Creates a scheduled task. Instructions are text, never shell commands.
|
|
85
97
|
Use --now to run once. Run completion is not delivery proof.
|
|
86
|
-
Edit replaces the full schedule
|
|
98
|
+
Edit replaces the full schedule from the revision you read (show prints it); a changed revision is rejected.
|
|
99
|
+
It applies only while the task has no queued or running occurrence. --stage applies it to future occurrences
|
|
100
|
+
and lets those occurrences finish with their captured instructions and settings. Edit needs the running relay.
|
|
101
|
+
Edit keeps the paused/enabled state; only pause/resume toggle it.
|
|
87
102
|
Pause/remove affect future work; cancel stops a particular run.
|
|
88
103
|
Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/weekday OR semantics.`);return}
|
|
89
|
-
const config=loadControlConfig()
|
|
104
|
+
const config=loadControlConfig()
|
|
105
|
+
// A root operator command (for example `docker compose exec relay ...`) acts as
|
|
106
|
+
// the runtime user that owns the control directory: saved definitions stay
|
|
107
|
+
// readable by the relay, and root never writes through agent-owned paths.
|
|
108
|
+
if(process.geteuid?.()===0){
|
|
109
|
+
const {uid,gid}=statSync(config.controlDir)
|
|
110
|
+
process.setgroups!([gid]);process.setgid!(gid);process.setuid!(uid)
|
|
111
|
+
}
|
|
112
|
+
const socketPath=socketPathFor(config.controlDir)
|
|
113
|
+
if(action==='running'){
|
|
114
|
+
if(id || scriptId || Object.keys(v).some(key=>key!=='json'))throw new Error('Running accepts only --json')
|
|
115
|
+
const result=await callDeliverySocket(socketPath,{op:'runningWork',payload:{callerRunId:process.env.EZ_RUN_ID || undefined}}) as RunningWorkView
|
|
116
|
+
console.log(v.json ? JSON.stringify(result,null,2) : formatRunningWork(result));return
|
|
117
|
+
}
|
|
118
|
+
const control=new ControlStore(config.controlDir,config.pairingTtlMs)
|
|
90
119
|
const owner=(await control.status()).owner
|
|
91
120
|
if(!owner)throw new Error('Pair an owner before scheduling')
|
|
92
|
-
const socketPath=socketPathFor(config.controlDir)
|
|
93
121
|
// Run records live in the relay memory ledger, reached through the delivery
|
|
94
|
-
// socket. Schedule files stay on disk, so list/show/create/
|
|
95
|
-
//
|
|
122
|
+
// socket. Schedule files stay on disk, so list/show/create/pause/resume/remove
|
|
123
|
+
// keep working with the relay stopped; edit and run-inspecting actions
|
|
96
124
|
// (context/failures/run/review/runs/cancel) require the running relay.
|
|
97
125
|
const socketUnavailable = (error: unknown): boolean =>
|
|
98
126
|
error instanceof Error && /Delivery relay unavailable/.test(error.message)
|
|
@@ -177,6 +205,10 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
|
|
|
177
205
|
}
|
|
178
206
|
}else if(action==='list')result=await Promise.all((await scheduler.list()).filter(owned).map(show))
|
|
179
207
|
else if(action==='runs')result=(await runs.list()).filter(r=>r.scheduled && r.scheduled.pairedAt===owner.pairedAt && ownsRun(owner,r))
|
|
208
|
+
else if(action==='retry'){
|
|
209
|
+
if(Object.keys(v).length || scriptId)throw new Error('Retry takes only an optional quota-blocked run ID')
|
|
210
|
+
result=await callDeliverySocket(socketPath,{op:'retryScheduleQuota',payload:{...(id ? {runId:id} : {}),...(caller ? {callerRunId:caller.id} : {})}})
|
|
211
|
+
}
|
|
180
212
|
else if(action==='trigger'){
|
|
181
213
|
if(!id || !v.key)throw new Error('Trigger requires SCHEDULE_ID and --key REQUEST_KEY')
|
|
182
214
|
if(Object.keys(v).some(key=>key!=='key'))throw new Error('Trigger uses saved task settings; overrides are not allowed')
|
|
@@ -185,6 +217,8 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
|
|
|
185
217
|
result=await callDeliverySocket(socketPath,{op:'triggerSchedule',payload:{scheduleId:id,revision:schedule.revision,key:v.key,...(caller ? {callerRunId:caller.id} : {})}})
|
|
186
218
|
}else if(action==='create' || action==='edit'){
|
|
187
219
|
if(action==='edit' && (!id || !owned(await scheduler.get(id))))throw new Error('Unknown schedule')
|
|
220
|
+
if(action==='edit' && !v['expected-revision'])throw new Error('Edit requires --expected-revision REVISION (the revision shown by show ID)')
|
|
221
|
+
if(action==='create' && (v['expected-revision'] || v.stage))throw new Error('--expected-revision and --stage apply to edit only')
|
|
188
222
|
if(action==='create' && id && (await scheduler.list()).some(s=>s.id===id))throw new Error('Schedule exists; use edit')
|
|
189
223
|
if([v.now,v.at,v.cron,v['every-seconds']].filter(Boolean).length!==1)throw new Error('Choose exactly one trigger')
|
|
190
224
|
if([v.text,v['text-file'],v.script].filter(Boolean).length!==1)throw new Error('Choose --text, --text-file or --script')
|
|
@@ -196,16 +230,21 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
|
|
|
196
230
|
v.cron ? {cron:v.cron,timezone:v.timezone!,start,until:v.until} : {everySeconds:Number(v['every-seconds']),start,until:v.until}
|
|
197
231
|
if(v['clear-preflight'] && v['preflight-file']) throw Error('Choose one preflight operation')
|
|
198
232
|
const previousSchedule = action === 'edit' ? await scheduler.get(id!) : undefined
|
|
199
|
-
//
|
|
200
|
-
const
|
|
233
|
+
// New tasks start enabled; edit compares the revision and keeps the paused/enabled state under the schedule lock.
|
|
234
|
+
const write = (input: Parameters<Scheduler['save']>[0]) => action === 'create' ? scheduler.save(input, true) :
|
|
235
|
+
// A bounded ledger read keeps the schedule lock short.
|
|
236
|
+
scheduler.edit(input, v['expected-revision']!, () => callDeliverySocket(socketPath, { op: 'list' }, 10_000).then(r => r as RunRecord[]).catch(error => {
|
|
237
|
+
if (socketUnavailable(error)) throw new Error('Relay unavailable: edit needs the running relay to confirm which occurrences are queued or running')
|
|
238
|
+
throw error
|
|
239
|
+
}), v.stage)
|
|
201
240
|
const preflight = v['clear-preflight'] ? undefined : v['preflight-file'] ? JSON.parse(await readFile(v['preflight-file'],'utf8')) : previousSchedule?.preflight
|
|
202
241
|
const origin = caller?.application ?? caller?.delivery
|
|
203
242
|
const delivery = previousSchedule ? previousSchedule.delivery : (origin ? {bindingId:origin.bindingId,scope:origin.scope} : undefined)
|
|
204
243
|
if (!delivery && !owner.telegramChatId) throw new Error('Create the schedule from an authenticated channel turn to bind its reply destination')
|
|
205
244
|
if (v.script) {
|
|
206
|
-
result=await show(await
|
|
207
|
-
originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
|
|
208
|
-
script:{id:v.script,args:v.arg ?? []}}
|
|
245
|
+
result=await show(await write({id:id || 's_'+randomUUID(),name:v.name || v.script,preflight,
|
|
246
|
+
originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
|
|
247
|
+
script:{id:v.script,args:v.arg ?? []}}))
|
|
209
248
|
console.log(JSON.stringify(result,null,2));return
|
|
210
249
|
}
|
|
211
250
|
const previous = previousSchedule?.execution
|
|
@@ -217,10 +256,10 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
|
|
|
217
256
|
const fallbacks = v['clear-fallbacks'] ? undefined : v.fallback ? v.fallback.map(fallbackSetup) : previous?.fallbacks
|
|
218
257
|
for (const setup of [preset, ...fallbacks ?? []])
|
|
219
258
|
if (setup.authProfile !== undefined) await provisionedHome(config.controlDir, setup.cli, setup.authProfile)
|
|
220
|
-
result=await show(await
|
|
259
|
+
result=await show(await write({id:id || 's_'+randomUUID(),name:v.name || 'Task',
|
|
221
260
|
preflight,
|
|
222
|
-
originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
|
|
223
|
-
execution:{sessionId:previous?.sessionId || randomUUID(),preset,...(fallbacks ? {fallbacks} : {})}}
|
|
261
|
+
originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
|
|
262
|
+
execution:{sessionId:previous?.sessionId || randomUUID(),preset,...(fallbacks ? {fallbacks} : {})}}))
|
|
224
263
|
}else{
|
|
225
264
|
if(!id)throw new Error('ID required')
|
|
226
265
|
if(action==='cancel'){
|