@jc_stack/ez-agents 0.1.0-beta.49 → 0.1.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -6
- package/CHANGELOG.md +15 -0
- package/README.md +9 -5
- package/bin/ez +16 -7
- package/docs/architecture/ai-selection.md +4 -0
- package/docs/docker-runtime.md +18 -2
- package/docs/plugin-guide.md +3 -3
- package/docs/plugins.md +13 -10
- package/docs/scheduling.md +55 -18
- package/docs/setup.md +2 -2
- package/docs/standalone-cli.md +4 -4
- package/docs/upgrades.md +9 -13
- package/package.json +1 -1
- package/src/agent-guidance.ts +2 -2
- package/src/ai.ts +17 -5
- package/src/application-channel.ts +12 -8
- package/src/executor.ts +9 -0
- package/src/host-executor-client.ts +19 -7
- package/src/index.ts +125 -21
- package/src/menu.ts +3 -2
- package/src/plugins/manager.mjs +43 -40
- package/src/runs.ts +61 -3
- package/src/schedule-cli.ts +18 -12
- package/src/scheduled-tasks.ts +42 -8
- package/src/scheduler.ts +1 -1
- package/src/session-close.ts +17 -0
- package/src/setup-fallback.ts +62 -25
- package/src/tools-line.d.mts +1 -0
- package/src/tools-line.mjs +3 -0
- package/src/updates/binding.mjs +4 -10
- package/src/updates/control.mjs +3 -2
- package/src/updates/supervisor.mjs +1 -1
- package/src/workspace.ts +12 -1
- package/test/ai.test.ts +48 -2
- package/test/application-channel.test.ts +24 -0
- package/test/application-controls.test.ts +4 -1
- package/test/intake-relay.test.ts +63 -2
- package/test/model-policy.test.ts +1 -1
- package/test/plugin-broker.test.mjs +48 -1
- package/test/plugin-manager.test.mjs +52 -38
- package/test/run-handoff.test.ts +132 -0
- package/test/scheduled-tasks.test.ts +24 -2
- package/test/session-close.test.ts +17 -0
- package/test/setup-fallback.test.ts +194 -16
- package/test/updates.test.mjs +19 -10
- package/test/workspace.test.ts +6 -0
|
@@ -1,17 +1,29 @@
|
|
|
1
1
|
// File transport across the Docker/host boundary. The host chooses the CLI,
|
|
2
2
|
// workspace and environment; a request cannot choose a command or credentials.
|
|
3
|
-
import { mkdir, readFile, writeFile, rename, rm } from 'node:fs/promises'
|
|
3
|
+
import { mkdir, readFile, writeFile, rename, rm, stat } from 'node:fs/promises'
|
|
4
4
|
import path from 'node:path'
|
|
5
5
|
import { isHostRunId } from './host-executor-protocol.js'
|
|
6
|
-
const [control, id] = process.argv.slice(2)
|
|
7
|
-
if (!control || !isHostRunId(id || '')) throw new Error('Invalid executor binding')
|
|
6
|
+
const [control, id, mode] = process.argv.slice(2)
|
|
7
|
+
if (!control || !isHostRunId(id || '') || (mode !== undefined && mode !== '--attach')) throw new Error('Invalid executor binding')
|
|
8
8
|
const directory = path.join(control, 'host-executor')
|
|
9
9
|
await mkdir(directory, {recursive:true, mode:0o700})
|
|
10
|
-
let input = ''
|
|
11
|
-
for await (const chunk of process.stdin) input += chunk
|
|
12
10
|
const base = path.join(directory, id)
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
if (mode === '--attach') {
|
|
12
|
+
// A previous relay submitted this run: watch it, never resubmit. Without a
|
|
13
|
+
// request, claim or event record its outcome is unknown.
|
|
14
|
+
// Check in lifecycle order: claim renames the request, and the host writes
|
|
15
|
+
// the exit event before removing the claim, so a live run is always seen.
|
|
16
|
+
const exists = (suffix: string) => stat(base+suffix).then(() => true, error => { if (error.code === 'ENOENT') return false; throw error })
|
|
17
|
+
if (!await exists('.request.json') && !await exists('.running.json') && !await exists('.events')) {
|
|
18
|
+
console.error('Handed-off host run has no request, claim or result; its outcome is unknown')
|
|
19
|
+
process.exit(1)
|
|
20
|
+
}
|
|
21
|
+
} else {
|
|
22
|
+
let input = ''
|
|
23
|
+
for await (const chunk of process.stdin) input += chunk
|
|
24
|
+
await writeFile(base+'.tmp', input, {mode:0o600, flag:'wx'})
|
|
25
|
+
await rename(base+'.tmp', base+'.request.json')
|
|
26
|
+
}
|
|
15
27
|
let interrupted = false
|
|
16
28
|
for (const signal of ['SIGTERM','SIGINT'] as const) process.once(signal, () => {
|
|
17
29
|
if (interrupted) return
|
package/src/index.ts
CHANGED
|
@@ -9,9 +9,9 @@ import { packageVersion } from './version.js'
|
|
|
9
9
|
import {authorizeDeliveryContext} from './delivery-context.mjs'
|
|
10
10
|
import { dispatchChannel } from './channel-backend.js'
|
|
11
11
|
import { randomUUID } from 'node:crypto'
|
|
12
|
-
import { stat } from 'node:fs/promises'
|
|
12
|
+
import { mkdir, stat, writeFile } from 'node:fs/promises'
|
|
13
13
|
import { Scheduler } from './scheduler.js'
|
|
14
|
-
import { Scripts } from './scripts.js'
|
|
14
|
+
import { Scripts, DEFAULT_SCRIPT_TIMEOUT_SECONDS } from './scripts.js'
|
|
15
15
|
import { redactFailure } from './failure.js'
|
|
16
16
|
import { ownedScheduledTasks, scheduledTaskDetailText, scheduledTasksText } from './scheduled-tasks.js'
|
|
17
17
|
import { EventSources, eventRunId, batchReady, type SourceEvent } from './event-sources.js'
|
|
@@ -25,12 +25,12 @@ import { isOwner, ownsRun } from './identity.js'
|
|
|
25
25
|
import type { Update } from 'grammy/types'
|
|
26
26
|
import { InboxStore, type IncomingItem } from './inbox.js'
|
|
27
27
|
import { loadConfig, type Config } from './config.js'
|
|
28
|
-
import { ControlStore, telegramOwner, ownerId, ownerEpoch } from './control-state.js'
|
|
28
|
+
import { ControlStore, telegramOwner, ownerId, ownerEpoch, type SessionState } from './control-state.js'
|
|
29
29
|
import { ApprovalStore } from './approval.js'
|
|
30
|
-
import { CliUnavailableError, startExecutorJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
|
|
31
|
-
import {
|
|
30
|
+
import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
|
|
31
|
+
import { attemptFailure, continuationLine, nextSetup, quotaScope, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
|
|
32
32
|
import { assertScheduledModel } from './model-policy.js'
|
|
33
|
-
import { RunStore, type RunRecord } from './runs.js'
|
|
33
|
+
import { RunStore, saveRunHandoff, takeRunHandoff, type RunRecord } from './runs.js'
|
|
34
34
|
import { splitTelegramText } from './reply.js'
|
|
35
35
|
import { markdownToTelegramHtml, escapeHtml } from './format.js'
|
|
36
36
|
import { sanitizeFileName, stageChatAttachment, workspaceFile } from './files.js'
|
|
@@ -38,6 +38,7 @@ import { transcribeAudio, synthesizeSpeech } from './audio.js'
|
|
|
38
38
|
import { normalizeReactionEmoji } from './reaction.js'
|
|
39
39
|
import { downloadTelegramFile } from './read-request.js'
|
|
40
40
|
import { createConversationMenu } from './conversation-menu.js'
|
|
41
|
+
import { queueSessionClose } from './session-close.js'
|
|
41
42
|
import { createAiMenu, mainCommands, mainKeyboard } from './menu.js'
|
|
42
43
|
import { chatPreset, initialPreset, persistedPreset, presetLabel, statusPreset, type AiPreset } from './ai.js'
|
|
43
44
|
import { discoverDefaults } from './client-defaults.js'
|
|
@@ -77,13 +78,27 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
77
78
|
const codexHome = join(config.controlDir, 'cli', 'codex')
|
|
78
79
|
const conversationMenu = createConversationMenu(control)
|
|
79
80
|
const aiMenu = createAiMenu(control, config.executorCli, undefined, config.workspace, codexHome,
|
|
80
|
-
undefined, opencodeDataHome(config.controlDir))
|
|
81
|
+
undefined, opencodeDataHome(config.controlDir), (session) => closeSession(session))
|
|
81
82
|
const durableWorkerChoice = () => control.captureChoice(aiMenu.initial)
|
|
82
83
|
const binDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'bin')
|
|
84
|
+
// Queue the replaced chat session's silent close turn (session-close.ts).
|
|
85
|
+
// A busy relay starts it after the active run, ahead of newer chat input.
|
|
86
|
+
const closeSession = async (session: SessionState | null): Promise<void> => {
|
|
87
|
+
if (config.channelBackendUrl) return
|
|
88
|
+
// The replacement is already saved; a failed close must not fail that reply.
|
|
89
|
+
try {
|
|
90
|
+
const run = await queueSessionClose(runs, (await control.status()).owner, session)
|
|
91
|
+
if (run) void startJob(run).catch((error) => console.error('Conversation close failed to start', safeError(error)))
|
|
92
|
+
} catch (error) { console.error('Conversation close failed to queue', safeError(error)) }
|
|
93
|
+
}
|
|
83
94
|
|
|
84
95
|
let activeTypingTimer: ReturnType<typeof setInterval> | null = null
|
|
85
96
|
let activeBackend = false
|
|
86
97
|
const ownerStopped = new WeakSet<ChildProcess>()
|
|
98
|
+
// Host-transport executors outlive this relay; a graceful stop hands them off.
|
|
99
|
+
const hostTransport = process.env.EZ_EXECUTOR_TRANSPORT === 'host' && !config.channelBackendUrl
|
|
100
|
+
const childRuns = new WeakMap<ChildProcess, string>()
|
|
101
|
+
const handedOff = new WeakSet<ChildProcess>()
|
|
87
102
|
let activeChild: ChildProcess | null = null
|
|
88
103
|
let runtimeStarted = false
|
|
89
104
|
let shuttingDown = false
|
|
@@ -114,6 +129,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
114
129
|
format: 'wav',
|
|
115
130
|
}) : undefined,
|
|
116
131
|
wake: () => { void drainSources().catch(error => console.error('Application queue unavailable', safeError(error))) },
|
|
132
|
+
closeSession: (session) => closeSession(session),
|
|
117
133
|
createTelegramPairing: async (bindingId, owner) => {
|
|
118
134
|
const username = bot?.botInfo?.username
|
|
119
135
|
if (!username || !/^[A-Za-z][A-Za-z0-9_]{4,31}$/.test(username))
|
|
@@ -178,7 +194,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
178
194
|
// stat uses the effective UID; access uses the relay's isolated real UID.
|
|
179
195
|
if (await stat(join(config.controlDir,'upgrade-pause.json')).then(()=>true,e=>{if(e.code==='ENOENT')return false;throw e})) return
|
|
180
196
|
if ((await runs.get(run.id))?.status !== 'queued') return
|
|
181
|
-
if (!telegramEnabled && !run.application && !run.delivery && !run.scheduled && !run.external) return
|
|
197
|
+
if (!telegramEnabled && !run.application && !run.delivery && !run.scheduled && !run.external && !run.sessionClose) return
|
|
182
198
|
if (!run.scheduled && await runs.running(false)) return
|
|
183
199
|
const owner = (await control.status()).owner
|
|
184
200
|
if (!owner || !ownsRun(owner, run)) {
|
|
@@ -280,8 +296,8 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
280
296
|
const settle = (change: Partial<SetupAttempt>) => {
|
|
281
297
|
attempts = [...attempts.slice(0, -1), { ...attempts[attempts.length - 1], ...change, endedAt: new Date().toISOString() }]
|
|
282
298
|
}
|
|
283
|
-
// Advance only after an eligible
|
|
284
|
-
// occurrence. Setups in an exhausted
|
|
299
|
+
// Advance only after an eligible failure (see attemptFailure) of a current,
|
|
300
|
+
// uncancelled occurrence. Setups in an exhausted scope are skipped and recorded.
|
|
285
301
|
const advance = async (): Promise<AiPreset | undefined> => {
|
|
286
302
|
let preset: AiPreset | undefined
|
|
287
303
|
const current = !shuttingDown && !await scheduler.cancelled(run.id) ? (await control.status()).owner : undefined
|
|
@@ -297,16 +313,18 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
297
313
|
await runs.patch(run.id, { attempts })
|
|
298
314
|
return preset
|
|
299
315
|
}
|
|
300
|
-
// Each attempt is a fresh native session
|
|
316
|
+
// Each attempt is a fresh native session: a rejected Claude attempt already
|
|
317
|
+
// claimed its ID, and a session cannot resume under another login's home.
|
|
301
318
|
const launchSetup = async (preset: AiPreset): Promise<{ child: ChildProcess; cleanup: () => Promise<void>; preset: AiPreset }> => {
|
|
302
319
|
for (;;) {
|
|
303
320
|
const sessionId = attempts.length ? randomUUID() : session.nativeSessionId || session.sessionId
|
|
321
|
+
const continuation = continuationLine(started.id, attempts)
|
|
304
322
|
if (ordered) {
|
|
305
|
-
attempts = [...attempts, { preset, quotaScope: quotaScope(preset), outcome: 'running', startedAt: new Date().toISOString() }]
|
|
323
|
+
attempts = [...attempts, { preset, quotaScope: quotaScope(preset), outcome: 'running', sessionId, startedAt: new Date().toISOString() }]
|
|
306
324
|
await runs.patch(run.id, { attempts })
|
|
307
325
|
}
|
|
308
326
|
try {
|
|
309
|
-
return { ...await launch(texts, script ? {
|
|
327
|
+
return { ...await launch(continuation ? [...texts, continuation] : texts, script ? {
|
|
310
328
|
workspace: config.workspace, timeoutMs: 0, repairEnabled: config.repairEnabled, runId: started.id,
|
|
311
329
|
controlDir: config.controlDir, binDir,
|
|
312
330
|
script: { ...script.ref, scheduleId: script.scheduled.id, scheduleRevision: script.scheduled.revision, dueAt: script.scheduled.dueAt },
|
|
@@ -371,6 +389,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
371
389
|
startup_ms: Math.round(executionStarted - launchStarted), resumed: session.hasStarted })
|
|
372
390
|
if (run.scheduled) background.set(run.id,child)
|
|
373
391
|
else activeChild = child
|
|
392
|
+
childRuns.set(child, run.id)
|
|
374
393
|
const finished = new Promise<number | null>((resolve) => {
|
|
375
394
|
if (child.exitCode !== null || child.signalCode !== null) resolve(child.exitCode)
|
|
376
395
|
else child.once('close', resolve)
|
|
@@ -413,6 +432,8 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
413
432
|
await withStartLock(async () => {
|
|
414
433
|
try {
|
|
415
434
|
await cleanup()
|
|
435
|
+
// The next relay records a handed-off run's result.
|
|
436
|
+
if (handedOff.has(child)) return
|
|
416
437
|
if (code === 0 && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(session.sessionId)
|
|
417
438
|
const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
|
|
418
439
|
if (timedOut) failureReason = 'script-timeout'
|
|
@@ -421,11 +442,10 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
421
442
|
// A rejection before work is a failure even when the CLI exits 0.
|
|
422
443
|
const failed = !cancelled && (code !== 0 || timedOut || Boolean(rejected))
|
|
423
444
|
if (startup) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
...(failed && rejected?.resetAt ? { resetAt: rejected.resetAt } : {}) })
|
|
445
|
+
const { continues, ...failure } = failed ? attemptFailure(observed!) : { continues: false }
|
|
446
|
+
settle({ outcome: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', exitCode: code, ...failure })
|
|
427
447
|
await runs.patch(run.id, { attempts })
|
|
428
|
-
eligible =
|
|
448
|
+
eligible = continues && !interrupted
|
|
429
449
|
}
|
|
430
450
|
if (!eligible) await finalize(code, cancelled, failed)
|
|
431
451
|
} catch (error) {
|
|
@@ -1029,7 +1049,9 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1029
1049
|
|
|
1030
1050
|
if (text === '/new') {
|
|
1031
1051
|
await control.aiState(aiMenu.initial)
|
|
1052
|
+
const previous = await control.getActiveSession()
|
|
1032
1053
|
const next = await control.resetSession()
|
|
1054
|
+
await closeSession(previous)
|
|
1033
1055
|
await ctx.reply(
|
|
1034
1056
|
`🔄 Started fresh conversation session (<code>${next.sessionId.slice(0, 8)}</code>). Agent files and memory preserved.`,
|
|
1035
1057
|
{ parse_mode: 'HTML' },
|
|
@@ -1217,7 +1239,9 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1217
1239
|
await ctx.reply(id ? `Incoming batch ${id} queued for retry.` : 'No failed incoming batch to retry.')
|
|
1218
1240
|
} else if (action === 'new') {
|
|
1219
1241
|
await control.aiState(aiMenu.initial)
|
|
1242
|
+
const previous = await control.getActiveSession()
|
|
1220
1243
|
const next = await control.resetSession()
|
|
1244
|
+
await closeSession(previous)
|
|
1221
1245
|
await ctx.answerCallbackQuery({ text: 'New session started' })
|
|
1222
1246
|
await ctx.reply(
|
|
1223
1247
|
`🔄 Started fresh conversation session (<code>${next.sessionId.slice(0, 8)}</code>).`,
|
|
@@ -1259,11 +1283,90 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1259
1283
|
})
|
|
1260
1284
|
}
|
|
1261
1285
|
|
|
1286
|
+
// Watch runs a previous relay handed off until they finish, with the normal
|
|
1287
|
+
// receipts. Never relaunch; existing loops still revoke and cancel them. An
|
|
1288
|
+
// unknown outcome is recorded interrupted, which holds its schedule.
|
|
1289
|
+
const adoptHandoff = async (): Promise<void> => {
|
|
1290
|
+
const handed = await takeRunHandoff(config.controlDir)
|
|
1291
|
+
if (!hostTransport) { if (handed.length) console.error('Ignoring run handoff without host transport', handed.length); return }
|
|
1292
|
+
// Admit every record before the first attach, so no trigger races a hold.
|
|
1293
|
+
const admitted = (await Promise.all(handed.map(run => runs.adopt(run).catch(() => null)))).filter((run): run is RunRecord => run !== null)
|
|
1294
|
+
for (const run of admitted) await withStartLock(async () => {
|
|
1295
|
+
let child: ChildProcess
|
|
1296
|
+
try {
|
|
1297
|
+
// The foreground lane is serial; a second foreground handoff is corrupt.
|
|
1298
|
+
if (!run.scheduled && activeChild) throw new Error('A second handed-off foreground run was cancelled; its outcome is unknown')
|
|
1299
|
+
child = await attachHostJob(config.controlDir, run.id)
|
|
1300
|
+
} catch (error) {
|
|
1301
|
+
// Never leave an unobserved executor: ask the host to stop it.
|
|
1302
|
+
await mkdir(join(config.controlDir, 'host-executor'), { recursive: true, mode: 0o700 })
|
|
1303
|
+
.then(() => writeFile(join(config.controlDir, 'host-executor', run.id + '.cancel'), '', { mode: 0o600 })).catch(() => {})
|
|
1304
|
+
await runs.patch(run.id, { status: 'failed', endedAt: new Date().toISOString(), interrupted: true, failureReason: 'handoff-outcome-unknown', failure: await failureEvidence(config.controlDir, safeError(error)) })
|
|
1305
|
+
return
|
|
1306
|
+
}
|
|
1307
|
+
// Listen before any await so a fast exit cannot be missed.
|
|
1308
|
+
const closed = new Promise<number | null>(resolve => child.once('close', resolve))
|
|
1309
|
+
let errorTail = '', outputTail = '', timedOut = false
|
|
1310
|
+
child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { if (run.script) outputTail = (outputTail + chunk).slice(-16384) })
|
|
1311
|
+
child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
|
|
1312
|
+
errorTail = (errorTail + chunk).slice(-16384)
|
|
1313
|
+
if (run.script) outputTail = (outputTail + chunk).slice(-16384)
|
|
1314
|
+
})
|
|
1315
|
+
if (run.scheduled) background.set(run.id, child)
|
|
1316
|
+
else activeChild = child
|
|
1317
|
+
childRuns.set(child, run.id)
|
|
1318
|
+
await runs.patch(run.id, { pid: child.pid })
|
|
1319
|
+
console.info('run adopted', { run_id: run.id, pid: child.pid })
|
|
1320
|
+
// A script keeps its registered bound from its original start.
|
|
1321
|
+
const limit = run.script ? await new Scripts(config.controlDir).get(run.script.id).then(script => script.timeoutSeconds, () => DEFAULT_SCRIPT_TIMEOUT_SECONDS) : 0
|
|
1322
|
+
const timer = limit ? setTimeout(() => { timedOut = true; terminateJob(child) }, Math.max(0, Date.parse(run.startedAt ?? run.createdAt) + limit * 1000 - Date.now())) : undefined
|
|
1323
|
+
const completion = closed.then(code => withStartLock(async () => {
|
|
1324
|
+
clearTimeout(timer)
|
|
1325
|
+
if (run.scheduled) background.delete(run.id)
|
|
1326
|
+
else if (activeChild === child) activeChild = null
|
|
1327
|
+
if (handedOff.has(child)) return
|
|
1328
|
+
const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
|
|
1329
|
+
const failed = !cancelled && code !== 0
|
|
1330
|
+
const status = cancelled ? 'cancelled' : failed ? 'failed' : 'completed'
|
|
1331
|
+
const unknown = errorTail.includes('outcome is unknown'), transport = errorTail.includes('Host executor client interrupted by')
|
|
1332
|
+
const failureReason = timedOut ? 'script-timeout' : unknown ? 'handoff-outcome-unknown' : transport ? 'host-executor-transport-interrupted'
|
|
1333
|
+
: errorTail.includes('Host CLI executor is offline') ? 'host-executor-offline' : 'executor-exit'
|
|
1334
|
+
if (code === 0 && run.execution && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(run.execution.sessionId)
|
|
1335
|
+
const at = new Date().toISOString(), last = run.attempts?.at(-1)
|
|
1336
|
+
await runs.patch(run.id, { status, endedAt: at, exitCode: code,
|
|
1337
|
+
...(last?.outcome === 'running' ? { attempts: [...run.attempts!.slice(0, -1), { ...last, outcome: status, exitCode: code, endedAt: at }] } : {}),
|
|
1338
|
+
...(run.script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
|
|
1339
|
+
...(failed ? { failureReason, interrupted: unknown || transport, failure: await failureEvidence(config.controlDir, safeError(errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
|
|
1340
|
+
await releaseExternal(run)
|
|
1341
|
+
console.info('run ended', { run_id: run.id, code, adopted: true })
|
|
1342
|
+
})).then(async () => {
|
|
1343
|
+
const next = await runs.nextQueued()
|
|
1344
|
+
if (next && !shuttingDown) await startJob(next)
|
|
1345
|
+
}).catch(error => console.error('Adopted run completion failed', safeError(error)))
|
|
1346
|
+
completions.add(completion)
|
|
1347
|
+
void completion.finally(() => completions.delete(completion))
|
|
1348
|
+
}).catch(error => console.error('Run adoption failed', run.id, safeError(error)))
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1262
1351
|
let stopWork: Promise<void> | undefined
|
|
1263
1352
|
const stop = (): Promise<void> => stopWork ?? (stopWork = (async () => {
|
|
1264
1353
|
shuttingDown = true
|
|
1265
1354
|
runtimeStarted = false
|
|
1266
1355
|
pollingAbort.abort()
|
|
1356
|
+
// Hand off running host work before any wait can outlast the stop grace
|
|
1357
|
+
// period. The lock lets an in-flight launch register; later ones see shutdown.
|
|
1358
|
+
if (hostTransport) await withStartLock(async () => {
|
|
1359
|
+
const handoff = new Map<ChildProcess, RunRecord>()
|
|
1360
|
+
for (const child of [...(activeChild ? [activeChild] : []), ...background.values()]) {
|
|
1361
|
+
const run = childRuns.has(child) ? await runs.get(childRuns.get(child)!) : null
|
|
1362
|
+
if (run?.status === 'running' && !ownerStopped.has(child) && child.exitCode === null && child.signalCode === null) handoff.set(child, run)
|
|
1363
|
+
}
|
|
1364
|
+
if (!handoff.size) return
|
|
1365
|
+
try { await saveRunHandoff(config.controlDir, [...handoff.values()]) }
|
|
1366
|
+
catch (error) { console.error('Run handoff failed; stopping host work', safeError(error)); return }
|
|
1367
|
+
// SIGKILL leaves no cancel marker, so the host keeps each executor running.
|
|
1368
|
+
for (const child of handoff.keys()) { handedOff.add(child); child.kill('SIGKILL') }
|
|
1369
|
+
})
|
|
1267
1370
|
await deliveryServer?.stop().catch(() => {})
|
|
1268
1371
|
deliveryServer = null
|
|
1269
1372
|
await applicationChannel.stop()
|
|
@@ -1275,8 +1378,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1275
1378
|
await Promise.all([sourceWork, intakeWork].map(work => work?.catch(() => {})))
|
|
1276
1379
|
// Finish registering in-flight launches before taking the child snapshot.
|
|
1277
1380
|
await withStartLock(async () => {
|
|
1278
|
-
|
|
1279
|
-
for (const child of background.values()) terminateJob(child)
|
|
1381
|
+
for (const child of [...(activeChild ? [activeChild] : []), ...background.values()]) if (!handedOff.has(child)) terminateJob(child)
|
|
1280
1382
|
if (activeTypingTimer) clearInterval(activeTypingTimer)
|
|
1281
1383
|
})
|
|
1282
1384
|
await Promise.all(completions)
|
|
@@ -1302,11 +1404,13 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1302
1404
|
deliveryServer = await serveDeliverySocket(config.controlDir, handleDeliveryOp, {
|
|
1303
1405
|
...(Number.isSafeInteger(deliveryTcpPort) && deliveryTcpPort >= 1024 && deliveryTcpPort <= 65535 ? { tcpPort: deliveryTcpPort } : {}),
|
|
1304
1406
|
})
|
|
1407
|
+
// Reconcile, then adopt handed-off runs, before any intake can launch work.
|
|
1408
|
+
await scheduler.recover(runs)
|
|
1409
|
+
await adoptHandoff()
|
|
1305
1410
|
if (config.applicationPort) await applicationChannel.listen(config.applicationPort, config.applicationHost)
|
|
1306
1411
|
runtimeStarted = true
|
|
1307
1412
|
const owner = (await control.status()).owner
|
|
1308
1413
|
if (telegramEnabled && telegramOwner(owner) && !config.channelBackendUrl) await telegramSource!.start(owner!)
|
|
1309
|
-
await scheduler.recover(runs)
|
|
1310
1414
|
sourceTimer = setInterval(() => {
|
|
1311
1415
|
void drainSources().catch(error => console.error('Event-source drain failed', safeError(error)))
|
|
1312
1416
|
}, 1000)
|
|
@@ -1380,7 +1484,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
|
|
|
1380
1484
|
}
|
|
1381
1485
|
}
|
|
1382
1486
|
}
|
|
1383
|
-
return { bot: bot!, telegramEnabled, isRunning: () => telegramEnabled ? Boolean(bot?.isRunning()) : runtimeStarted, start, stop, drainOutbox, drainInbox, drainSources, drainTaskRequests, applicationChannel }
|
|
1487
|
+
return { bot: bot!, telegramEnabled, isRunning: () => telegramEnabled ? Boolean(bot?.isRunning()) : runtimeStarted, start, stop, drainOutbox, drainInbox, drainSources, drainTaskRequests, adoptHandoff, applicationChannel }
|
|
1384
1488
|
}
|
|
1385
1489
|
|
|
1386
1490
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
package/src/menu.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises'
|
|
|
3
3
|
import path from 'node:path'
|
|
4
4
|
import { randomBytes } from 'node:crypto'
|
|
5
5
|
import { InlineKeyboard, type Context } from 'grammy'
|
|
6
|
-
import { ControlStore, type ControlGuard } from './control-state.js'
|
|
6
|
+
import { ControlStore, type ControlGuard, type SessionState } from './control-state.js'
|
|
7
7
|
import { chatPreset, installed, persistedPreset, presetLabel, presetProvider, readModels, sameChoice, sameEngine, validateSelection, type AiPreset, type ModelChoice } from './ai.js'
|
|
8
8
|
import { discoverDefaults } from './client-defaults.js'
|
|
9
9
|
import { claudeConfigHome } from './executor.js'
|
|
@@ -31,7 +31,7 @@ const matchesModel = (preset: AiPreset, model: ModelChoice) =>
|
|
|
31
31
|
// Short-lived opaque button IDs: no model names or executable arguments from callbacks.
|
|
32
32
|
// These are operational settings, not a second conversational/agent loop.
|
|
33
33
|
export const createAiMenu = (control: ControlStore, cli: string, catalog = readModels, workspace = process.cwd(), codexHome?: string,
|
|
34
|
-
isInstalled = installed, opencodeDataHome?: string) => {
|
|
34
|
+
isInstalled = installed, opencodeDataHome?: string, closeSession?: (session: SessionState | null) => Promise<void>) => {
|
|
35
35
|
const initial = chatPreset(cli)
|
|
36
36
|
const host = process.env.EZ_EXECUTOR_TRANSPORT === 'host'
|
|
37
37
|
if (host) catalog = async () => JSON.parse(await readFile(path.join(process.env.EZ_CONTROL_DIR!, 'host-executor/models.json'),'utf8'))
|
|
@@ -68,6 +68,7 @@ export const createAiMenu = (control: ControlStore, cli: string, catalog = readM
|
|
|
68
68
|
const current = state.presets.find((p) => p.id === state.selectedId)!
|
|
69
69
|
const fresh = !sameEngine(current, preset) || presetProvider(current) !== presetProvider(preset) || Boolean(session && !session.cli)
|
|
70
70
|
if (!await control.selectPreset(preset.id, expectedSession, fresh, guard)) throw new Error('AI binding changed. Refresh available AIs before trying again.')
|
|
71
|
+
if (fresh) await closeSession?.(session)
|
|
71
72
|
return { preset, fresh }
|
|
72
73
|
}
|
|
73
74
|
const choose = async (ctx: Context, preset: AiPreset) => {
|
package/src/plugins/manager.mjs
CHANGED
|
@@ -7,7 +7,8 @@ import { fileURLToPath } from 'node:url';
|
|
|
7
7
|
import { createHash, randomUUID, randomBytes } from 'node:crypto';
|
|
8
8
|
import { spawn } from 'node:child_process';
|
|
9
9
|
import { callDeliverySocket } from '../delivery-socket-client.mjs';
|
|
10
|
-
import {
|
|
10
|
+
import { TOOLS_LINE } from '../tools-line.mjs';
|
|
11
|
+
import { readFileSync, readlinkSync, realpathSync } from 'node:fs';
|
|
11
12
|
|
|
12
13
|
const reserved = new Set(['status','updates','plugins','tools','message','owner','approval','react','setup','help','version']);
|
|
13
14
|
const id = value => { if(typeof value !== 'string' || !/^[a-z][a-z0-9-]{0,39}$/.test(value)) throw Error('Invalid identifier'); return value; };
|
|
@@ -41,39 +42,31 @@ export async function stewardOwned(file) {
|
|
|
41
42
|
const parent = await fs.stat(path.dirname(file));
|
|
42
43
|
await fs.chown(file, parent.uid, parent.gid);
|
|
43
44
|
}
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if(config?.hostConfig) {
|
|
51
|
-
const host=await json(config.hostConfig);
|
|
52
|
-
if(host.isolation==='isolated') launcher='ez';
|
|
53
|
-
}
|
|
45
|
+
// Retire the former `ez tools` footer once: replace the marked block with
|
|
46
|
+
// TOOLS_LINE and leave every other byte untouched. The name is kept because
|
|
47
|
+
// upgraders released before this change call it on the target package;
|
|
48
|
+
// remove it when no installed core predates the change.
|
|
49
|
+
export async function bindToolDiscovery(_home,workspace) {
|
|
50
|
+
workspace=await fs.realpath(workspace);
|
|
54
51
|
const start='<!-- ez tools: begin -->',end='<!-- ez tools: end -->';
|
|
55
|
-
const block=start+'\nUse `'+launcher+'`; run it with `--help` or `tools list --details` to discover installed capabilities, and read the matching skill when needed.\nKeep the channel responsive: during a foreground owner turn, prefer `ezenciel-agents-schedule create --now ...` when work can continue independently. A `[schedule ...]` turn executes its assigned work directly and must not schedule it again.\n'+end;
|
|
56
|
-
if(Buffer.byteLength(block)>600)throw Error('Tool discovery guidance exceeds the 600-byte core budget');
|
|
57
|
-
return {block,start,end};
|
|
58
|
-
}
|
|
59
|
-
export async function bindToolDiscovery(home,workspace) {
|
|
60
|
-
home=await fs.realpath(home);workspace=await fs.realpath(workspace);
|
|
61
|
-
const {block,start,end}=await toolDiscoveryBlock(home);
|
|
62
52
|
for(const name of ['AGENTS.md','AGENTS.override.md']) {
|
|
63
53
|
const file=path.join(workspace,name);
|
|
64
54
|
const stat=await fs.lstat(file).catch(e=>{if(e.code==='ENOENT')return null;throw e;});
|
|
65
|
-
if(!stat
|
|
66
|
-
if(
|
|
67
|
-
const prior=
|
|
55
|
+
if(!stat)continue;
|
|
56
|
+
if(!stat.isFile())throw Error('Tool instructions must be a regular file');
|
|
57
|
+
const prior=await fs.readFile(file,'utf8');
|
|
68
58
|
const from=prior.indexOf(start),to=prior.indexOf(end);
|
|
69
|
-
if(
|
|
70
|
-
|
|
71
|
-
|
|
59
|
+
if(from<0&&to<0)continue;
|
|
60
|
+
if((from<0)!==(to<0)||to<from||prior.indexOf(start,from+start.length)>=0||prior.indexOf(end,to+end.length)>=0)throw Error('Malformed tool discovery block');
|
|
61
|
+
const next=prior.slice(0,from)+(prior.includes(TOOLS_LINE)?'':TOOLS_LINE)+prior.slice(to+end.length);
|
|
72
62
|
const tmp=file+'.'+randomUUID()+'.tmp';
|
|
73
63
|
try {await fs.writeFile(tmp,next,{mode:0o600,flag:'wx'});await fs.rename(tmp,file);}finally{await fs.rm(tmp,{force:true});}
|
|
74
64
|
}
|
|
75
65
|
}
|
|
76
66
|
const invocationDirectory = home => path.join(home,'command-invocations');
|
|
67
|
+
// A lease pid is only meaningful inside the PID namespace that wrote it. The
|
|
68
|
+
// isolated broker writes leases from its container, which the host cannot probe.
|
|
69
|
+
const pidNamespace=(()=>{try{return readlinkSync('/proc/self/ns/pid');}catch{return undefined;}})();
|
|
77
70
|
async function activeInvocations(home) {
|
|
78
71
|
const directory=invocationDirectory(home),files=await fs.readdir(directory).catch(error=>{if(error.code==='ENOENT')return [];throw error;});
|
|
79
72
|
for(const file of files) {
|
|
@@ -81,8 +74,11 @@ async function activeInvocations(home) {
|
|
|
81
74
|
const lease=await json(path.join(directory,file)).catch(error=>{if(error.code==='ENOENT')return null;throw error;});
|
|
82
75
|
if(!lease)continue;
|
|
83
76
|
if(!Number.isSafeInteger(lease.pid)||lease.pid<1||typeof lease.container!=='string'||!lease.container)throw Error('Invalid command invocation lease; inspect before recovery');
|
|
84
|
-
|
|
85
|
-
|
|
77
|
+
const foreign=typeof lease.pidNamespace==='string'&&lease.pidNamespace!==pidNamespace;
|
|
78
|
+
if(!foreign) {try {process.kill(lease.pid,0);return true;} catch(error) {if(error.code!=='ESRCH'&&error.code!=='EPERM')throw error;}}
|
|
79
|
+
const state=await run(['container','inspect','--format','{{.State.Running}}',lease.container],{capture:true});
|
|
80
|
+
// Foreign leases: the call container's own state is the only liveness signal.
|
|
81
|
+
if(state.code===0&&foreign&&state.stdout.trim()==='true')return true;
|
|
86
82
|
if(state.code===0)throw Error(`Stale command invocation lease; inspect container ${lease.container} before recovery`);
|
|
87
83
|
if(!/No such (?:object|container)/i.test(state.stderr+state.stdout))throw Error('Unable to verify a stale command invocation lease; inspect Docker before recovery');
|
|
88
84
|
await fs.rm(path.join(directory,file));
|
|
@@ -91,7 +87,7 @@ async function activeInvocations(home) {
|
|
|
91
87
|
}
|
|
92
88
|
async function invocationLease(home,container) {
|
|
93
89
|
const directory=invocationDirectory(home);await privateDir(directory);
|
|
94
|
-
const file=path.join(directory,randomUUID()+'.json');await fs.writeFile(file,JSON.stringify({pid:process.pid,container}),{mode:0o600,flag:'wx'});
|
|
90
|
+
const file=path.join(directory,randomUUID()+'.json');await fs.writeFile(file,JSON.stringify({pid:process.pid,container,...(pidNamespace?{pidNamespace}:{})}),{mode:0o600,flag:'wx'});
|
|
95
91
|
await stewardOwned(file);
|
|
96
92
|
return async()=>{await fs.rm(file,{force:true});};
|
|
97
93
|
}
|
|
@@ -150,9 +146,11 @@ export async function snapshot(source) {
|
|
|
150
146
|
return {source,files,manifest,deployment,sharedRevisions,revision:`sha256:${digest.digest('hex')}`};
|
|
151
147
|
}
|
|
152
148
|
export function validate(m,d,files) {
|
|
153
|
-
keys(m,['schemaVersion','id','version','description','commands','skills']);
|
|
149
|
+
keys(m,['schemaVersion','id','version','description','commands','skills','example']);
|
|
154
150
|
if(m.schemaVersion!==1 || (typeof m.version!=='string' || !/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.test(m.version))) throw Error('Unsupported manifest version');
|
|
155
151
|
id(m.id); strings(m.skills);
|
|
152
|
+
// One representative call shown by `tools list --details`; usage stays in --help/skill.
|
|
153
|
+
if(m.example!==undefined && (typeof m.example!=='string' || m.example.length>160 || /[\0-\x1f\x7f]/.test(m.example) || !Object.hasOwn(m.commands||{},m.example.split(' ')[0]))) throw Error('Manifest example must be one line of at most 160 characters starting with a declared command');
|
|
156
154
|
keys(d,['schemaVersion','services','commands','exports',...(d.schemaVersion>=2?['secrets']:[]),...(d.schemaVersion===3?['sharedServices']:[])]);
|
|
157
155
|
if(![1,2,3].includes(d.schemaVersion) || !d.services || !d.commands) throw Error('Unsupported deployment descriptor');
|
|
158
156
|
for(const [name,s] of Object.entries(d.services)) {
|
|
@@ -397,11 +395,13 @@ export function run(argv,{capture=false,container,signal,stdin,onStdout,onStart,
|
|
|
397
395
|
child.once('close',async(code,childSignal)=>{process.off('SIGTERM',term);process.off('SIGINT',int);
|
|
398
396
|
clearTimeout(killTimer);
|
|
399
397
|
clearTimeout(timeout);signal?.removeEventListener('abort',term);
|
|
400
|
-
|
|
398
|
+
// --rm only removes a container that ran. Compose stopping between
|
|
399
|
+
// create and start (cancelled, signalled or failed) strands it Created.
|
|
400
|
+
if(container&&(cancelled||code!==0)) {
|
|
401
401
|
try {
|
|
402
402
|
await removeCommandContainer(container);
|
|
403
403
|
} catch(error) {
|
|
404
|
-
const uncertain=error instanceof Error?error:Error('
|
|
404
|
+
const uncertain=error instanceof Error?error:Error('Command container cleanup failed');
|
|
405
405
|
uncertain.containerCleanupUncertain=true;return reject(uncertain);
|
|
406
406
|
}
|
|
407
407
|
}
|
|
@@ -409,7 +409,7 @@ export function run(argv,{capture=false,container,signal,stdin,onStdout,onStart,
|
|
|
409
409
|
resolve({code:cancelled?130:code??(childSignal?130:1),stdout,stderr});});
|
|
410
410
|
});
|
|
411
411
|
}
|
|
412
|
-
// Compose --rm can race
|
|
412
|
+
// Compose --rm can race this cleanup. Confirm disappearance instead of treating
|
|
413
413
|
// Docker's in-progress removal as either failure or completed cleanup.
|
|
414
414
|
export async function removeCommandContainer(container,execute=run) {
|
|
415
415
|
const cleanup=await execute(['container','rm','--force',container],{capture:true});
|
|
@@ -422,7 +422,7 @@ export async function removeCommandContainer(container,execute=run) {
|
|
|
422
422
|
await new Promise(resolve=>setTimeout(resolve,100));
|
|
423
423
|
}
|
|
424
424
|
}
|
|
425
|
-
throw Error(`
|
|
425
|
+
throw Error(`Command container cleanup failed: ${cleanup.stderr||cleanup.stdout}`);
|
|
426
426
|
}
|
|
427
427
|
|
|
428
428
|
async function checked(args) {
|
|
@@ -530,7 +530,6 @@ export async function init(home,workspace,catalogFile,hostConfig,standalone=fals
|
|
|
530
530
|
if(standalone && hostConfig) throw Error('Standalone setup cannot bind a relay host config');
|
|
531
531
|
if(typeof home!=='string'||typeof workspace!=='string'||!path.isAbsolute(home)||!path.isAbsolute(workspace)||/[\r\n\0$:,]/.test(home+workspace)) throw Error('Explicit absolute home/workspace required');
|
|
532
532
|
workspace=await fs.realpath(workspace);await privateDir(home);home=await fs.realpath(home);if(hostConfig)hostConfig=await fs.realpath(hostConfig);
|
|
533
|
-
await toolDiscoveryBlock(home,hostConfig);
|
|
534
533
|
if(await fs.lstat(path.join(home,'registry.json')).catch(()=>null)) throw Error('Registry already exists; refusing replacement');
|
|
535
534
|
catalogFile=path.resolve(catalogFile||fileURLToPath(new URL('../../default-plugins.json',import.meta.url)));
|
|
536
535
|
const sources=await json(catalogFile);
|
|
@@ -563,7 +562,6 @@ export async function init(home,workspace,catalogFile,hostConfig,standalone=fals
|
|
|
563
562
|
agent.binDir=bin;agent.toolsHome=home;await atomic(hostConfig,host);
|
|
564
563
|
}
|
|
565
564
|
});
|
|
566
|
-
await bindToolDiscovery(home,workspace);
|
|
567
565
|
if(hostConfig && path.basename(hostConfig)==='host-executor.json') await (await import('../updates/binding.mjs')).bindUpdates(home,hostConfig);
|
|
568
566
|
return {ok:true,launcher:path.join(home,'bin','ez'),workspace};
|
|
569
567
|
}
|
|
@@ -651,12 +649,17 @@ export async function main(args) {
|
|
|
651
649
|
return emit({plugin:name,revision:record.revision,path:skill,content:await fs.readFile(file,'utf8')});
|
|
652
650
|
}
|
|
653
651
|
if(action==='list') {
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
652
|
+
// Self-sufficient index: aliases, purpose, skill and one call per plugin.
|
|
653
|
+
if(group==='tools' && args.length===1 && args[0]==='--details')return emit(Object.fromEntries(Object.entries(r.plugins).map(([name,p])=>{
|
|
654
|
+
const aliases=Object.keys(p.manifest.commands);
|
|
655
|
+
return [name,{
|
|
656
|
+
aliases,
|
|
657
|
+
purpose:typeof p.manifest.description==='string'?p.manifest.description.replace(/\s+/g,' ').trim().slice(0,200):'',
|
|
658
|
+
skills:p.manifest.skills.map(skill=>path.join(p.source,skill)),
|
|
659
|
+
skillReads:p.manifest.skills.map(skill=>['ez','tools','skill',name,skill]),
|
|
660
|
+
example:`ez ${p.manifest.example ?? `${aliases[0]} --help`}`,
|
|
661
|
+
}];
|
|
662
|
+
})));
|
|
660
663
|
if(args.length)throw Error('Use tools list [--details] or plugins list');
|
|
661
664
|
return emit(group==='tools'?r.commands:r.plugins);
|
|
662
665
|
}
|