@jc_stack/ez-agents 0.1.0-beta.34 → 0.1.0-beta.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +1 -0
- package/.env.example +5 -16
- package/AGENTS.md +17 -9
- package/CHANGELOG.md +67 -0
- package/Dockerfile +7 -9
- package/README.md +10 -13
- package/bin/ez +97 -0
- package/bin/ezenciel-agents-plugin-broker.mjs +16 -0
- package/bin/ezenciel-agents-provision-telegram.mjs +13 -0
- package/bin/ezenciel-agents.mjs +1 -1
- package/compose.yaml +40 -4
- package/docker/entrypoint.sh +1 -1
- package/docker/healthcheck.mjs +12 -4
- package/docker/run.ts +5 -6
- package/docker/smoke.mjs +28 -6
- package/docs/application-channel.md +32 -10
- package/docs/architecture/authority-boundaries.md +2 -2
- package/docs/channel-backend.md +4 -3
- package/docs/docker-runtime.md +79 -11
- package/docs/local-qa.md +2 -2
- package/docs/plugin-guide.md +202 -0
- package/docs/plugins.md +48 -2
- package/docs/releasing.md +13 -0
- package/docs/responsive-channels.md +1 -1
- package/docs/scheduling.md +4 -4
- package/docs/setup.md +11 -5
- package/docs/upgrades.md +46 -23
- package/package.json +5 -5
- package/scripts/smoke-scheduler.ts +10 -3
- package/scripts/smoke.ts +14 -11
- package/src/agent-guidance.ts +6 -9
- package/src/agent-install.ts +28 -7
- package/src/application-channel.ts +78 -7
- package/src/application-cli.ts +2 -2
- package/src/approval-cli.ts +14 -7
- package/src/approval.ts +27 -37
- package/src/channel-backend.ts +2 -3
- package/src/config.ts +12 -27
- package/src/control-state.ts +175 -8
- package/src/conversation-menu.ts +6 -17
- package/src/delivery-socket-client.d.mts +4 -0
- package/src/delivery-socket-client.mjs +104 -0
- package/src/delivery-socket.ts +354 -0
- package/src/execution-authority.ts +21 -5
- package/src/executor.ts +20 -23
- package/src/files.ts +38 -11
- package/src/host-executor.ts +28 -14
- package/src/inbox.ts +15 -49
- package/src/index.ts +95 -52
- package/src/install-tools.mjs +97 -22
- package/src/ledger-endpoint.d.mts +2 -0
- package/src/ledger-endpoint.mjs +17 -0
- package/src/menu.ts +7 -2
- package/src/message-history.ts +13 -18
- package/src/message.ts +35 -18
- package/src/plugin-broker.mjs +396 -0
- package/src/plugins/connection.mjs +2 -2
- package/src/plugins/manager.mjs +132 -23
- package/src/plugins/native-tasks.mjs +2 -2
- package/src/prompt-suffix.ts +12 -0
- package/src/react.ts +6 -4
- package/src/runs.ts +156 -243
- package/src/schedule-cli.ts +29 -10
- package/src/scheduled-tasks.ts +53 -10
- package/src/scheduler.ts +20 -7
- package/src/service.ts +3 -3
- package/src/software-status.ts +36 -4
- package/src/task-executor.ts +2 -2
- package/src/task-workspace.ts +4 -3
- package/src/tasks.ts +1 -1
- package/src/telegram-provisioning-cli.ts +22 -0
- package/src/telegram-provisioning.ts +214 -0
- package/src/updates/control.mjs +77 -7
- package/src/updates/runtime.mjs +28 -8
- package/src/updates/status.mjs +25 -3
- package/src/updates/supervisor.mjs +15 -16
- package/src/version.ts +20 -0
- package/src/workspace.ts +3 -9
- package/templates/failure-review.md +1 -1
- package/templates/maintainer-purpose.md +1 -1
- package/test/agent-guidance.test.ts +14 -80
- package/test/agent-install.test.ts +14 -0
- package/test/ai.test.ts +36 -0
- package/test/application-channel.test.ts +61 -13
- package/test/application-only.test.ts +14 -13
- package/test/channel-backend.test.ts +3 -7
- package/test/channel-delivery.test.ts +4 -1
- package/test/channel-owner.test.ts +6 -3
- package/test/codex-context.test.ts +2 -2
- package/test/config.test.ts +38 -43
- package/test/control-state.test.ts +83 -4
- package/test/conversation-menu.test.ts +2 -4
- package/test/delivery-transport.test.ts +72 -0
- package/test/docker-runtime.test.ts +27 -14
- package/test/engine-handoff.test.ts +7 -6
- package/test/event-sources.test.ts +2 -2
- package/test/execution-authority.test.ts +9 -9
- package/test/executor.test.ts +10 -11
- package/test/failure.test.ts +12 -5
- package/test/files.test.ts +2 -2
- package/test/helpers/ledger.ts +18 -0
- package/test/host-executor.test.ts +74 -0
- package/test/inbox.test.ts +8 -10
- package/test/install-config.test.ts +1 -0
- package/test/install-tools.test.mjs +74 -7
- package/test/intake-relay.test.ts +58 -11
- package/test/message-history.test.ts +39 -47
- package/test/native-tasks.test.ts +1 -1
- package/test/plugin-broker.test.mjs +172 -0
- package/test/plugin-manager.test.mjs +119 -2
- package/test/reaction.test.ts +4 -0
- package/test/relay.test.ts +5 -7
- package/test/repair-policy.test.ts +1 -4
- package/test/schedule-cli.test.ts +14 -9
- package/test/scheduled-tasks.test.ts +29 -6
- package/test/scheduler-host.test.ts +5 -5
- package/test/scheduler-relay.test.ts +15 -9
- package/test/security.test.ts +6 -3
- package/test/software-status.test.ts +21 -3
- package/test/telegram-provisioning.test.ts +160 -0
- package/test/updates.test.mjs +93 -13
- package/test/workspace.test.ts +4 -1
- package/bin/ezenciel-agents-watch.mjs +0 -8
- package/compose.workforce-watch.yaml +0 -33
- package/docker/recovery.ts +0 -11
- package/docs/pagerduty.md +0 -42
- package/docs/workforce-watch.md +0 -101
- package/src/pagerduty.ts +0 -109
- package/src/reply-context.ts +0 -43
- package/src/update-attention.ts +0 -18
- package/src/workforce-watch-cli.ts +0 -14
- package/src/workforce-watch.ts +0 -155
- package/templates/agent/AGENTS.md +0 -5
- package/templates/agent-guidance.md +0 -17
- package/test/pagerduty.test.ts +0 -104
- package/test/update-attention.test.ts +0 -20
- package/test/workforce-watch.test.ts +0 -180
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import test from 'node:test'
|
|
3
|
+
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { provisionTelegramBot } from '../src/telegram-provisioning.js'
|
|
7
|
+
|
|
8
|
+
const token = '123456:abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMN'
|
|
9
|
+
|
|
10
|
+
const fixture = async () => {
|
|
11
|
+
const root = await mkdtemp(path.join(tmpdir(), 'ez-telegram-provisioning-'))
|
|
12
|
+
const composeFile = path.join(root, 'compose.yml')
|
|
13
|
+
const configFile = path.join(root, 'telegram-provisioning.json')
|
|
14
|
+
const relayEnvFile = path.join(root, 'secrets', 'relay.env')
|
|
15
|
+
const overrideFile = path.join(root, 'telegram.compose.yml')
|
|
16
|
+
const deploymentEnvFile = path.join(root, 'docker.env')
|
|
17
|
+
await writeFile(composeFile, 'services: {}\n')
|
|
18
|
+
await writeFile(deploymentEnvFile, "EZ_RELAY_IMAGE='ezenciel-agents:0.1.0-beta.36.rc.4'\n", { mode: 0o600 })
|
|
19
|
+
await writeFile(configFile, JSON.stringify({
|
|
20
|
+
version: 1,
|
|
21
|
+
composeFile,
|
|
22
|
+
projectDirectory: root,
|
|
23
|
+
projectName: 'tenant-aifit',
|
|
24
|
+
service: 'relay',
|
|
25
|
+
relayEnvFile,
|
|
26
|
+
overrideFile,
|
|
27
|
+
}), { mode: 0o600 })
|
|
28
|
+
return { root, composeFile, configFile, relayEnvFile, overrideFile, deploymentEnvFile }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
test('Telegram provisioning writes only a private relay secret and restarts the exact relay', async (t) => {
|
|
32
|
+
const value = await fixture()
|
|
33
|
+
t.after(async () => { await import('node:fs/promises').then(({ rm }) => rm(value.root, { recursive: true, force: true })) })
|
|
34
|
+
const calls: string[][] = []
|
|
35
|
+
const environments: NodeJS.ProcessEnv[] = []
|
|
36
|
+
const priorImage = process.env.EZ_RELAY_IMAGE
|
|
37
|
+
process.env.EZ_RELAY_IMAGE = 'stale:shell-export'
|
|
38
|
+
try {
|
|
39
|
+
await provisionTelegramBot(value.configFile, token, async (args, environment) => { calls.push(args); environments.push(environment) })
|
|
40
|
+
} finally {
|
|
41
|
+
if (priorImage === undefined) delete process.env.EZ_RELAY_IMAGE; else process.env.EZ_RELAY_IMAGE = priorImage
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// The deployment's docker.env is the only source for its runtime values:
|
|
45
|
+
// shell exports never override the installed image or mounts.
|
|
46
|
+
assert.equal('EZ_RELAY_IMAGE' in environments[0], false)
|
|
47
|
+
assert.equal('TELEGRAM_BOT_TOKEN' in environments[0], false)
|
|
48
|
+
assert.equal(environments[0].PATH, process.env.PATH)
|
|
49
|
+
assert.equal(await readFile(value.relayEnvFile, 'utf8'), `TELEGRAM_BOT_TOKEN=${token}\n`)
|
|
50
|
+
assert.equal((await stat(value.relayEnvFile)).mode & 0o777, 0o600)
|
|
51
|
+
const override = await readFile(value.overrideFile, 'utf8')
|
|
52
|
+
// The installed deployment supplies the relay image; the override never sets one.
|
|
53
|
+
assert.ok(!override.includes('image:'))
|
|
54
|
+
assert.match(override, /relay_env/)
|
|
55
|
+
assert.match(override, /EZ_TELEGRAM_ENABLED: "true"/)
|
|
56
|
+
assert.ok(!override.includes(token))
|
|
57
|
+
assert.equal((await stat(value.overrideFile)).mode & 0o777, 0o600)
|
|
58
|
+
assert.deepEqual(calls, [[
|
|
59
|
+
'--project-directory', value.root,
|
|
60
|
+
'--project-name', 'tenant-aifit',
|
|
61
|
+
'--env-file', value.deploymentEnvFile,
|
|
62
|
+
'-f', value.composeFile,
|
|
63
|
+
'-f', value.overrideFile,
|
|
64
|
+
'up', '-d', '--wait', 'relay',
|
|
65
|
+
]])
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
test('provisioning rejects a self-set image and custom deployments without a Docker env file still start', async (t) => {
|
|
69
|
+
const value = await fixture()
|
|
70
|
+
t.after(async () => { await rm(value.root, { recursive: true, force: true }) })
|
|
71
|
+
const configured = JSON.parse(await readFile(value.configFile, 'utf8'))
|
|
72
|
+
await writeFile(value.configFile, JSON.stringify({ ...configured, image: 'eizenciel-agents:0.1.0-beta.36.rc.3' }))
|
|
73
|
+
await assert.rejects(provisionTelegramBot(value.configFile, token, async () => {}), /Invalid Telegram provisioning configuration/)
|
|
74
|
+
await writeFile(value.configFile, JSON.stringify(configured))
|
|
75
|
+
await rm(value.deploymentEnvFile)
|
|
76
|
+
const calls: string[][] = []
|
|
77
|
+
await provisionTelegramBot(value.configFile, token, async (args) => { calls.push(args) })
|
|
78
|
+
assert.ok(!calls[0].includes('--env-file'))
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
test('provisioning fails closed when the deployment docker.env has no installed image', async (t) => {
|
|
82
|
+
const value = await fixture()
|
|
83
|
+
t.after(async () => { await rm(value.root, { recursive: true, force: true }) })
|
|
84
|
+
await writeFile(value.deploymentEnvFile, "EZ_AGENT_WORKSPACE='/private/agents/tenant/mind'\n", { mode: 0o600 })
|
|
85
|
+
await assert.rejects(provisionTelegramBot(value.configFile, token, async () => {}), /must define EZ_RELAY_IMAGE/)
|
|
86
|
+
await assert.rejects(readFile(value.relayEnvFile, 'utf8'), { code: 'ENOENT' })
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
test('Telegram provisioning restores the botless relay when startup fails', async (t) => {
|
|
90
|
+
const value = await fixture()
|
|
91
|
+
t.after(async () => { await import('node:fs/promises').then(({ rm }) => rm(value.root, { recursive: true, force: true })) })
|
|
92
|
+
const calls: string[][] = []
|
|
93
|
+
await assert.rejects(
|
|
94
|
+
provisionTelegramBot(value.configFile, token, async (args) => {
|
|
95
|
+
calls.push(args)
|
|
96
|
+
if (calls.length === 1) throw new Error('Bot rejected')
|
|
97
|
+
}),
|
|
98
|
+
/Bot rejected/,
|
|
99
|
+
)
|
|
100
|
+
await assert.rejects(readFile(value.relayEnvFile, 'utf8'), { code: 'ENOENT' })
|
|
101
|
+
await assert.rejects(readFile(value.overrideFile, 'utf8'), { code: 'ENOENT' })
|
|
102
|
+
assert.equal(calls.length, 2)
|
|
103
|
+
assert.ok(!calls[1].includes(value.overrideFile))
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test('Telegram provisioning rejects invalid tokens and configs before touching the deployment', async (t) => {
|
|
107
|
+
const value = await fixture()
|
|
108
|
+
t.after(async () => { await import('node:fs/promises').then(({ rm }) => rm(value.root, { recursive: true, force: true })) })
|
|
109
|
+
await assert.rejects(provisionTelegramBot(value.configFile, 'not-a-token', async () => {}), /Invalid Telegram bot token/)
|
|
110
|
+
await assert.rejects(provisionTelegramBot(value.configFile, `${'1'.repeat(600)}:x`, async () => {}), /Invalid Telegram bot token/)
|
|
111
|
+
await assert.rejects(readFile(value.relayEnvFile, 'utf8'), { code: 'ENOENT' })
|
|
112
|
+
await writeFile(value.configFile, JSON.stringify({ version: 1 }))
|
|
113
|
+
await assert.rejects(provisionTelegramBot(value.configFile, token, async () => {}), /Invalid Telegram provisioning configuration/)
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
test('Telegram rotation failure restores both files and restarts with the previous override', async (t) => {
|
|
117
|
+
const value = await fixture()
|
|
118
|
+
t.after(async () => { await import('node:fs/promises').then(({ rm }) => rm(value.root, { recursive: true, force: true })) })
|
|
119
|
+
const { mkdir } = await import('node:fs/promises')
|
|
120
|
+
await mkdir(path.dirname(value.relayEnvFile), { recursive: true })
|
|
121
|
+
await writeFile(value.relayEnvFile, 'TELEGRAM_BOT_TOKEN=111111:old-old-old-old-old-old-old-old\n', { mode: 0o600 })
|
|
122
|
+
await writeFile(value.overrideFile, 'previous-override\n')
|
|
123
|
+
const calls: string[][] = []
|
|
124
|
+
await assert.rejects(
|
|
125
|
+
provisionTelegramBot(value.configFile, token, async (args) => {
|
|
126
|
+
calls.push(args)
|
|
127
|
+
if (calls.length === 1) throw new Error('Bot rejected rotation')
|
|
128
|
+
}),
|
|
129
|
+
/Bot rejected rotation/,
|
|
130
|
+
)
|
|
131
|
+
assert.equal(await readFile(value.relayEnvFile, 'utf8'), 'TELEGRAM_BOT_TOKEN=111111:old-old-old-old-old-old-old-old\n')
|
|
132
|
+
assert.equal(await readFile(value.overrideFile, 'utf8'), 'previous-override\n')
|
|
133
|
+
assert.equal(calls.length, 2)
|
|
134
|
+
assert.ok(calls[1].includes(value.overrideFile))
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test('Telegram provisioning refuses a live lock but recovers a stale one', async (t) => {
|
|
138
|
+
const value = await fixture()
|
|
139
|
+
t.after(async () => { await import('node:fs/promises').then(({ rm }) => rm(value.root, { recursive: true, force: true })) })
|
|
140
|
+
const calls: string[][] = []
|
|
141
|
+
await writeFile(path.join(value.root, 'telegram-provisioning.lock'), JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() }))
|
|
142
|
+
await assert.rejects(provisionTelegramBot(value.configFile, token, async (args) => { calls.push(args) }), /already in progress/)
|
|
143
|
+
assert.equal(calls.length, 0)
|
|
144
|
+
await writeFile(path.join(value.root, 'telegram-provisioning.lock'), JSON.stringify({ pid: 2147483647, startedAt: '2020-01-01T00:00:00.000Z' }))
|
|
145
|
+
await provisionTelegramBot(value.configFile, token, async (args) => { calls.push(args) })
|
|
146
|
+
assert.equal(calls.length, 1)
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
test('Telegram provisioning CLI rejects TTY input and oversized stdin', async () => {
|
|
150
|
+
const { runTelegramProvisioningCli } = await import('../src/telegram-provisioning-cli.js')
|
|
151
|
+
await assert.rejects(runTelegramProvisioningCli(['--config', 'x'], { isTTY: true } as never), /stdin/)
|
|
152
|
+
let destroyed = false
|
|
153
|
+
const oversized = {
|
|
154
|
+
isTTY: false,
|
|
155
|
+
destroy() { destroyed = true },
|
|
156
|
+
async *[Symbol.asyncIterator]() { yield 'x'.repeat(600) },
|
|
157
|
+
}
|
|
158
|
+
await assert.rejects(runTelegramProvisioningCli(['--config', 'x'], oversized as never), /too long/)
|
|
159
|
+
assert.equal(destroyed, true)
|
|
160
|
+
})
|
package/test/updates.test.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { gzipSync } from 'node:zlib';
|
|
|
7
7
|
import { spawn, execFile } from 'node:child_process';
|
|
8
8
|
import { promisify } from 'node:util';
|
|
9
9
|
import { extract, digest, version, newer, compatible } from '../src/updates/artifact.mjs';
|
|
10
|
-
import { prepare, submit, command, read, jobPath, eligibility, jobs } from '../src/updates/control.mjs';
|
|
10
|
+
import { prepare, submit, command, read, jobPath, eligibility, jobs, cleanupStaleBackups, reviewedPluginDeploymentMigration } from '../src/updates/control.mjs';
|
|
11
11
|
import { perform, environment, packageManager } from '../src/updates/runtime.mjs';
|
|
12
12
|
import { atomic, snapshot, compose, prepareCommand } from '../src/plugins/manager.mjs';
|
|
13
13
|
import { bindUpdates } from '../src/updates/binding.mjs';
|
|
@@ -19,7 +19,7 @@ test('supervisor forwards only provider keys declared by the agent installation'
|
|
|
19
19
|
const host={agents:[{codexProviders:[{envKey:'OPENROUTER_API_KEY'}]}]};
|
|
20
20
|
assert.deepEqual(providerEnvironment(host,{OPENROUTER_API_KEY:'provider-secret',TELEGRAM_BOT_TOKEN:'relay-secret'}),{OPENROUTER_API_KEY:'provider-secret'});
|
|
21
21
|
assert.throws(()=>providerEnvironment({agents:[{codexProviders:[{envKey:'bad-key'}]}]},{}),/environment key/);
|
|
22
|
-
for(const envKey of ['PATH','HOME','CODEX_HOME','NODE_OPTIONS','EZ_CONTROL_DIR','TELEGRAM_BOT_TOKEN'
|
|
22
|
+
for(const envKey of ['PATH','HOME','CODEX_HOME','NODE_OPTIONS','EZ_CONTROL_DIR','TELEGRAM_BOT_TOKEN'])
|
|
23
23
|
assert.throws(()=>providerEnvironment({agents:[{codexProviders:[{envKey}]}]},{}),/Reserved Codex provider environment key/);
|
|
24
24
|
});
|
|
25
25
|
const contract=kind=>({protocol:1,kind,stateSchema:1,mainProtocol:1});
|
|
@@ -107,20 +107,39 @@ test('relay healthcheck emits bounded predicate evidence without state contents'
|
|
|
107
107
|
});
|
|
108
108
|
test('plugin status verifies images and reports stopped, mismatched and unreachable runtimes honestly',async t=>{
|
|
109
109
|
const f=await fixture(t,'plugin'),id='a'.repeat(64),image='sha256:'+'b'.repeat(64);
|
|
110
|
-
for(const mode of ['running','ndjson','stopped','missing','mismatched','offline']) {
|
|
110
|
+
for(const mode of ['running','ndjson','warned','stopped','missing','mismatched','offline']) {
|
|
111
111
|
const calls=[];
|
|
112
112
|
const run=async(c,args)=>{
|
|
113
113
|
calls.push([c,...args]);assert.equal(c,'docker');
|
|
114
114
|
if(mode==='offline')throw Error('synthetic secret must not escape');
|
|
115
|
-
if(args.includes('ps')) {const rows=mode==='missing'?[]:[{Service:'sample',State:mode==='stopped'?'exited':'running',Health:'healthy',ID:id}];
|
|
115
|
+
if(args.includes('ps')) {const rows=mode==='missing'?[]:[{Service:'sample',State:mode==='stopped'?'exited':'running',Health:'healthy',ID:id}];const lines=mode==='ndjson'?rows.map(r=>JSON.stringify(r)).join('\n'):JSON.stringify(rows);return mode==='warned'?'time="now" level=warning msg="the attribute version is obsolete"\n'+rows.map(r=>JSON.stringify(r)).join('\n'):lines;}
|
|
116
116
|
assert(args.includes('inspect'));return mode==='mismatched'&&args[0]==='image'?'sha256:'+'c'.repeat(64):image;
|
|
117
117
|
};
|
|
118
118
|
const s=await runtimeStatus(f.home,run),p=s.plugins[0];
|
|
119
|
-
assert.equal(p.installedVersion,'0.1.0');assert.equal(p.runningVersion,['running','ndjson'].includes(mode)?'0.1.0':null);
|
|
119
|
+
assert.equal(p.installedVersion,'0.1.0');assert.equal(p.runningVersion,['running','ndjson','warned'].includes(mode)?'0.1.0':null);
|
|
120
120
|
assert.equal(p.state,mode==='offline'?'unknown':['stopped','missing'].includes(mode)?'stopped':'running');
|
|
121
121
|
assert(!JSON.stringify(s).includes('synthetic secret'));assert(calls.every(c=>!c.includes('exec')&&!c.includes('start')&&!c.includes('up')));
|
|
122
122
|
}
|
|
123
123
|
});
|
|
124
|
+
test('isolated plugin status uses bundled Compose v1 and inspect readback',async t=>{
|
|
125
|
+
const f=await fixture(t,'plugin'),id='a'.repeat(64),image='sha256:'+'b'.repeat(64),previous=process.env.EZ_DOCKER_COMPOSE;
|
|
126
|
+
process.env.EZ_DOCKER_COMPOSE='standalone';
|
|
127
|
+
try {
|
|
128
|
+
const calls=[],run=async(c,args)=>{
|
|
129
|
+
calls.push([c,...args]);assert.equal(c,'docker');
|
|
130
|
+
if(args.at(-1)==='-q')return id;
|
|
131
|
+
if(args[0]==='inspect'&&args.length===2)return JSON.stringify([{Config:{Labels:{'com.docker.compose.service':'sample'}},State:{Status:'running',Health:{Status:'healthy'}}}]);
|
|
132
|
+
if(args[0]==='inspect')return image;
|
|
133
|
+
if(args[0]==='image')return image;
|
|
134
|
+
throw Error('unexpected status command');
|
|
135
|
+
};
|
|
136
|
+
const p=(await runtimeStatus(f.home,run)).plugins[0];
|
|
137
|
+
assert.equal(p.state,'running');assert.equal(p.runningVersion,'0.1.0');
|
|
138
|
+
assert.deepEqual(calls[0].slice(-3),['ps','--all','-q']);
|
|
139
|
+
} finally {
|
|
140
|
+
if(previous===undefined)delete process.env.EZ_DOCKER_COMPOSE;else process.env.EZ_DOCKER_COMPOSE=previous;
|
|
141
|
+
}
|
|
142
|
+
});
|
|
124
143
|
test('SemVer ordering and compatibility reject ranges, malformed values and downgrades',()=>{
|
|
125
144
|
for(const s of ['latest','../1','1.0','01.0.0','1.0.0-01'])assert.throws(()=>version(s));
|
|
126
145
|
assert(newer('0.1.0-beta.10','0.1.0-beta.2'));assert(newer('0.1.0','0.1.0-beta.10'));assert(!newer('0.1.0-beta.2','0.1.0'));assert(!newer('1.0.0','1.0.0'));
|
|
@@ -154,6 +173,63 @@ test('identity, state migration, deployment changes and stale candidates fail be
|
|
|
154
173
|
await mutate({ezRelease:contract('main'),version:'0.0.1'});await assert.rejects(prepare(f.home,'main',{file:await f.pack()}),/newer/);
|
|
155
174
|
await mutate({version:'0.1.1'});await fs.writeFile(path.join(f.source,'compose.yaml'),'privileged: true');await assert.rejects(prepare(f.home,'main',{file:await f.pack()}),/deployment/);
|
|
156
175
|
});
|
|
176
|
+
test('legacy runtime deployment migration compares effective Compose and preserves explicit bindings',async t=>{
|
|
177
|
+
const f=await fixture(t),oldCompose=`services:
|
|
178
|
+
relay:
|
|
179
|
+
environment:
|
|
180
|
+
EZ_AGENT_PURPOSE_FILE: /run/agent-purpose.md
|
|
181
|
+
EZ_EXECUTOR_TRANSPORT: \${EZ_EXECUTOR_TRANSPORT:-host}
|
|
182
|
+
configs:
|
|
183
|
+
agent_purpose:
|
|
184
|
+
file: \${EZ_AGENT_PURPOSE_FILE:-./templates/agent/SOUL.md}
|
|
185
|
+
`,nextCompose=oldCompose.replace(' EZ_EXECUTOR_TRANSPORT:', ' EZ_ISOLATION: ${EZ_ISOLATION:-}\n EZ_EXECUTOR_TRANSPORT:').replace('./templates/agent/SOUL.md','./templates/agent-purpose.md');
|
|
186
|
+
await fs.writeFile(path.join(f.old,'compose.yaml'),oldCompose);await fs.writeFile(path.join(f.source,'compose.yaml'),nextCompose);
|
|
187
|
+
await fs.writeFile(path.join(f.config.deploymentDir,'docker.env'),`COMPOSE_FILE='${f.old}/compose.yaml'
|
|
188
|
+
COMPOSE_PROJECT_NAME='ez-agent-fixture'
|
|
189
|
+
EZ_AGENT_PURPOSE_FILE='${path.join(f.config.deploymentDir,'purpose.md')}'
|
|
190
|
+
EZ_EXECUTOR_TRANSPORT='host'
|
|
191
|
+
`);
|
|
192
|
+
const job=await prepare(f.home,'main',{file:await f.pack()});assert.equal(job.deploymentMigration.id,'legacy-runtime-v1');
|
|
193
|
+
await fs.writeFile(path.join(f.config.deploymentDir,'docker.env'),`COMPOSE_FILE='${f.old}/compose.yaml'
|
|
194
|
+
COMPOSE_PROJECT_NAME='ez-agent-fixture'
|
|
195
|
+
EZ_EXECUTOR_TRANSPORT='host'
|
|
196
|
+
`);await assert.rejects(eligibility(f.home,'main',f.source,false),/deployment/);
|
|
197
|
+
});
|
|
198
|
+
test('legacy runtime migration admits homes without docker.env when purpose fallback is already bound',async t=>{
|
|
199
|
+
const f=await fixture(t),oldCompose=`services:
|
|
200
|
+
relay:
|
|
201
|
+
environment:
|
|
202
|
+
EZ_AGENT_PURPOSE_FILE: /run/agent-purpose.md
|
|
203
|
+
EZ_EXECUTOR_TRANSPORT: \${EZ_EXECUTOR_TRANSPORT:-host}
|
|
204
|
+
configs:
|
|
205
|
+
agent_purpose:
|
|
206
|
+
file: \${EZ_AGENT_PURPOSE_FILE:-./templates/agent-purpose.md}
|
|
207
|
+
`,nextCompose=oldCompose.replace(' EZ_EXECUTOR_TRANSPORT:', ' EZ_ISOLATION: \${EZ_ISOLATION:-}\n EZ_EXECUTOR_TRANSPORT:');
|
|
208
|
+
await fs.writeFile(path.join(f.old,'compose.yaml'),oldCompose);await fs.writeFile(path.join(f.source,'compose.yaml'),nextCompose);
|
|
209
|
+
await fs.rm(path.join(f.config.deploymentDir,'docker.env'));
|
|
210
|
+
const job=await prepare(f.home,'main',{file:await f.pack()});assert.equal(job.deploymentMigration.id,'legacy-runtime-v1');
|
|
211
|
+
});
|
|
212
|
+
test('compatibility bridge hands off to the published beta.34 release',async t=>{
|
|
213
|
+
const f=await fixture(t),prior=await read(path.join(f.old,'package.json')),candidate=await read(path.join(f.source,'package.json'));
|
|
214
|
+
await atomic(path.join(f.old,'package.json'),{...prior,version:'0.1.0-beta.34.compat.1'});
|
|
215
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.34'});
|
|
216
|
+
await assert.doesNotReject(eligibility(f.home,'main',f.source,false));
|
|
217
|
+
});
|
|
218
|
+
test('compatibility bridge hands off any matching canonical beta release',async t=>{
|
|
219
|
+
const f=await fixture(t),prior=await read(path.join(f.old,'package.json')),candidate=await read(path.join(f.source,'package.json'));
|
|
220
|
+
await atomic(path.join(f.old,'package.json'),{...prior,version:'0.1.0-beta.35.compat.1'});
|
|
221
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.35'});
|
|
222
|
+
await assert.doesNotReject(eligibility(f.home,'main',f.source,false));
|
|
223
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.34'});
|
|
224
|
+
await assert.rejects(eligibility(f.home,'main',f.source,false),/newer/);
|
|
225
|
+
});
|
|
226
|
+
test('reviewed Library beta14 command migration is exact and fail-closed',()=>{
|
|
227
|
+
const manifest=(version,document)=>({schemaVersion:1,id:'library',version,description:'library',commands:{library:{executable:'bin/ez-library.mjs',args:[]},'library-query':{executable:'bin/ez-library.mjs',args:[],channelQuery:true},...(document?{'library-document':{executable:'bin/ez-library.mjs',args:[],channelQuery:true}}:{})},skills:['skills/library/SKILL.md']});
|
|
228
|
+
const deployment=(query,document)=>({schemaVersion:3,services:{library:{buildTarget:'runtime',volumes:{data:'/state'},healthcheck:['node','/app/bin/ez-library.mjs','--version'],memoryMiB:4096,cpus:2}},commands:{'library-query':{service:'library',argv:['node','/app/bin/ez-library.mjs',query]},...(document?{'library-document':{service:'library',argv:['node','/app/bin/ez-library.mjs','document']}}:{})},sharedServices:{}});
|
|
229
|
+
const old={manifest:manifest('0.1.0-beta.13.qa.4',true),deployment:deployment('query-context',true)},next={manifest:manifest('0.1.0-beta.14',false),deployment:deployment('search',false)};
|
|
230
|
+
assert.equal(reviewedPluginDeploymentMigration('library',old,next).id,'library-query-contract-v1');
|
|
231
|
+
const unsafe=structuredClone(next);unsafe.deployment.services.library.cpus=4;assert.equal(reviewedPluginDeploymentMigration('library',old,unsafe),undefined);
|
|
232
|
+
});
|
|
157
233
|
test('main transaction stages before stopping, pins rollback image, preserves state and rebinds root',async t=>{
|
|
158
234
|
const f=await fixture(t),job=await queued(f),r=runtime(f);
|
|
159
235
|
await fs.writeFile(path.join(f.agent.workspace,'memory.md'),'retain me');
|
|
@@ -167,6 +243,13 @@ test('main transaction stages before stopping, pins rollback image, preserves st
|
|
|
167
243
|
const status=await command(f.home,['status']);assert(!JSON.stringify(status).includes('private-test-token'));assert(!('rollback'in status.jobs[0]));
|
|
168
244
|
assert.equal((await fs.stat(path.join(jobPath(f.home,job.id),'job.json'))).mode&0o777,0o600);
|
|
169
245
|
});
|
|
246
|
+
test('update cleanup keeps the newest backup per target and ignores unfinished jobs',async t=>{
|
|
247
|
+
const f=await fixture(t),older='11111111-1111-1111-1111-111111111111',latest='22222222-2222-2222-2222-222222222222',queued='33333333-3333-3333-3333-333333333333',rolledBack='44444444-4444-4444-4444-444444444444';
|
|
248
|
+
const seed=async(id,target,status,endedAt)=>{const dir=jobPath(f.home,id);await fs.mkdir(path.join(dir,'backup'),{recursive:true});await fs.writeFile(path.join(dir,'backup','marker'),'private');await atomic(path.join(dir,'job.json'),{id,target,status,createdAt:endedAt,endedAt});};
|
|
249
|
+
await seed(older,'main','completed','2026-01-01T00:00:00.000Z');await seed(latest,'main','completed','2026-01-02T00:00:00.000Z');await seed(queued,'sample','queued','2026-01-03T00:00:00.000Z');await seed(rolledBack,'main','rolled-back','2026-01-03T00:00:00.000Z');
|
|
250
|
+
const result=await cleanupStaleBackups(f.home);assert.deepEqual(result.kept,[latest]);assert.deepEqual(result.removed.sort(),[older,rolledBack].sort());
|
|
251
|
+
await assert.rejects(fs.access(path.join(jobPath(f.home,older),'backup')));await fs.access(path.join(jobPath(f.home,latest),'backup'));await fs.access(path.join(jobPath(f.home,queued),'backup'));await assert.rejects(fs.access(path.join(jobPath(f.home,rolledBack),'backup')));
|
|
252
|
+
});
|
|
170
253
|
test('failed preparation never stops runtime; failed activation rolls back code without rewinding state',async t=>{
|
|
171
254
|
for(const stage of ['build','health']) {
|
|
172
255
|
const f=await fixture(t),job=await queued(f),r=runtime(f,{fail:(_c,a)=>stage==='build'?a[0]==='build':a.includes('up')});
|
|
@@ -303,9 +386,10 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
|
|
|
303
386
|
await fs.writeFile(wrapper,`import {supervise} from ${JSON.stringify(module)};const a=new AbortController();process.on('SIGTERM',()=>a.abort());await supervise(${JSON.stringify(f.config.deploymentDir)},a.signal,{discover:async()=>{${provider==='pnpm' ? "throw Error('Synthetic discovery failure')" : 'return []'}}});`);
|
|
304
387
|
const start=()=>{const p=spawn(process.execPath,[wrapper],{env:{...process.env,PATH:fake},stdio:['ignore','pipe','pipe']});let output='';p.stdout.on('data',b=>output+=b);p.stderr.on('data',b=>output+=b);return {p,output:()=>output};};
|
|
305
388
|
const wait=async fn=>{for(let i=0;i<150;i++){const result=await fn();if(result)return result;await new Promise(r=>setTimeout(r,100));}throw Error('Timed out');};
|
|
306
|
-
const first=start();t.after(()=>{first.p.kill('SIGTERM');});
|
|
389
|
+
const first=start(),firstClosed=new Promise(resolve=>first.p.once('close',resolve));t.after(()=>{first.p.kill('SIGTERM');});
|
|
307
390
|
const heartbeat=()=>read(path.join(f.agent.controlDir,'host-executor/heartbeat.json')).catch(()=>null);
|
|
308
391
|
const oldBeat=await wait(heartbeat);
|
|
392
|
+
if(provider==='pnpm')await wait(()=>first.output().includes('Update discovery failed; host remains running.'));
|
|
309
393
|
const running=path.join(f.agent.controlDir,'host-executor/r_request.running.json');await fs.writeFile(running,'{}');
|
|
310
394
|
const job=await prepare(f.home,'main',{file:await f.pack()});
|
|
311
395
|
const requester=await exec(process.execPath,['--input-type=module','-e',`import {submit} from ${JSON.stringify(new URL('../src/updates/control.mjs',import.meta.url).href)};console.log(JSON.stringify(await submit(${JSON.stringify(f.home)},${JSON.stringify(job.id)},false)));`]);
|
|
@@ -313,14 +397,10 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
|
|
|
313
397
|
await new Promise(r=>setTimeout(r,1800));assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).status,'queued');
|
|
314
398
|
await fs.rm(running);
|
|
315
399
|
await wait(async()=>{const j=await read(path.join(jobPath(f.home,job.id),'job.json'));if(j.status==='failed'||j.status==='rolled-back')throw Error(JSON.stringify(j)+first.output());return j.status==='completed';});
|
|
316
|
-
const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert(first.
|
|
400
|
+
const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert.equal(await firstClosed,0,first.output());
|
|
317
401
|
if(provider==='pnpm'){assert.match(first.output(),/Update discovery failed; host remains running/);assert.doesNotMatch(first.output(),/Synthetic discovery failure/);}
|
|
318
|
-
//
|
|
319
|
-
await
|
|
320
|
-
try{return (await read(path.join(f.agent.controlDir,'update-attention.json'))).id===digest(job.id);}
|
|
321
|
-
catch(error){if(error.code==='ENOENT')return false;throw error;}
|
|
322
|
-
});
|
|
323
|
-
const closed=new Promise(r=>first.p.once('close',r));first.p.kill('SIGTERM');await closed;
|
|
402
|
+
// Adversarial: a completed update must not publish an agent wakeup notice.
|
|
403
|
+
await assert.rejects(fs.readFile(path.join(f.agent.controlDir,'update-attention.json')),error=>error.code==='ENOENT');
|
|
324
404
|
const active=(await read(path.join(f.home,'config.json'))).packageRoot;assert(active.endsWith('/runtime'));
|
|
325
405
|
assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).packageManager.command,provider);
|
|
326
406
|
const second=start();t.after(()=>second.p.kill('SIGTERM'));
|
package/test/workspace.test.ts
CHANGED
|
@@ -45,6 +45,8 @@ test('fresh mind is private; repeat initialization preserves customization and o
|
|
|
45
45
|
assert.ok(!(await readdir(workspace)).includes('SOUL.md'))
|
|
46
46
|
assert.ok(!(await readdir(workspace)).includes('USER.md'))
|
|
47
47
|
assert.ok(!(await readdir(workspace)).includes('MEMORY.md'))
|
|
48
|
+
assert.ok(!(await readdir(workspace)).includes('inbox'))
|
|
49
|
+
assert.ok(!(await readdir(workspace)).includes('work'))
|
|
48
50
|
await writeFile(path.join(workspace, 'AGENTS.md'), 'A customized research partner')
|
|
49
51
|
await writeFile(path.join(workspace, 'MEMORY.md'), 'Existing knowledge')
|
|
50
52
|
await writeFile(path.join(workspace, 'AGENT.md'), 'Legacy custom guidance')
|
|
@@ -53,7 +55,7 @@ test('fresh mind is private; repeat initialization preserves customization and o
|
|
|
53
55
|
assert.equal(await readFile(path.join(workspace, 'MEMORY.md'), 'utf8'), 'Existing knowledge')
|
|
54
56
|
assert.equal(await readFile(path.join(workspace, 'AGENT.md'), 'utf8'), 'Legacy custom guidance')
|
|
55
57
|
assert.ok(!(await readdir(workspace)).some(name => name.endsWith('.tmp')))
|
|
56
|
-
assert.
|
|
58
|
+
assert.doesNotMatch(await readFile(path.join(workspace, 'AGENTS.md'), 'utf8'), /ez shared guidance/)
|
|
57
59
|
} finally { await rm(root, { recursive: true, force: true }) }
|
|
58
60
|
})
|
|
59
61
|
|
|
@@ -83,6 +85,7 @@ test('new agents seed a purpose-scoped AGENTS.md once and preserve subsequent ed
|
|
|
83
85
|
await writeFile(purpose, 'You are the shopping assistant for this family.\n\n### Responsibilities\n\nOrganize confirmed household shopping requests.\n')
|
|
84
86
|
await initializeWorkspace(workspace, purpose)
|
|
85
87
|
const agents = await readFile(path.join(workspace, 'AGENTS.md'), 'utf8')
|
|
88
|
+
assert.doesNotMatch(agents, /inbox\/|work\/|ezenciel-agents-message/)
|
|
86
89
|
assert.match(agents, /## Purpose\n\nYou are the shopping assistant for this family\./)
|
|
87
90
|
assert.match(agents, /### Responsibilities\n\nOrganize confirmed household shopping requests\./)
|
|
88
91
|
await writeFile(path.join(workspace, 'AGENTS.md'), 'My evolving purpose\n')
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { spawn } from 'node:child_process'
|
|
3
|
-
import { createRequire } from 'node:module'
|
|
4
|
-
import { dirname, join } from 'node:path'
|
|
5
|
-
import { fileURLToPath } from 'node:url'
|
|
6
|
-
const here=dirname(fileURLToPath(import.meta.url)),require=createRequire(import.meta.url),tsx=require.resolve('tsx'),entry=join(here,'..','src','workforce-watch-cli.ts')
|
|
7
|
-
const child=spawn(process.execPath,['--import',tsx,entry],{stdio:'inherit'})
|
|
8
|
-
child.on('exit',(code,signal)=>{if(signal)process.kill(process.pid,signal);process.exit(code??1)})
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
services:
|
|
2
|
-
workforce-watch:
|
|
3
|
-
image: ${EZ_WORKFORCE_WATCH_IMAGE:-ezenciel-agents:local}
|
|
4
|
-
build:
|
|
5
|
-
context: .
|
|
6
|
-
target: workforce-watch
|
|
7
|
-
init: true
|
|
8
|
-
restart: unless-stopped
|
|
9
|
-
read_only: true
|
|
10
|
-
cap_drop: [ALL]
|
|
11
|
-
security_opt: [no-new-privileges:true]
|
|
12
|
-
environment:
|
|
13
|
-
EZ_WATCH_PORT: 8080
|
|
14
|
-
EZ_WATCH_ENROLL_TOKEN: ${EZ_WATCH_ENROLL_TOKEN:?Set a private enrollment token}
|
|
15
|
-
EZ_WATCH_TELEGRAM_CHAT_ID: ${EZ_WATCH_TELEGRAM_CHAT_ID:-}
|
|
16
|
-
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
|
|
17
|
-
secrets: [workforce_watch_pagerduty]
|
|
18
|
-
volumes:
|
|
19
|
-
- workforce-watch-state:/state
|
|
20
|
-
tmpfs: [/tmp]
|
|
21
|
-
ports:
|
|
22
|
-
- ${EZ_WATCH_BIND_ADDRESS:-127.0.0.1}:${EZ_WATCH_PORT:-9919}:8080
|
|
23
|
-
healthcheck:
|
|
24
|
-
test: [CMD, node, -e, "fetch('http://127.0.0.1:8080/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
|
|
25
|
-
interval: 30s
|
|
26
|
-
timeout: 5s
|
|
27
|
-
retries: 3
|
|
28
|
-
start_period: 10s
|
|
29
|
-
volumes:
|
|
30
|
-
workforce-watch-state:
|
|
31
|
-
secrets:
|
|
32
|
-
workforce_watch_pagerduty:
|
|
33
|
-
file: ${EZ_WATCH_PAGERDUTY_SECRET_FILE:?Set EZ_WATCH_PAGERDUTY_SECRET_FILE to the protected workforce PagerDuty routing-key file}
|
package/docker/recovery.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { RunStore } from '../src/runs.js'
|
|
2
|
-
|
|
3
|
-
// Called only after the deployment's kernel writer lock is held. No worker
|
|
4
|
-
// from a previous container can still own this deployment; PIDs may be reused.
|
|
5
|
-
export const recoverInterruptedRuns = async (controlDir: string, channelBackend = false) => {
|
|
6
|
-
const store = new RunStore(controlDir)
|
|
7
|
-
for (const run of await store.list()) {
|
|
8
|
-
if (run.status === 'running')
|
|
9
|
-
await store.patch(run.id, { status: channelBackend && !run.pid ? 'queued' : 'failed', endedAt: new Date().toISOString() })
|
|
10
|
-
}
|
|
11
|
-
}
|
package/docs/pagerduty.md
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# PagerDuty critical-outage paging
|
|
2
|
-
|
|
3
|
-
PagerDuty alerting is an optional core capability. It observes Stocks from the
|
|
4
|
-
Ez runtime and sends Events API v2 trigger and resolve events; Stocks never
|
|
5
|
-
receives the PagerDuty routing key.
|
|
6
|
-
|
|
7
|
-
Create an Events API v2 integration in the PagerDuty service that should page
|
|
8
|
-
you. Put only its routing key in the deployment's private `relay.env` file:
|
|
9
|
-
|
|
10
|
-
```dotenv
|
|
11
|
-
PAGERDUTY_ROUTING_KEY=<Events API v2 routing key>
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Then add the tunnel-visible endpoint and monitor settings to the deployment's
|
|
15
|
-
non-secret `docker.env` file:
|
|
16
|
-
|
|
17
|
-
```dotenv
|
|
18
|
-
EZ_PAGERDUTY_STOCKS_HEALTH_URL=http://10.97.0.1:3006/health/critical
|
|
19
|
-
EZ_PAGERDUTY_POLL_SECONDS=30
|
|
20
|
-
EZ_PAGERDUTY_FAILURE_THRESHOLD=3
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
The core checks the endpoint every 30 seconds by default. Three consecutive
|
|
24
|
-
unhealthy checks trigger one incident with the stable deduplication key
|
|
25
|
-
`ez:stocks:critical-health`; recovery resolves that same incident. A failed
|
|
26
|
-
PagerDuty submission remains eligible for a later trigger. On startup, the first
|
|
27
|
-
healthy check resolves any incident left by the previous process, including an
|
|
28
|
-
uncertain trigger delivery. Failed resolves retry without treating healthy
|
|
29
|
-
Stocks as an outage. No key or health
|
|
30
|
-
response is written to agent workspaces, executor environments, or logs.
|
|
31
|
-
|
|
32
|
-
`/health/critical` is intentionally narrower than Stocks `/health`: it pages
|
|
33
|
-
only when the Stocks source worker is unavailable. Provider freshness and other
|
|
34
|
-
research-quality degradation remain visible in normal Stocks health/status but
|
|
35
|
-
do not page. The endpoint returns only a status and component summary and must
|
|
36
|
-
be reachable from the core runtime over the private tunnel. Run core outside the
|
|
37
|
-
Stocks VM if you want a VM outage to be page-worthy.
|
|
38
|
-
|
|
39
|
-
After deployment, use PagerDuty's test/escalation tooling or temporarily point
|
|
40
|
-
the monitor at an intentionally unreachable private test target, then restore
|
|
41
|
-
the real target and confirm the PagerDuty incident resolves. This is an explicit
|
|
42
|
-
operational test and may notify your escalation policy.
|
package/docs/workforce-watch.md
DELETED
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
# Workforce Watch
|
|
2
|
-
|
|
3
|
-
Workforce Watch is a separate, small Docker service for missed agent check-ins
|
|
4
|
-
and explicit terminal failures. It is not an agent, scheduler, log store or
|
|
5
|
-
remote-execution channel. It retains only the five latest compact activity
|
|
6
|
-
records supplied by each enrolled worker.
|
|
7
|
-
|
|
8
|
-
Run it in a failure domain separate from the workers it watches. In particular,
|
|
9
|
-
it must not rely on the Stocks VM to report a Stocks VM outage, and a separate
|
|
10
|
-
host is required to detect a Mac-wide outage.
|
|
11
|
-
|
|
12
|
-
## Start the monitor
|
|
13
|
-
|
|
14
|
-
Set private monitor-only values, never agent-workspace or source-control values:
|
|
15
|
-
|
|
16
|
-
```text
|
|
17
|
-
EZ_WATCH_ENROLL_TOKEN=<private fleet enrollment secret>
|
|
18
|
-
EZ_WATCH_PAGERDUTY_SECRET_FILE=/absolute/path/to/a-0600-workforce-pagerduty-key
|
|
19
|
-
TELEGRAM_BOT_TOKEN=<alert bot token>
|
|
20
|
-
EZ_WATCH_TELEGRAM_CHAT_ID=<owner chat id>
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
PagerDuty is the fleet incident owner. Workforce Watch sends a `trigger` for
|
|
24
|
-
each missed check-in or terminal failure, then a `resolve` with the stable
|
|
25
|
-
deduplication key `ez:workforce:<worker-id>` after sustained recovery. Use one
|
|
26
|
-
approved Events API v2 routing key for the fleet; it may be the existing
|
|
27
|
-
PagerDuty integration when that is the desired escalation policy. Never put it
|
|
28
|
-
in a worker's environment. The secret file contains only that key, is mode
|
|
29
|
-
`0600`, and is mounted as `/run/secrets/workforce_watch_pagerduty`; it is never
|
|
30
|
-
a Compose or container environment value. Telegram is optional, supplementary
|
|
31
|
-
owner visibility. PagerDuty is required: a Watch deployment without its
|
|
32
|
-
routing-key secret fails closed rather than silently running without paging.
|
|
33
|
-
|
|
34
|
-
The owner may explicitly authorize the CTO bot token temporarily. A dedicated
|
|
35
|
-
alert bot remains preferable because it preserves an independent delivery identity.
|
|
36
|
-
|
|
37
|
-
```sh
|
|
38
|
-
docker compose -f compose.workforce-watch.yaml up -d --build
|
|
39
|
-
curl http://127.0.0.1:9919/healthz
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
The default bind is loopback. Put private TLS networking or a private reverse
|
|
43
|
-
proxy in front of it before remote workers use it; do not expose enrollment or
|
|
44
|
-
check-in traffic publicly.
|
|
45
|
-
|
|
46
|
-
## Enroll and check in
|
|
47
|
-
|
|
48
|
-
The worker selects its monitor through its configured URL. It submits the fleet
|
|
49
|
-
enrollment secret once; the response contains the worker-specific secret. Store
|
|
50
|
-
that response only in the worker's private service environment and discard it
|
|
51
|
-
from shell history and logs.
|
|
52
|
-
|
|
53
|
-
```sh
|
|
54
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/enroll" \
|
|
55
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN" \
|
|
56
|
-
-H 'content-type: application/json' \
|
|
57
|
-
--data '{"workerId":"stocks-production","checkInSeconds":300,"graceSeconds":180,"severity":"critical"}'
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
The supervisor, not the LLM, posts check-ins using the returned worker secret:
|
|
61
|
-
|
|
62
|
-
```sh
|
|
63
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/workers/stocks-production/check-in" \
|
|
64
|
-
-H "Authorization: Bearer $EZ_WATCH_WORKER_TOKEN" \
|
|
65
|
-
-H 'content-type: application/json' \
|
|
66
|
-
--data '{"status":"ok","activity":"strategy receipt delivered","runId":"daily-strategy-2026-09-11"}'
|
|
67
|
-
```
|
|
68
|
-
|
|
69
|
-
An unrecoverable condition sends `{"status":"failed","terminal":true,...}`.
|
|
70
|
-
Optional `activity`, `error`, `runId`, and `logsHint` fields are size-limited and
|
|
71
|
-
appear in an alert. They must be public-safe operational context, never stdout,
|
|
72
|
-
prompts, holdings, credentials, or provider payloads.
|
|
73
|
-
|
|
74
|
-
## Inspect
|
|
75
|
-
|
|
76
|
-
The enrollment token reads redacted state; worker secrets never appear:
|
|
77
|
-
|
|
78
|
-
```sh
|
|
79
|
-
curl --fail-with-body "$EZ_WATCH_URL/v1/workers" \
|
|
80
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN"
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
For host replacement or a suspected worker-secret exposure, rotate that worker's
|
|
84
|
-
secret with the enrollment token. The old secret stops working immediately:
|
|
85
|
-
|
|
86
|
-
```sh
|
|
87
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/workers/stocks-production/rotate" \
|
|
88
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN"
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
Store the returned worker secret in the replacement supervisor before stopping
|
|
92
|
-
the prior one.
|
|
93
|
-
|
|
94
|
-
The service triggers once for a missed deadline or terminal failure. It sends a
|
|
95
|
-
recovery notice only after two clean check-ins by default to prevent flapping.
|
|
96
|
-
|
|
97
|
-
If a worker fails again during a partially delivered recovery, channels that
|
|
98
|
-
already accepted recovery receive a fresh trigger. Channels still open retain
|
|
99
|
-
their delivery state, so they do not receive duplicate opening alerts. This
|
|
100
|
-
applies to terminal failures, failed check-ins, and missed check-ins, including
|
|
101
|
-
after a Watch restart.
|