@jc_stack/ez-agents 0.1.0-beta.33 → 0.1.0-beta.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +1 -0
- package/.env.example +9 -16
- package/AGENTS.md +17 -9
- package/CHANGELOG.md +92 -0
- package/Dockerfile +11 -9
- package/README.md +32 -27
- package/THIRD_PARTY_NOTICES.md +4 -4
- package/bin/ez +97 -0
- package/bin/ezenciel-agents-plugin-broker.mjs +16 -0
- package/bin/ezenciel-agents-provision-telegram.mjs +13 -0
- package/bin/ezenciel-agents.mjs +1 -1
- package/compose.yaml +42 -5
- package/docker/entrypoint.sh +1 -1
- package/docker/healthcheck.mjs +12 -4
- package/docker/run.ts +5 -6
- package/docker/smoke.mjs +32 -7
- package/docs/application-channel.md +38 -10
- package/docs/architecture/ai-selection.md +3 -1
- package/docs/architecture/authority-boundaries.md +2 -2
- package/docs/channel-backend.md +4 -3
- package/docs/docker-runtime.md +106 -21
- package/docs/host-service.md +17 -6
- package/docs/local-qa.md +2 -2
- package/docs/plugin-guide.md +202 -0
- package/docs/plugins.md +63 -2
- package/docs/releasing.md +13 -0
- package/docs/repair.md +1 -1
- package/docs/responsive-channels.md +1 -1
- package/docs/scheduling.md +4 -4
- package/docs/setup.md +96 -16
- package/docs/standalone-cli.md +6 -5
- package/docs/upgrades.md +46 -23
- package/package.json +5 -5
- package/scripts/smoke-scheduler.ts +12 -4
- package/scripts/smoke.ts +14 -11
- package/src/agent-guidance.ts +6 -9
- package/src/agent-install.ts +44 -9
- package/src/ai-cli.ts +9 -7
- package/src/ai.ts +7 -7
- package/src/application-channel.ts +88 -17
- package/src/application-cli.ts +2 -2
- package/src/approval-cli.ts +14 -7
- package/src/approval.ts +27 -37
- package/src/channel-backend.ts +2 -3
- package/src/codex-session.ts +5 -3
- package/src/config.ts +18 -28
- package/src/control-state.ts +187 -9
- package/src/conversation-menu.ts +6 -17
- package/src/delivery-socket-client.d.mts +4 -0
- package/src/delivery-socket-client.mjs +104 -0
- package/src/delivery-socket.ts +354 -0
- package/src/execution-authority.ts +21 -5
- package/src/executor.ts +110 -31
- package/src/files.ts +38 -11
- package/src/host-executor.ts +50 -17
- package/src/inbox.ts +15 -49
- package/src/index.ts +96 -52
- package/src/install-config.ts +2 -2
- package/src/install-tools.mjs +100 -23
- package/src/isolation.ts +25 -0
- package/src/ledger-endpoint.d.mts +2 -0
- package/src/ledger-endpoint.mjs +17 -0
- package/src/menu.ts +16 -8
- package/src/message-history.ts +13 -18
- package/src/message.ts +35 -18
- package/src/plugin-broker.mjs +396 -0
- package/src/plugins/connection.mjs +2 -2
- package/src/plugins/manager.mjs +178 -26
- package/src/plugins/native-tasks.mjs +2 -2
- package/src/prompt-suffix.ts +12 -0
- package/src/react.ts +6 -4
- package/src/runs.ts +156 -243
- package/src/schedule-cli.ts +29 -10
- package/src/scheduled-tasks.ts +53 -10
- package/src/scheduler.ts +20 -7
- package/src/service.ts +3 -3
- package/src/setup.ts +11 -7
- package/src/software-status.ts +36 -4
- package/src/task-executor.ts +6 -3
- package/src/task-workspace.ts +4 -3
- package/src/tasks.ts +1 -1
- package/src/telegram-provisioning-cli.ts +22 -0
- package/src/telegram-provisioning.ts +214 -0
- package/src/updates/control.mjs +77 -7
- package/src/updates/runtime.mjs +29 -9
- package/src/updates/status.mjs +25 -3
- package/src/updates/supervisor.mjs +31 -18
- package/src/version.ts +20 -0
- package/src/workspace-confine.ts +38 -0
- package/src/workspace.ts +24 -20
- package/templates/agent-purpose.md +1 -0
- package/templates/failure-review.md +1 -1
- package/templates/maintainer-purpose.md +1 -1
- package/test/agent-guidance.test.ts +18 -88
- package/test/agent-install.test.ts +48 -0
- package/test/ai-cli.test.ts +18 -0
- package/test/ai.test.ts +84 -0
- package/test/application-channel.test.ts +66 -15
- package/test/application-controls.test.ts +56 -3
- package/test/application-only.test.ts +16 -13
- package/test/channel-backend.test.ts +3 -7
- package/test/channel-delivery.test.ts +4 -1
- package/test/channel-owner.test.ts +6 -3
- package/test/codex-context.test.ts +63 -3
- package/test/codex-session.test.ts +6 -1
- package/test/config.test.ts +47 -37
- package/test/control-state.test.ts +83 -4
- package/test/conversation-menu.test.ts +2 -4
- package/test/delivery-transport.test.ts +72 -0
- package/test/docker-runtime.test.ts +27 -14
- package/test/engine-handoff.test.ts +12 -8
- package/test/event-sources.test.ts +2 -2
- package/test/execution-authority.test.ts +9 -9
- package/test/executor.test.ts +47 -11
- package/test/failure.test.ts +18 -5
- package/test/files.test.ts +2 -2
- package/test/helpers/ledger.ts +18 -0
- package/test/host-executor.test.ts +80 -0
- package/test/inbox.test.ts +8 -10
- package/test/install-config.test.ts +18 -6
- package/test/install-tools.test.mjs +80 -7
- package/test/intake-relay.test.ts +58 -11
- package/test/message-history.test.ts +39 -47
- package/test/native-tasks.test.ts +1 -1
- package/test/plugin-broker.test.mjs +172 -0
- package/test/plugin-manager.test.mjs +143 -5
- package/test/reaction.test.ts +4 -0
- package/test/relay.test.ts +5 -7
- package/test/repair-policy.test.ts +1 -4
- package/test/schedule-cli.test.ts +14 -9
- package/test/scheduled-tasks.test.ts +29 -6
- package/test/scheduler-host.test.ts +5 -5
- package/test/scheduler-relay.test.ts +15 -9
- package/test/scheduler.test.ts +2 -2
- package/test/security.test.ts +6 -3
- package/test/software-status.test.ts +21 -3
- package/test/telegram-provisioning.test.ts +160 -0
- package/test/updates.test.mjs +102 -13
- package/test/workspace-confine.test.ts +47 -0
- package/test/workspace.test.ts +27 -11
- package/bin/ezenciel-agents-watch.mjs +0 -8
- package/compose.workforce-watch.yaml +0 -33
- package/docker/recovery.ts +0 -11
- package/docs/pagerduty.md +0 -42
- package/docs/workforce-watch.md +0 -101
- package/src/pagerduty.ts +0 -109
- package/src/reply-context.ts +0 -44
- package/src/update-attention.ts +0 -18
- package/src/workforce-watch-cli.ts +0 -14
- package/src/workforce-watch.ts +0 -155
- package/templates/agent/AGENTS.md +0 -15
- package/templates/agent/SOUL.md +0 -11
- package/templates/agent/USER.md +0 -5
- package/templates/agent-guidance.md +0 -114
- package/test/pagerduty.test.ts +0 -104
- package/test/update-attention.test.ts +0 -20
- package/test/workforce-watch.test.ts +0 -180
package/test/updates.test.mjs
CHANGED
|
@@ -7,12 +7,21 @@ import { gzipSync } from 'node:zlib';
|
|
|
7
7
|
import { spawn, execFile } from 'node:child_process';
|
|
8
8
|
import { promisify } from 'node:util';
|
|
9
9
|
import { extract, digest, version, newer, compatible } from '../src/updates/artifact.mjs';
|
|
10
|
-
import { prepare, submit, command, read, jobPath, eligibility, jobs } from '../src/updates/control.mjs';
|
|
10
|
+
import { prepare, submit, command, read, jobPath, eligibility, jobs, cleanupStaleBackups, reviewedPluginDeploymentMigration } from '../src/updates/control.mjs';
|
|
11
11
|
import { perform, environment, packageManager } from '../src/updates/runtime.mjs';
|
|
12
12
|
import { atomic, snapshot, compose, prepareCommand } from '../src/plugins/manager.mjs';
|
|
13
13
|
import { bindUpdates } from '../src/updates/binding.mjs';
|
|
14
14
|
import { status as runtimeStatus } from '../src/updates/status.mjs';
|
|
15
|
+
import { providerEnvironment } from '../src/updates/supervisor.mjs';
|
|
15
16
|
const exec=promisify(execFile);
|
|
17
|
+
|
|
18
|
+
test('supervisor forwards only provider keys declared by the agent installation',()=>{
|
|
19
|
+
const host={agents:[{codexProviders:[{envKey:'OPENROUTER_API_KEY'}]}]};
|
|
20
|
+
assert.deepEqual(providerEnvironment(host,{OPENROUTER_API_KEY:'provider-secret',TELEGRAM_BOT_TOKEN:'relay-secret'}),{OPENROUTER_API_KEY:'provider-secret'});
|
|
21
|
+
assert.throws(()=>providerEnvironment({agents:[{codexProviders:[{envKey:'bad-key'}]}]},{}),/environment key/);
|
|
22
|
+
for(const envKey of ['PATH','HOME','CODEX_HOME','NODE_OPTIONS','EZ_CONTROL_DIR','TELEGRAM_BOT_TOKEN'])
|
|
23
|
+
assert.throws(()=>providerEnvironment({agents:[{codexProviders:[{envKey}]}]},{}),/Reserved Codex provider environment key/);
|
|
24
|
+
});
|
|
16
25
|
const contract=kind=>({protocol:1,kind,stateSchema:1,mainProtocol:1});
|
|
17
26
|
function tar(entries) {
|
|
18
27
|
const chunks=[];
|
|
@@ -83,7 +92,7 @@ test('status distinguishes installed, running, legacy and stale main versions wi
|
|
|
83
92
|
}
|
|
84
93
|
await fs.writeFile(relay,'broken');s=await command(f.home,['status']);assert.equal(s.main.state,'unknown');
|
|
85
94
|
await fs.rm(relay);s=await command(f.home,['status']);assert.equal(s.main.state,'offline');
|
|
86
|
-
const result=await exec(process.execPath,[new URL('../bin/ezenciel-agents-tools.mjs',import.meta.url).pathname,'--home',f.home,'status']);
|
|
95
|
+
const result=await exec(process.execPath,[new URL('../bin/ezenciel-agents-tools.mjs',import.meta.url).pathname,'--home',f.home,'status'],{cwd:f.agent.workspace});
|
|
87
96
|
assert.equal(JSON.parse(result.stdout).main.installedVersion,'0.1.0');
|
|
88
97
|
});
|
|
89
98
|
test('relay healthcheck emits bounded predicate evidence without state contents',async t=>{
|
|
@@ -98,20 +107,39 @@ test('relay healthcheck emits bounded predicate evidence without state contents'
|
|
|
98
107
|
});
|
|
99
108
|
test('plugin status verifies images and reports stopped, mismatched and unreachable runtimes honestly',async t=>{
|
|
100
109
|
const f=await fixture(t,'plugin'),id='a'.repeat(64),image='sha256:'+'b'.repeat(64);
|
|
101
|
-
for(const mode of ['running','ndjson','stopped','missing','mismatched','offline']) {
|
|
110
|
+
for(const mode of ['running','ndjson','warned','stopped','missing','mismatched','offline']) {
|
|
102
111
|
const calls=[];
|
|
103
112
|
const run=async(c,args)=>{
|
|
104
113
|
calls.push([c,...args]);assert.equal(c,'docker');
|
|
105
114
|
if(mode==='offline')throw Error('synthetic secret must not escape');
|
|
106
|
-
if(args.includes('ps')) {const rows=mode==='missing'?[]:[{Service:'sample',State:mode==='stopped'?'exited':'running',Health:'healthy',ID:id}];
|
|
115
|
+
if(args.includes('ps')) {const rows=mode==='missing'?[]:[{Service:'sample',State:mode==='stopped'?'exited':'running',Health:'healthy',ID:id}];const lines=mode==='ndjson'?rows.map(r=>JSON.stringify(r)).join('\n'):JSON.stringify(rows);return mode==='warned'?'time="now" level=warning msg="the attribute version is obsolete"\n'+rows.map(r=>JSON.stringify(r)).join('\n'):lines;}
|
|
107
116
|
assert(args.includes('inspect'));return mode==='mismatched'&&args[0]==='image'?'sha256:'+'c'.repeat(64):image;
|
|
108
117
|
};
|
|
109
118
|
const s=await runtimeStatus(f.home,run),p=s.plugins[0];
|
|
110
|
-
assert.equal(p.installedVersion,'0.1.0');assert.equal(p.runningVersion,['running','ndjson'].includes(mode)?'0.1.0':null);
|
|
119
|
+
assert.equal(p.installedVersion,'0.1.0');assert.equal(p.runningVersion,['running','ndjson','warned'].includes(mode)?'0.1.0':null);
|
|
111
120
|
assert.equal(p.state,mode==='offline'?'unknown':['stopped','missing'].includes(mode)?'stopped':'running');
|
|
112
121
|
assert(!JSON.stringify(s).includes('synthetic secret'));assert(calls.every(c=>!c.includes('exec')&&!c.includes('start')&&!c.includes('up')));
|
|
113
122
|
}
|
|
114
123
|
});
|
|
124
|
+
test('isolated plugin status uses bundled Compose v1 and inspect readback',async t=>{
|
|
125
|
+
const f=await fixture(t,'plugin'),id='a'.repeat(64),image='sha256:'+'b'.repeat(64),previous=process.env.EZ_DOCKER_COMPOSE;
|
|
126
|
+
process.env.EZ_DOCKER_COMPOSE='standalone';
|
|
127
|
+
try {
|
|
128
|
+
const calls=[],run=async(c,args)=>{
|
|
129
|
+
calls.push([c,...args]);assert.equal(c,'docker');
|
|
130
|
+
if(args.at(-1)==='-q')return id;
|
|
131
|
+
if(args[0]==='inspect'&&args.length===2)return JSON.stringify([{Config:{Labels:{'com.docker.compose.service':'sample'}},State:{Status:'running',Health:{Status:'healthy'}}}]);
|
|
132
|
+
if(args[0]==='inspect')return image;
|
|
133
|
+
if(args[0]==='image')return image;
|
|
134
|
+
throw Error('unexpected status command');
|
|
135
|
+
};
|
|
136
|
+
const p=(await runtimeStatus(f.home,run)).plugins[0];
|
|
137
|
+
assert.equal(p.state,'running');assert.equal(p.runningVersion,'0.1.0');
|
|
138
|
+
assert.deepEqual(calls[0].slice(-3),['ps','--all','-q']);
|
|
139
|
+
} finally {
|
|
140
|
+
if(previous===undefined)delete process.env.EZ_DOCKER_COMPOSE;else process.env.EZ_DOCKER_COMPOSE=previous;
|
|
141
|
+
}
|
|
142
|
+
});
|
|
115
143
|
test('SemVer ordering and compatibility reject ranges, malformed values and downgrades',()=>{
|
|
116
144
|
for(const s of ['latest','../1','1.0','01.0.0','1.0.0-01'])assert.throws(()=>version(s));
|
|
117
145
|
assert(newer('0.1.0-beta.10','0.1.0-beta.2'));assert(newer('0.1.0','0.1.0-beta.10'));assert(!newer('0.1.0-beta.2','0.1.0'));assert(!newer('1.0.0','1.0.0'));
|
|
@@ -145,6 +173,63 @@ test('identity, state migration, deployment changes and stale candidates fail be
|
|
|
145
173
|
await mutate({ezRelease:contract('main'),version:'0.0.1'});await assert.rejects(prepare(f.home,'main',{file:await f.pack()}),/newer/);
|
|
146
174
|
await mutate({version:'0.1.1'});await fs.writeFile(path.join(f.source,'compose.yaml'),'privileged: true');await assert.rejects(prepare(f.home,'main',{file:await f.pack()}),/deployment/);
|
|
147
175
|
});
|
|
176
|
+
test('legacy runtime deployment migration compares effective Compose and preserves explicit bindings',async t=>{
|
|
177
|
+
const f=await fixture(t),oldCompose=`services:
|
|
178
|
+
relay:
|
|
179
|
+
environment:
|
|
180
|
+
EZ_AGENT_PURPOSE_FILE: /run/agent-purpose.md
|
|
181
|
+
EZ_EXECUTOR_TRANSPORT: \${EZ_EXECUTOR_TRANSPORT:-host}
|
|
182
|
+
configs:
|
|
183
|
+
agent_purpose:
|
|
184
|
+
file: \${EZ_AGENT_PURPOSE_FILE:-./templates/agent/SOUL.md}
|
|
185
|
+
`,nextCompose=oldCompose.replace(' EZ_EXECUTOR_TRANSPORT:', ' EZ_ISOLATION: ${EZ_ISOLATION:-}\n EZ_EXECUTOR_TRANSPORT:').replace('./templates/agent/SOUL.md','./templates/agent-purpose.md');
|
|
186
|
+
await fs.writeFile(path.join(f.old,'compose.yaml'),oldCompose);await fs.writeFile(path.join(f.source,'compose.yaml'),nextCompose);
|
|
187
|
+
await fs.writeFile(path.join(f.config.deploymentDir,'docker.env'),`COMPOSE_FILE='${f.old}/compose.yaml'
|
|
188
|
+
COMPOSE_PROJECT_NAME='ez-agent-fixture'
|
|
189
|
+
EZ_AGENT_PURPOSE_FILE='${path.join(f.config.deploymentDir,'purpose.md')}'
|
|
190
|
+
EZ_EXECUTOR_TRANSPORT='host'
|
|
191
|
+
`);
|
|
192
|
+
const job=await prepare(f.home,'main',{file:await f.pack()});assert.equal(job.deploymentMigration.id,'legacy-runtime-v1');
|
|
193
|
+
await fs.writeFile(path.join(f.config.deploymentDir,'docker.env'),`COMPOSE_FILE='${f.old}/compose.yaml'
|
|
194
|
+
COMPOSE_PROJECT_NAME='ez-agent-fixture'
|
|
195
|
+
EZ_EXECUTOR_TRANSPORT='host'
|
|
196
|
+
`);await assert.rejects(eligibility(f.home,'main',f.source,false),/deployment/);
|
|
197
|
+
});
|
|
198
|
+
test('legacy runtime migration admits homes without docker.env when purpose fallback is already bound',async t=>{
|
|
199
|
+
const f=await fixture(t),oldCompose=`services:
|
|
200
|
+
relay:
|
|
201
|
+
environment:
|
|
202
|
+
EZ_AGENT_PURPOSE_FILE: /run/agent-purpose.md
|
|
203
|
+
EZ_EXECUTOR_TRANSPORT: \${EZ_EXECUTOR_TRANSPORT:-host}
|
|
204
|
+
configs:
|
|
205
|
+
agent_purpose:
|
|
206
|
+
file: \${EZ_AGENT_PURPOSE_FILE:-./templates/agent-purpose.md}
|
|
207
|
+
`,nextCompose=oldCompose.replace(' EZ_EXECUTOR_TRANSPORT:', ' EZ_ISOLATION: \${EZ_ISOLATION:-}\n EZ_EXECUTOR_TRANSPORT:');
|
|
208
|
+
await fs.writeFile(path.join(f.old,'compose.yaml'),oldCompose);await fs.writeFile(path.join(f.source,'compose.yaml'),nextCompose);
|
|
209
|
+
await fs.rm(path.join(f.config.deploymentDir,'docker.env'));
|
|
210
|
+
const job=await prepare(f.home,'main',{file:await f.pack()});assert.equal(job.deploymentMigration.id,'legacy-runtime-v1');
|
|
211
|
+
});
|
|
212
|
+
test('compatibility bridge hands off to the published beta.34 release',async t=>{
|
|
213
|
+
const f=await fixture(t),prior=await read(path.join(f.old,'package.json')),candidate=await read(path.join(f.source,'package.json'));
|
|
214
|
+
await atomic(path.join(f.old,'package.json'),{...prior,version:'0.1.0-beta.34.compat.1'});
|
|
215
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.34'});
|
|
216
|
+
await assert.doesNotReject(eligibility(f.home,'main',f.source,false));
|
|
217
|
+
});
|
|
218
|
+
test('compatibility bridge hands off any matching canonical beta release',async t=>{
|
|
219
|
+
const f=await fixture(t),prior=await read(path.join(f.old,'package.json')),candidate=await read(path.join(f.source,'package.json'));
|
|
220
|
+
await atomic(path.join(f.old,'package.json'),{...prior,version:'0.1.0-beta.35.compat.1'});
|
|
221
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.35'});
|
|
222
|
+
await assert.doesNotReject(eligibility(f.home,'main',f.source,false));
|
|
223
|
+
await atomic(path.join(f.source,'package.json'),{...candidate,version:'0.1.0-beta.34'});
|
|
224
|
+
await assert.rejects(eligibility(f.home,'main',f.source,false),/newer/);
|
|
225
|
+
});
|
|
226
|
+
test('reviewed Library beta14 command migration is exact and fail-closed',()=>{
|
|
227
|
+
const manifest=(version,document)=>({schemaVersion:1,id:'library',version,description:'library',commands:{library:{executable:'bin/ez-library.mjs',args:[]},'library-query':{executable:'bin/ez-library.mjs',args:[],channelQuery:true},...(document?{'library-document':{executable:'bin/ez-library.mjs',args:[],channelQuery:true}}:{})},skills:['skills/library/SKILL.md']});
|
|
228
|
+
const deployment=(query,document)=>({schemaVersion:3,services:{library:{buildTarget:'runtime',volumes:{data:'/state'},healthcheck:['node','/app/bin/ez-library.mjs','--version'],memoryMiB:4096,cpus:2}},commands:{'library-query':{service:'library',argv:['node','/app/bin/ez-library.mjs',query]},...(document?{'library-document':{service:'library',argv:['node','/app/bin/ez-library.mjs','document']}}:{})},sharedServices:{}});
|
|
229
|
+
const old={manifest:manifest('0.1.0-beta.13.qa.4',true),deployment:deployment('query-context',true)},next={manifest:manifest('0.1.0-beta.14',false),deployment:deployment('search',false)};
|
|
230
|
+
assert.equal(reviewedPluginDeploymentMigration('library',old,next).id,'library-query-contract-v1');
|
|
231
|
+
const unsafe=structuredClone(next);unsafe.deployment.services.library.cpus=4;assert.equal(reviewedPluginDeploymentMigration('library',old,unsafe),undefined);
|
|
232
|
+
});
|
|
148
233
|
test('main transaction stages before stopping, pins rollback image, preserves state and rebinds root',async t=>{
|
|
149
234
|
const f=await fixture(t),job=await queued(f),r=runtime(f);
|
|
150
235
|
await fs.writeFile(path.join(f.agent.workspace,'memory.md'),'retain me');
|
|
@@ -158,6 +243,13 @@ test('main transaction stages before stopping, pins rollback image, preserves st
|
|
|
158
243
|
const status=await command(f.home,['status']);assert(!JSON.stringify(status).includes('private-test-token'));assert(!('rollback'in status.jobs[0]));
|
|
159
244
|
assert.equal((await fs.stat(path.join(jobPath(f.home,job.id),'job.json'))).mode&0o777,0o600);
|
|
160
245
|
});
|
|
246
|
+
test('update cleanup keeps the newest backup per target and ignores unfinished jobs',async t=>{
|
|
247
|
+
const f=await fixture(t),older='11111111-1111-1111-1111-111111111111',latest='22222222-2222-2222-2222-222222222222',queued='33333333-3333-3333-3333-333333333333',rolledBack='44444444-4444-4444-4444-444444444444';
|
|
248
|
+
const seed=async(id,target,status,endedAt)=>{const dir=jobPath(f.home,id);await fs.mkdir(path.join(dir,'backup'),{recursive:true});await fs.writeFile(path.join(dir,'backup','marker'),'private');await atomic(path.join(dir,'job.json'),{id,target,status,createdAt:endedAt,endedAt});};
|
|
249
|
+
await seed(older,'main','completed','2026-01-01T00:00:00.000Z');await seed(latest,'main','completed','2026-01-02T00:00:00.000Z');await seed(queued,'sample','queued','2026-01-03T00:00:00.000Z');await seed(rolledBack,'main','rolled-back','2026-01-03T00:00:00.000Z');
|
|
250
|
+
const result=await cleanupStaleBackups(f.home);assert.deepEqual(result.kept,[latest]);assert.deepEqual(result.removed.sort(),[older,rolledBack].sort());
|
|
251
|
+
await assert.rejects(fs.access(path.join(jobPath(f.home,older),'backup')));await fs.access(path.join(jobPath(f.home,latest),'backup'));await fs.access(path.join(jobPath(f.home,queued),'backup'));await assert.rejects(fs.access(path.join(jobPath(f.home,rolledBack),'backup')));
|
|
252
|
+
});
|
|
161
253
|
test('failed preparation never stops runtime; failed activation rolls back code without rewinding state',async t=>{
|
|
162
254
|
for(const stage of ['build','health']) {
|
|
163
255
|
const f=await fixture(t),job=await queued(f),r=runtime(f,{fail:(_c,a)=>stage==='build'?a[0]==='build':a.includes('up')});
|
|
@@ -294,9 +386,10 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
|
|
|
294
386
|
await fs.writeFile(wrapper,`import {supervise} from ${JSON.stringify(module)};const a=new AbortController();process.on('SIGTERM',()=>a.abort());await supervise(${JSON.stringify(f.config.deploymentDir)},a.signal,{discover:async()=>{${provider==='pnpm' ? "throw Error('Synthetic discovery failure')" : 'return []'}}});`);
|
|
295
387
|
const start=()=>{const p=spawn(process.execPath,[wrapper],{env:{...process.env,PATH:fake},stdio:['ignore','pipe','pipe']});let output='';p.stdout.on('data',b=>output+=b);p.stderr.on('data',b=>output+=b);return {p,output:()=>output};};
|
|
296
388
|
const wait=async fn=>{for(let i=0;i<150;i++){const result=await fn();if(result)return result;await new Promise(r=>setTimeout(r,100));}throw Error('Timed out');};
|
|
297
|
-
const first=start();t.after(()=>{first.p.kill('SIGTERM');});
|
|
389
|
+
const first=start(),firstClosed=new Promise(resolve=>first.p.once('close',resolve));t.after(()=>{first.p.kill('SIGTERM');});
|
|
298
390
|
const heartbeat=()=>read(path.join(f.agent.controlDir,'host-executor/heartbeat.json')).catch(()=>null);
|
|
299
391
|
const oldBeat=await wait(heartbeat);
|
|
392
|
+
if(provider==='pnpm')await wait(()=>first.output().includes('Update discovery failed; host remains running.'));
|
|
300
393
|
const running=path.join(f.agent.controlDir,'host-executor/r_request.running.json');await fs.writeFile(running,'{}');
|
|
301
394
|
const job=await prepare(f.home,'main',{file:await f.pack()});
|
|
302
395
|
const requester=await exec(process.execPath,['--input-type=module','-e',`import {submit} from ${JSON.stringify(new URL('../src/updates/control.mjs',import.meta.url).href)};console.log(JSON.stringify(await submit(${JSON.stringify(f.home)},${JSON.stringify(job.id)},false)));`]);
|
|
@@ -304,14 +397,10 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
|
|
|
304
397
|
await new Promise(r=>setTimeout(r,1800));assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).status,'queued');
|
|
305
398
|
await fs.rm(running);
|
|
306
399
|
await wait(async()=>{const j=await read(path.join(jobPath(f.home,job.id),'job.json'));if(j.status==='failed'||j.status==='rolled-back')throw Error(JSON.stringify(j)+first.output());return j.status==='completed';});
|
|
307
|
-
const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert(first.
|
|
400
|
+
const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert.equal(await firstClosed,0,first.output());
|
|
308
401
|
if(provider==='pnpm'){assert.match(first.output(),/Update discovery failed; host remains running/);assert.doesNotMatch(first.output(),/Synthetic discovery failure/);}
|
|
309
|
-
//
|
|
310
|
-
await
|
|
311
|
-
try{return (await read(path.join(f.agent.controlDir,'update-attention.json'))).id===digest(job.id);}
|
|
312
|
-
catch(error){if(error.code==='ENOENT')return false;throw error;}
|
|
313
|
-
});
|
|
314
|
-
const closed=new Promise(r=>first.p.once('close',r));first.p.kill('SIGTERM');await closed;
|
|
402
|
+
// Adversarial: a completed update must not publish an agent wakeup notice.
|
|
403
|
+
await assert.rejects(fs.readFile(path.join(f.agent.controlDir,'update-attention.json')),error=>error.code==='ENOENT');
|
|
315
404
|
const active=(await read(path.join(f.home,'config.json'))).packageRoot;assert(active.endsWith('/runtime'));
|
|
316
405
|
assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).packageManager.command,provider);
|
|
317
406
|
const second=start();t.after(()=>second.p.kill('SIGTERM'));
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import test from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
|
|
4
|
+
import { spawn } from 'node:child_process'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import path from 'node:path'
|
|
7
|
+
import { macosWorkspaceProfile, workspaceSiblingDenies } from '../src/workspace-confine.js'
|
|
8
|
+
|
|
9
|
+
test('sibling confinement skips $HOME and filesystem root', () => {
|
|
10
|
+
assert.deepEqual(workspaceSiblingDenies('/Users/ada/agent', '/Users/ada'), [])
|
|
11
|
+
assert.deepEqual(workspaceSiblingDenies('/', '/Users/ada'), [])
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
test('sibling confinement denies the tenant farm, not the bound workspace', () => {
|
|
15
|
+
assert.deepEqual(workspaceSiblingDenies('/data/tenants/juan', '/Users/ada'), [
|
|
16
|
+
{ deny: '/data/tenants', allow: '/data/tenants/juan' },
|
|
17
|
+
])
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
test('macos profile blocks sibling reads and writes', async (t) => {
|
|
21
|
+
if (process.platform !== 'darwin') { t.skip(); return }
|
|
22
|
+
const root = await mkdtemp(path.join(tmpdir(), 'ez-confine-'))
|
|
23
|
+
t.after(() => rm(root, { recursive: true, force: true }))
|
|
24
|
+
const mine = path.join(root, 'juan')
|
|
25
|
+
const other = path.join(root, 'other')
|
|
26
|
+
const shared = path.join(root, 'shared')
|
|
27
|
+
const additional = path.join(root, 'additional')
|
|
28
|
+
await mkdir(mine); await mkdir(other); await mkdir(shared); await mkdir(additional)
|
|
29
|
+
await writeFile(path.join(mine, 'own.txt'), 'own')
|
|
30
|
+
await writeFile(path.join(other, 'secret.txt'), 'secret')
|
|
31
|
+
await writeFile(path.join(shared, 'shared.txt'), 'shared')
|
|
32
|
+
await writeFile(path.join(additional, 'additional.txt'), 'additional')
|
|
33
|
+
const profile = path.join(root, 'workspace.sb')
|
|
34
|
+
await writeFile(profile, macosWorkspaceProfile(workspaceSiblingDenies(mine, '/Users/ada'), [shared, additional]))
|
|
35
|
+
const run = (file: string) => new Promise<{ code: number | null; stderr: string }>(resolve => {
|
|
36
|
+
const child = spawn('sandbox-exec', ['-f', profile, '/bin/cat', file], { stdio: ['ignore', 'pipe', 'pipe'] })
|
|
37
|
+
let stderr = ''
|
|
38
|
+
child.stderr.on('data', chunk => { stderr += chunk })
|
|
39
|
+
child.on('close', code => resolve({ code, stderr }))
|
|
40
|
+
})
|
|
41
|
+
assert.equal((await run(path.join(mine, 'own.txt'))).code, 0)
|
|
42
|
+
assert.equal((await run(path.join(shared, 'shared.txt'))).code, 0)
|
|
43
|
+
assert.equal((await run(path.join(additional, 'additional.txt'))).code, 0)
|
|
44
|
+
const blocked = await run(path.join(other, 'secret.txt'))
|
|
45
|
+
assert.notEqual(blocked.code, 0)
|
|
46
|
+
assert.match(blocked.stderr, /not permitted|Operation not permitted/i)
|
|
47
|
+
})
|
package/test/workspace.test.ts
CHANGED
|
@@ -38,42 +38,58 @@ test('fresh mind is private; repeat initialization preserves customization and o
|
|
|
38
38
|
const root = await mkdtemp(path.join(tmpdir(), 'ez-mind-'))
|
|
39
39
|
const workspace = path.join(root, 'agent')
|
|
40
40
|
try {
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
const purpose = path.join(root, 'purpose.md')
|
|
42
|
+
await writeFile(purpose, 'Research partner\n')
|
|
43
|
+
assert.deepEqual(await initializeWorkspace(workspace, purpose), ['AGENTS.md'])
|
|
44
|
+
assert.equal((await stat(path.join(workspace, 'AGENTS.md'))).mode & 0o777, 0o600)
|
|
45
|
+
assert.ok(!(await readdir(workspace)).includes('SOUL.md'))
|
|
46
|
+
assert.ok(!(await readdir(workspace)).includes('USER.md'))
|
|
43
47
|
assert.ok(!(await readdir(workspace)).includes('MEMORY.md'))
|
|
44
|
-
await
|
|
48
|
+
assert.ok(!(await readdir(workspace)).includes('inbox'))
|
|
49
|
+
assert.ok(!(await readdir(workspace)).includes('work'))
|
|
50
|
+
await writeFile(path.join(workspace, 'AGENTS.md'), 'A customized research partner')
|
|
45
51
|
await writeFile(path.join(workspace, 'MEMORY.md'), 'Existing knowledge')
|
|
46
52
|
await writeFile(path.join(workspace, 'AGENT.md'), 'Legacy custom guidance')
|
|
47
53
|
assert.deepEqual(await initializeWorkspace(workspace), [])
|
|
48
|
-
assert.
|
|
54
|
+
assert.match(await readFile(path.join(workspace, 'AGENTS.md'), 'utf8'), /A customized research partner/)
|
|
49
55
|
assert.equal(await readFile(path.join(workspace, 'MEMORY.md'), 'utf8'), 'Existing knowledge')
|
|
50
56
|
assert.equal(await readFile(path.join(workspace, 'AGENT.md'), 'utf8'), 'Legacy custom guidance')
|
|
51
57
|
assert.ok(!(await readdir(workspace)).some(name => name.endsWith('.tmp')))
|
|
52
|
-
assert.
|
|
58
|
+
assert.doesNotMatch(await readFile(path.join(workspace, 'AGENTS.md'), 'utf8'), /ez shared guidance/)
|
|
53
59
|
} finally { await rm(root, { recursive: true, force: true }) }
|
|
54
60
|
})
|
|
55
61
|
|
|
62
|
+
test('fresh workspace refuses to invent a generic purpose', async () => {
|
|
63
|
+
const root = await mkdtemp(path.join(tmpdir(), 'ez-purpose-required-'))
|
|
64
|
+
try {
|
|
65
|
+
await assert.rejects(initializeWorkspace(path.join(root, 'mind'), undefined), /purpose file is required/)
|
|
66
|
+
} finally { await rm(root, {recursive:true, force:true}) }
|
|
67
|
+
})
|
|
68
|
+
|
|
56
69
|
test('seed refuses a symlink without overwriting its target', async () => {
|
|
57
70
|
const root = await mkdtemp(path.join(tmpdir(), 'ez-mind-'))
|
|
58
71
|
try {
|
|
59
72
|
const target = path.join(root, 'outside.md')
|
|
60
73
|
await writeFile(target, 'untouched')
|
|
61
|
-
await symlink(target, path.join(root, '
|
|
74
|
+
await symlink(target, path.join(root, 'AGENTS.md'))
|
|
62
75
|
await assert.rejects(initializeWorkspace(root), /regular file/)
|
|
63
76
|
assert.equal(await readFile(target, 'utf8'), 'untouched')
|
|
64
77
|
} finally { await rm(root, { recursive: true, force: true }) }
|
|
65
78
|
})
|
|
66
79
|
|
|
67
|
-
test('new agents seed
|
|
80
|
+
test('new agents seed a purpose-scoped AGENTS.md once and preserve subsequent edits', async () => {
|
|
68
81
|
const root = await mkdtemp(path.join(tmpdir(), 'ez-purpose-'))
|
|
69
82
|
try {
|
|
70
83
|
const purpose = path.join(root, 'purpose.md')
|
|
71
84
|
const workspace = path.join(root, 'mind')
|
|
72
|
-
await writeFile(purpose, '
|
|
85
|
+
await writeFile(purpose, 'You are the shopping assistant for this family.\n\n### Responsibilities\n\nOrganize confirmed household shopping requests.\n')
|
|
73
86
|
await initializeWorkspace(workspace, purpose)
|
|
74
|
-
|
|
75
|
-
|
|
87
|
+
const agents = await readFile(path.join(workspace, 'AGENTS.md'), 'utf8')
|
|
88
|
+
assert.doesNotMatch(agents, /inbox\/|work\/|ezenciel-agents-message/)
|
|
89
|
+
assert.match(agents, /## Purpose\n\nYou are the shopping assistant for this family\./)
|
|
90
|
+
assert.match(agents, /### Responsibilities\n\nOrganize confirmed household shopping requests\./)
|
|
91
|
+
await writeFile(path.join(workspace, 'AGENTS.md'), 'My evolving purpose\n')
|
|
76
92
|
await initializeWorkspace(workspace, purpose)
|
|
77
|
-
assert.
|
|
93
|
+
assert.match(await readFile(path.join(workspace, 'AGENTS.md'), 'utf8'), /My evolving purpose/)
|
|
78
94
|
} finally { await rm(root, {recursive:true, force:true}) }
|
|
79
95
|
})
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { spawn } from 'node:child_process'
|
|
3
|
-
import { createRequire } from 'node:module'
|
|
4
|
-
import { dirname, join } from 'node:path'
|
|
5
|
-
import { fileURLToPath } from 'node:url'
|
|
6
|
-
const here=dirname(fileURLToPath(import.meta.url)),require=createRequire(import.meta.url),tsx=require.resolve('tsx'),entry=join(here,'..','src','workforce-watch-cli.ts')
|
|
7
|
-
const child=spawn(process.execPath,['--import',tsx,entry],{stdio:'inherit'})
|
|
8
|
-
child.on('exit',(code,signal)=>{if(signal)process.kill(process.pid,signal);process.exit(code??1)})
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
services:
|
|
2
|
-
workforce-watch:
|
|
3
|
-
image: ${EZ_WORKFORCE_WATCH_IMAGE:-ezenciel-agents:local}
|
|
4
|
-
build:
|
|
5
|
-
context: .
|
|
6
|
-
target: workforce-watch
|
|
7
|
-
init: true
|
|
8
|
-
restart: unless-stopped
|
|
9
|
-
read_only: true
|
|
10
|
-
cap_drop: [ALL]
|
|
11
|
-
security_opt: [no-new-privileges:true]
|
|
12
|
-
environment:
|
|
13
|
-
EZ_WATCH_PORT: 8080
|
|
14
|
-
EZ_WATCH_ENROLL_TOKEN: ${EZ_WATCH_ENROLL_TOKEN:?Set a private enrollment token}
|
|
15
|
-
EZ_WATCH_TELEGRAM_CHAT_ID: ${EZ_WATCH_TELEGRAM_CHAT_ID:-}
|
|
16
|
-
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
|
|
17
|
-
secrets: [workforce_watch_pagerduty]
|
|
18
|
-
volumes:
|
|
19
|
-
- workforce-watch-state:/state
|
|
20
|
-
tmpfs: [/tmp]
|
|
21
|
-
ports:
|
|
22
|
-
- ${EZ_WATCH_BIND_ADDRESS:-127.0.0.1}:${EZ_WATCH_PORT:-9919}:8080
|
|
23
|
-
healthcheck:
|
|
24
|
-
test: [CMD, node, -e, "fetch('http://127.0.0.1:8080/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
|
|
25
|
-
interval: 30s
|
|
26
|
-
timeout: 5s
|
|
27
|
-
retries: 3
|
|
28
|
-
start_period: 10s
|
|
29
|
-
volumes:
|
|
30
|
-
workforce-watch-state:
|
|
31
|
-
secrets:
|
|
32
|
-
workforce_watch_pagerduty:
|
|
33
|
-
file: ${EZ_WATCH_PAGERDUTY_SECRET_FILE:?Set EZ_WATCH_PAGERDUTY_SECRET_FILE to the protected workforce PagerDuty routing-key file}
|
package/docker/recovery.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { RunStore } from '../src/runs.js'
|
|
2
|
-
|
|
3
|
-
// Called only after the deployment's kernel writer lock is held. No worker
|
|
4
|
-
// from a previous container can still own this deployment; PIDs may be reused.
|
|
5
|
-
export const recoverInterruptedRuns = async (controlDir: string, channelBackend = false) => {
|
|
6
|
-
const store = new RunStore(controlDir)
|
|
7
|
-
for (const run of await store.list()) {
|
|
8
|
-
if (run.status === 'running')
|
|
9
|
-
await store.patch(run.id, { status: channelBackend && !run.pid ? 'queued' : 'failed', endedAt: new Date().toISOString() })
|
|
10
|
-
}
|
|
11
|
-
}
|
package/docs/pagerduty.md
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# PagerDuty critical-outage paging
|
|
2
|
-
|
|
3
|
-
PagerDuty alerting is an optional core capability. It observes Stocks from the
|
|
4
|
-
Ez runtime and sends Events API v2 trigger and resolve events; Stocks never
|
|
5
|
-
receives the PagerDuty routing key.
|
|
6
|
-
|
|
7
|
-
Create an Events API v2 integration in the PagerDuty service that should page
|
|
8
|
-
you. Put only its routing key in the deployment's private `relay.env` file:
|
|
9
|
-
|
|
10
|
-
```dotenv
|
|
11
|
-
PAGERDUTY_ROUTING_KEY=<Events API v2 routing key>
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Then add the tunnel-visible endpoint and monitor settings to the deployment's
|
|
15
|
-
non-secret `docker.env` file:
|
|
16
|
-
|
|
17
|
-
```dotenv
|
|
18
|
-
EZ_PAGERDUTY_STOCKS_HEALTH_URL=http://10.97.0.1:3006/health/critical
|
|
19
|
-
EZ_PAGERDUTY_POLL_SECONDS=30
|
|
20
|
-
EZ_PAGERDUTY_FAILURE_THRESHOLD=3
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
The core checks the endpoint every 30 seconds by default. Three consecutive
|
|
24
|
-
unhealthy checks trigger one incident with the stable deduplication key
|
|
25
|
-
`ez:stocks:critical-health`; recovery resolves that same incident. A failed
|
|
26
|
-
PagerDuty submission remains eligible for a later trigger. On startup, the first
|
|
27
|
-
healthy check resolves any incident left by the previous process, including an
|
|
28
|
-
uncertain trigger delivery. Failed resolves retry without treating healthy
|
|
29
|
-
Stocks as an outage. No key or health
|
|
30
|
-
response is written to agent workspaces, executor environments, or logs.
|
|
31
|
-
|
|
32
|
-
`/health/critical` is intentionally narrower than Stocks `/health`: it pages
|
|
33
|
-
only when the Stocks source worker is unavailable. Provider freshness and other
|
|
34
|
-
research-quality degradation remain visible in normal Stocks health/status but
|
|
35
|
-
do not page. The endpoint returns only a status and component summary and must
|
|
36
|
-
be reachable from the core runtime over the private tunnel. Run core outside the
|
|
37
|
-
Stocks VM if you want a VM outage to be page-worthy.
|
|
38
|
-
|
|
39
|
-
After deployment, use PagerDuty's test/escalation tooling or temporarily point
|
|
40
|
-
the monitor at an intentionally unreachable private test target, then restore
|
|
41
|
-
the real target and confirm the PagerDuty incident resolves. This is an explicit
|
|
42
|
-
operational test and may notify your escalation policy.
|
package/docs/workforce-watch.md
DELETED
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
# Workforce Watch
|
|
2
|
-
|
|
3
|
-
Workforce Watch is a separate, small Docker service for missed agent check-ins
|
|
4
|
-
and explicit terminal failures. It is not an agent, scheduler, log store or
|
|
5
|
-
remote-execution channel. It retains only the five latest compact activity
|
|
6
|
-
records supplied by each enrolled worker.
|
|
7
|
-
|
|
8
|
-
Run it in a failure domain separate from the workers it watches. In particular,
|
|
9
|
-
it must not rely on the Stocks VM to report a Stocks VM outage, and a separate
|
|
10
|
-
host is required to detect a Mac-wide outage.
|
|
11
|
-
|
|
12
|
-
## Start the monitor
|
|
13
|
-
|
|
14
|
-
Set private monitor-only values, never agent-workspace or source-control values:
|
|
15
|
-
|
|
16
|
-
```text
|
|
17
|
-
EZ_WATCH_ENROLL_TOKEN=<private fleet enrollment secret>
|
|
18
|
-
EZ_WATCH_PAGERDUTY_SECRET_FILE=/absolute/path/to/a-0600-workforce-pagerduty-key
|
|
19
|
-
TELEGRAM_BOT_TOKEN=<alert bot token>
|
|
20
|
-
EZ_WATCH_TELEGRAM_CHAT_ID=<owner chat id>
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
PagerDuty is the fleet incident owner. Workforce Watch sends a `trigger` for
|
|
24
|
-
each missed check-in or terminal failure, then a `resolve` with the stable
|
|
25
|
-
deduplication key `ez:workforce:<worker-id>` after sustained recovery. Use one
|
|
26
|
-
approved Events API v2 routing key for the fleet; it may be the existing
|
|
27
|
-
PagerDuty integration when that is the desired escalation policy. Never put it
|
|
28
|
-
in a worker's environment. The secret file contains only that key, is mode
|
|
29
|
-
`0600`, and is mounted as `/run/secrets/workforce_watch_pagerduty`; it is never
|
|
30
|
-
a Compose or container environment value. Telegram is optional, supplementary
|
|
31
|
-
owner visibility. PagerDuty is required: a Watch deployment without its
|
|
32
|
-
routing-key secret fails closed rather than silently running without paging.
|
|
33
|
-
|
|
34
|
-
The owner may explicitly authorize the CTO bot token temporarily. A dedicated
|
|
35
|
-
alert bot remains preferable because it preserves an independent delivery identity.
|
|
36
|
-
|
|
37
|
-
```sh
|
|
38
|
-
docker compose -f compose.workforce-watch.yaml up -d --build
|
|
39
|
-
curl http://127.0.0.1:9919/healthz
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
The default bind is loopback. Put private TLS networking or a private reverse
|
|
43
|
-
proxy in front of it before remote workers use it; do not expose enrollment or
|
|
44
|
-
check-in traffic publicly.
|
|
45
|
-
|
|
46
|
-
## Enroll and check in
|
|
47
|
-
|
|
48
|
-
The worker selects its monitor through its configured URL. It submits the fleet
|
|
49
|
-
enrollment secret once; the response contains the worker-specific secret. Store
|
|
50
|
-
that response only in the worker's private service environment and discard it
|
|
51
|
-
from shell history and logs.
|
|
52
|
-
|
|
53
|
-
```sh
|
|
54
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/enroll" \
|
|
55
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN" \
|
|
56
|
-
-H 'content-type: application/json' \
|
|
57
|
-
--data '{"workerId":"stocks-production","checkInSeconds":300,"graceSeconds":180,"severity":"critical"}'
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
The supervisor, not the LLM, posts check-ins using the returned worker secret:
|
|
61
|
-
|
|
62
|
-
```sh
|
|
63
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/workers/stocks-production/check-in" \
|
|
64
|
-
-H "Authorization: Bearer $EZ_WATCH_WORKER_TOKEN" \
|
|
65
|
-
-H 'content-type: application/json' \
|
|
66
|
-
--data '{"status":"ok","activity":"strategy receipt delivered","runId":"daily-strategy-2026-09-11"}'
|
|
67
|
-
```
|
|
68
|
-
|
|
69
|
-
An unrecoverable condition sends `{"status":"failed","terminal":true,...}`.
|
|
70
|
-
Optional `activity`, `error`, `runId`, and `logsHint` fields are size-limited and
|
|
71
|
-
appear in an alert. They must be public-safe operational context, never stdout,
|
|
72
|
-
prompts, holdings, credentials, or provider payloads.
|
|
73
|
-
|
|
74
|
-
## Inspect
|
|
75
|
-
|
|
76
|
-
The enrollment token reads redacted state; worker secrets never appear:
|
|
77
|
-
|
|
78
|
-
```sh
|
|
79
|
-
curl --fail-with-body "$EZ_WATCH_URL/v1/workers" \
|
|
80
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN"
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
For host replacement or a suspected worker-secret exposure, rotate that worker's
|
|
84
|
-
secret with the enrollment token. The old secret stops working immediately:
|
|
85
|
-
|
|
86
|
-
```sh
|
|
87
|
-
curl --fail-with-body -X POST "$EZ_WATCH_URL/v1/workers/stocks-production/rotate" \
|
|
88
|
-
-H "Authorization: Bearer $EZ_WATCH_ENROLL_TOKEN"
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
Store the returned worker secret in the replacement supervisor before stopping
|
|
92
|
-
the prior one.
|
|
93
|
-
|
|
94
|
-
The service triggers once for a missed deadline or terminal failure. It sends a
|
|
95
|
-
recovery notice only after two clean check-ins by default to prevent flapping.
|
|
96
|
-
|
|
97
|
-
If a worker fails again during a partially delivered recovery, channels that
|
|
98
|
-
already accepted recovery receive a fresh trigger. Channels still open retain
|
|
99
|
-
their delivery state, so they do not receive duplicate opening alerts. This
|
|
100
|
-
applies to terminal failures, failed check-ins, and missed check-ins, including
|
|
101
|
-
after a Watch restart.
|
package/src/pagerduty.ts
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
export type PagerDutyStocksMonitorOptions = {
|
|
2
|
-
routingKey: string
|
|
3
|
-
healthUrl: string
|
|
4
|
-
pollMs: number
|
|
5
|
-
failureThreshold: number
|
|
6
|
-
fetcher?: typeof fetch
|
|
7
|
-
onError?: (error: Error) => void
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
const PAGERDUTY_EVENTS_URL = 'https://events.pagerduty.com/v2/enqueue'
|
|
11
|
-
const STOCKS_DEDUP_KEY = 'ez:stocks:critical-health'
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Monitors the deliberately narrow Stocks critical-health endpoint. It keeps
|
|
15
|
-
* state in memory on purpose: PagerDuty's deduplication key is the durable
|
|
16
|
-
* incident authority, while a restart should need a fresh sustained failure.
|
|
17
|
-
*/
|
|
18
|
-
export class PagerDutyStocksMonitor {
|
|
19
|
-
private readonly fetcher: typeof fetch
|
|
20
|
-
private consecutiveFailures = 0
|
|
21
|
-
private incidentOpen = false
|
|
22
|
-
private recoveryPending = true
|
|
23
|
-
private checking = false
|
|
24
|
-
private timer?: ReturnType<typeof setInterval>
|
|
25
|
-
|
|
26
|
-
constructor(private readonly options: PagerDutyStocksMonitorOptions) {
|
|
27
|
-
this.fetcher = options.fetcher ?? fetch
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
start(): void {
|
|
31
|
-
if (this.timer) return
|
|
32
|
-
void this.check()
|
|
33
|
-
this.timer = setInterval(() => { void this.check() }, this.options.pollMs)
|
|
34
|
-
this.timer.unref()
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
stop(): void {
|
|
38
|
-
if (this.timer) clearInterval(this.timer)
|
|
39
|
-
this.timer = undefined
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async check(): Promise<void> {
|
|
43
|
-
if (this.checking) return
|
|
44
|
-
this.checking = true
|
|
45
|
-
try {
|
|
46
|
-
const response = await this.fetcher(this.options.healthUrl, {
|
|
47
|
-
headers: { accept: 'application/json' },
|
|
48
|
-
redirect: 'error',
|
|
49
|
-
signal: AbortSignal.timeout(5_000),
|
|
50
|
-
})
|
|
51
|
-
if (!response.ok) throw new Error(`Stocks health returned HTTP ${response.status}`)
|
|
52
|
-
const health: unknown = await response.json()
|
|
53
|
-
if (!health || typeof health !== 'object' || (health as { status?: unknown }).status !== 'ok')
|
|
54
|
-
throw new Error('Stocks critical health is not ok')
|
|
55
|
-
|
|
56
|
-
this.consecutiveFailures = 0
|
|
57
|
-
if (this.recoveryPending) {
|
|
58
|
-
try {
|
|
59
|
-
await this.send('resolve')
|
|
60
|
-
this.incidentOpen = false
|
|
61
|
-
this.recoveryPending = false
|
|
62
|
-
} catch (error) { this.report(error) }
|
|
63
|
-
}
|
|
64
|
-
} catch (error) {
|
|
65
|
-
this.consecutiveFailures += 1
|
|
66
|
-
if (!this.incidentOpen && this.consecutiveFailures >= this.options.failureThreshold) {
|
|
67
|
-
try {
|
|
68
|
-
this.recoveryPending = true
|
|
69
|
-
await this.send('trigger')
|
|
70
|
-
this.incidentOpen = true
|
|
71
|
-
} catch (sendError) {
|
|
72
|
-
this.report(sendError)
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
this.report(error)
|
|
76
|
-
} finally {
|
|
77
|
-
this.checking = false
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
private async send(action: 'trigger' | 'resolve'): Promise<void> {
|
|
82
|
-
const response = await this.fetcher(PAGERDUTY_EVENTS_URL, {
|
|
83
|
-
method: 'POST',
|
|
84
|
-
headers: { 'content-type': 'application/json' },
|
|
85
|
-
body: JSON.stringify({
|
|
86
|
-
routing_key: this.options.routingKey,
|
|
87
|
-
event_action: action,
|
|
88
|
-
dedup_key: STOCKS_DEDUP_KEY,
|
|
89
|
-
payload: {
|
|
90
|
-
summary: action === 'trigger'
|
|
91
|
-
? 'Stocks critical health is unavailable'
|
|
92
|
-
: 'Stocks critical health recovered',
|
|
93
|
-
source: 'ez-core',
|
|
94
|
-
severity: 'critical',
|
|
95
|
-
component: 'stocks',
|
|
96
|
-
custom_details: { failed_checks: this.consecutiveFailures },
|
|
97
|
-
},
|
|
98
|
-
}),
|
|
99
|
-
redirect: 'error',
|
|
100
|
-
signal: AbortSignal.timeout(5_000),
|
|
101
|
-
})
|
|
102
|
-
if (!response.ok) throw new Error(`PagerDuty Events API returned HTTP ${response.status}`)
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
private report(error: unknown): void {
|
|
106
|
-
const message = error instanceof Error ? error.message : 'unknown error'
|
|
107
|
-
this.options.onError?.(new Error(message))
|
|
108
|
-
}
|
|
109
|
-
}
|