@prohost/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +229 -0
- package/dist/agent/api.d.ts +45 -0
- package/dist/agent/api.js +80 -0
- package/dist/agent/command.d.ts +10 -0
- package/dist/agent/command.js +109 -0
- package/dist/agent/contract.d.ts +94 -0
- package/dist/agent/contract.js +125 -0
- package/dist/agent/credentials.d.ts +72 -0
- package/dist/agent/credentials.js +92 -0
- package/dist/agent/exec.d.ts +48 -0
- package/dist/agent/exec.js +151 -0
- package/dist/agent/pair.d.ts +44 -0
- package/dist/agent/pair.js +149 -0
- package/dist/agent/prompt.d.ts +24 -0
- package/dist/agent/prompt.js +77 -0
- package/dist/agent/run.d.ts +78 -0
- package/dist/agent/run.js +400 -0
- package/dist/agent/runstore.d.ts +50 -0
- package/dist/agent/runstore.js +107 -0
- package/dist/agent/signature.d.ts +35 -0
- package/dist/agent/signature.js +99 -0
- package/dist/forward.d.ts +39 -0
- package/dist/forward.js +68 -0
- package/dist/index.d.ts +51 -0
- package/dist/index.js +216 -0
- package/dist/listen.d.ts +48 -0
- package/dist/listen.js +113 -0
- package/dist/version.d.ts +12 -0
- package/dist/version.js +12 -0
- package/package.json +52 -0
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `prohost agent run` — the event loop that makes a local agent a teammate.
|
|
3
|
+
*
|
|
4
|
+
* Connects to the same authenticated `/ws/webhooks` stream `prohost listen`
|
|
5
|
+
* uses, and for every `agent.run_requested` frame: verifies the HMAC, builds a
|
|
6
|
+
* prompt, runs the host's ``--exec`` command with that prompt on stdin, posts
|
|
7
|
+
* stdout back into the conversation, and closes the run out. A run is *always*
|
|
8
|
+
* completed — a crashed or timed-out command reports ``failed`` rather than
|
|
9
|
+
* being left hanging.
|
|
10
|
+
*
|
|
11
|
+
* Two deliberate safety properties:
|
|
12
|
+
*
|
|
13
|
+
* * **Runs are serialized.** One local agent, one run at a time; concurrent
|
|
14
|
+
* frames queue. Agents hold working directories and rate limits, and running
|
|
15
|
+
* several at once is a good way to corrupt both.
|
|
16
|
+
* * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
|
|
17
|
+
* attempt is mirrored to this socket. Without dedupe a retry would execute
|
|
18
|
+
* the command a second time and post a second reply.
|
|
19
|
+
*/
|
|
20
|
+
import os from 'node:os';
|
|
21
|
+
import WebSocket from 'ws';
|
|
22
|
+
import { completeRun, postReply } from './api.js';
|
|
23
|
+
import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, eventData, parseRunRequest, } from './contract.js';
|
|
24
|
+
import { execAgent } from './exec.js';
|
|
25
|
+
import { buildPrompt, replyFromStdout } from './prompt.js';
|
|
26
|
+
import { RunStore } from './runstore.js';
|
|
27
|
+
import { verifySignature } from './signature.js';
|
|
28
|
+
import { isWebhookEvent } from '../forward.js';
|
|
29
|
+
import { DEFAULT_WS_URL } from '../listen.js';
|
|
30
|
+
/** Default ceiling on one agent invocation. */
|
|
31
|
+
export const DEFAULT_EXEC_TIMEOUT_MS = 300_000;
|
|
32
|
+
/** Reconnect backoff — full jitter between ``BASE`` and the current cap. */
|
|
33
|
+
const RECONNECT_BASE_DELAY_MS = 1_000;
|
|
34
|
+
const RECONNECT_MAX_DELAY_MS = 30_000;
|
|
35
|
+
/** Ack status codes reported back over the socket for each frame. */
|
|
36
|
+
const ACK_ACCEPTED = 200;
|
|
37
|
+
const ACK_IGNORED = 204;
|
|
38
|
+
const ACK_REJECTED = 401;
|
|
39
|
+
function newMetrics() {
|
|
40
|
+
return {
|
|
41
|
+
connections: 0,
|
|
42
|
+
eventsReceived: 0,
|
|
43
|
+
runsStarted: 0,
|
|
44
|
+
runsSucceeded: 0,
|
|
45
|
+
runsFailed: 0,
|
|
46
|
+
signatureRejections: 0,
|
|
47
|
+
ignored: 0,
|
|
48
|
+
duplicates: 0,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function timestamp() {
|
|
52
|
+
return new Date().toISOString().slice(11, 19);
|
|
53
|
+
}
|
|
54
|
+
function defaultSleep(ms) {
|
|
55
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
56
|
+
}
|
|
57
|
+
/** Full-jitter exponential backoff, capped. */
|
|
58
|
+
function backoffDelay(attempt) {
|
|
59
|
+
const cap = Math.min(RECONNECT_MAX_DELAY_MS, RECONNECT_BASE_DELAY_MS * 2 ** Math.max(0, attempt - 1));
|
|
60
|
+
return Math.floor(Math.random() * cap);
|
|
61
|
+
}
|
|
62
|
+
/** Backoff schedule for retrying a completion the server didn't accept. */
|
|
63
|
+
const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000];
|
|
64
|
+
/**
|
|
65
|
+
* Complete a run, retrying transient failures.
|
|
66
|
+
*
|
|
67
|
+
* Completion is the one call that must land: nothing else can finalize an
|
|
68
|
+
* external run, so a dropped one leaves the row `running` — spinner up, the
|
|
69
|
+
* agent's next wake-up on that surface suppressed — until the 2h stale-run
|
|
70
|
+
* sweeper reaps it. A 409 (`already_resolved`) counts as done. A 4xx other
|
|
71
|
+
* than 429 is a decision, not a blip, so it isn't retried.
|
|
72
|
+
*/
|
|
73
|
+
async function completeWithRetry(api, path, outcome, log, sleep) {
|
|
74
|
+
let result = await completeRun(api, path, outcome);
|
|
75
|
+
for (const delay of COMPLETION_RETRY_DELAYS_MS) {
|
|
76
|
+
if (result.ok)
|
|
77
|
+
return result;
|
|
78
|
+
const retryable = result.status === 0 || result.status === 429 || result.status >= 500;
|
|
79
|
+
if (!retryable)
|
|
80
|
+
return result;
|
|
81
|
+
log(` … completion failed (HTTP ${result.status}); retrying in ${delay / 1000}s`);
|
|
82
|
+
await sleep(delay);
|
|
83
|
+
result = await completeRun(api, path, outcome);
|
|
84
|
+
}
|
|
85
|
+
return result;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Handle one verified run end-to-end.
|
|
89
|
+
*
|
|
90
|
+
* Every path except a dry run — including every failure — calls the completion
|
|
91
|
+
* endpoint before returning.
|
|
92
|
+
*/
|
|
93
|
+
async function handleRun(run, options, api, log, sleep) {
|
|
94
|
+
const label = `run ${run.run_id.slice(0, 12)}`;
|
|
95
|
+
const surface = run.conversation_id ?? run.kind ?? 'no conversation';
|
|
96
|
+
log(`${timestamp()} ▶ ${label} started (${surface})`);
|
|
97
|
+
const prompt = buildPrompt(run, {
|
|
98
|
+
agentName: options.credentials.agent?.name ?? 'your agent',
|
|
99
|
+
mcpUrl: options.credentials.mcp_url,
|
|
100
|
+
});
|
|
101
|
+
const result = await execAgent({
|
|
102
|
+
command: options.exec,
|
|
103
|
+
prompt,
|
|
104
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
|
|
105
|
+
cwd: options.cwd,
|
|
106
|
+
spawnImpl: options.spawnImpl,
|
|
107
|
+
});
|
|
108
|
+
const seconds = (result.durationMs / 1000).toFixed(1);
|
|
109
|
+
if (!result.ok) {
|
|
110
|
+
const reason = result.error ??
|
|
111
|
+
(result.exitCode === null
|
|
112
|
+
? 'agent command did not run'
|
|
113
|
+
: `agent command exited ${result.exitCode}`);
|
|
114
|
+
const detail = result.stderr.trim().slice(-500);
|
|
115
|
+
log(`${timestamp()} ✗ ${label} failed after ${seconds}s — ${reason}`);
|
|
116
|
+
if (detail)
|
|
117
|
+
log(` stderr: ${detail}`);
|
|
118
|
+
if (options.dryRun) {
|
|
119
|
+
log(` [dry-run] would report the run as failed`);
|
|
120
|
+
return 'skipped';
|
|
121
|
+
}
|
|
122
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: false, error: detail ? `${reason}: ${detail}` : reason }, log, sleep);
|
|
123
|
+
if (!completed.ok) {
|
|
124
|
+
log(` ! could not report failure (HTTP ${completed.status}) ${completed.error ?? ''}`);
|
|
125
|
+
return 'unreported';
|
|
126
|
+
}
|
|
127
|
+
return 'failed';
|
|
128
|
+
}
|
|
129
|
+
const reply = replyFromStdout(result.stdout);
|
|
130
|
+
if (!reply) {
|
|
131
|
+
log(`${timestamp()} ✗ ${label} produced no output after ${seconds}s`);
|
|
132
|
+
if (options.dryRun)
|
|
133
|
+
return 'skipped';
|
|
134
|
+
const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: 'agent produced no output' }, log, sleep);
|
|
135
|
+
return reported.ok ? 'failed' : 'unreported';
|
|
136
|
+
}
|
|
137
|
+
if (options.dryRun) {
|
|
138
|
+
const destination = run.reply_surface === 'none' ? '(no reply surface)' : `${run.reply_surface} ${run.reply_path}`;
|
|
139
|
+
log(`${timestamp()} ◦ ${label} dry-run — would post to ${destination}:`);
|
|
140
|
+
for (const line of reply.split('\n'))
|
|
141
|
+
log(` │ ${line}`);
|
|
142
|
+
log(` [dry-run] nothing was sent and the run was left open`);
|
|
143
|
+
return 'skipped';
|
|
144
|
+
}
|
|
145
|
+
// A run with no reply surface is a legitimate outcome, not an error: it was
|
|
146
|
+
// started by a trigger rather than by someone addressing the agent, so there
|
|
147
|
+
// is nowhere to post. Complete it as succeeded so any thinking indicator
|
|
148
|
+
// clears.
|
|
149
|
+
if (!run.reply_path || !run.reply_body_key) {
|
|
150
|
+
const why = run.reply_surface === 'none'
|
|
151
|
+
? 'no reply surface'
|
|
152
|
+
: `${run.reply_surface} runs have nowhere to post yet`;
|
|
153
|
+
log(`${timestamp()} ✓ ${label} finished in ${seconds}s (${why})`);
|
|
154
|
+
for (const line of reply.split('\n'))
|
|
155
|
+
log(` │ ${line}`);
|
|
156
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep);
|
|
157
|
+
if (!completed.ok) {
|
|
158
|
+
log(` ! could not mark the run complete (HTTP ${completed.status})`);
|
|
159
|
+
return 'unreported';
|
|
160
|
+
}
|
|
161
|
+
return 'succeeded';
|
|
162
|
+
}
|
|
163
|
+
const posted = await postReply(api, run.reply_path, run.reply_body_key, reply, run.run_id);
|
|
164
|
+
if (!posted.ok) {
|
|
165
|
+
log(`${timestamp()} ✗ ${label} reply rejected (HTTP ${posted.status}) ${posted.error ?? ''}`);
|
|
166
|
+
const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed with HTTP ${posted.status}` }, log, sleep);
|
|
167
|
+
return reported.ok ? 'failed' : 'unreported';
|
|
168
|
+
}
|
|
169
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep);
|
|
170
|
+
if (!completed.ok) {
|
|
171
|
+
// The reply is already in the conversation, so this is a bookkeeping
|
|
172
|
+
// failure, not a lost answer — say so precisely. Returning false lets the
|
|
173
|
+
// caller forget the run so a redelivery can retry the completion; the
|
|
174
|
+
// reply's idempotency key stops that from posting twice.
|
|
175
|
+
log(`${timestamp()} ! ${label} replied, but marking it complete failed (HTTP ${completed.status})`);
|
|
176
|
+
return 'unreported';
|
|
177
|
+
}
|
|
178
|
+
log(`${timestamp()} ✓ ${label} replied in ${seconds}s (${reply.length} chars)`);
|
|
179
|
+
return 'succeeded';
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Run the harness until the abort signal fires (or ``maxConnections`` is hit).
|
|
183
|
+
*
|
|
184
|
+
* Reconnects on any drop with jittered exponential backoff; an ``auth_error``
|
|
185
|
+
* is fatal, because retrying a rejected credential just hammers the server.
|
|
186
|
+
*/
|
|
187
|
+
export async function runAgentHarness(options) {
|
|
188
|
+
const log = options.onLine ?? ((line) => process.stdout.write(`${line}\n`));
|
|
189
|
+
const Ctor = options.webSocketCtor ?? WebSocket;
|
|
190
|
+
const sleep = options.sleepImpl ?? defaultSleep;
|
|
191
|
+
const machine = options.machine ?? os.hostname();
|
|
192
|
+
const url = options.url ?? options.credentials.ws_url ?? DEFAULT_WS_URL;
|
|
193
|
+
const metrics = newMetrics();
|
|
194
|
+
const api = {
|
|
195
|
+
baseUrl: options.credentials.base_url,
|
|
196
|
+
apiKey: options.credentials.api_key,
|
|
197
|
+
fetchImpl: options.fetchImpl,
|
|
198
|
+
};
|
|
199
|
+
// Dedupe survives restarts: a run is recorded before its command starts, so a
|
|
200
|
+
// redelivery can never execute the agent twice.
|
|
201
|
+
const runs = RunStore.load(options.env, log);
|
|
202
|
+
// Runs this process has taken on. A redelivery for one of these is just a
|
|
203
|
+
// duplicate — the run is in flight right here. Only a run in the ledger that
|
|
204
|
+
// this process never handled is a candidate for restart recovery.
|
|
205
|
+
const handledThisSession = new Set();
|
|
206
|
+
let queue = Promise.resolve();
|
|
207
|
+
let fatal;
|
|
208
|
+
let attempt = 0;
|
|
209
|
+
const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
|
|
210
|
+
while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
|
|
211
|
+
metrics.connections += 1;
|
|
212
|
+
const authedThisTime = await connectOnce();
|
|
213
|
+
if (options.signal?.aborted || fatal !== undefined || exhausted())
|
|
214
|
+
break;
|
|
215
|
+
// A session that never authed is more likely a persistent problem than a
|
|
216
|
+
// blip, so only a session that did auth resets the backoff.
|
|
217
|
+
attempt = authedThisTime ? 1 : attempt + 1;
|
|
218
|
+
const delay = backoffDelay(attempt);
|
|
219
|
+
log(`${timestamp()} … disconnected; reconnecting in ${(delay / 1000).toFixed(1)}s`);
|
|
220
|
+
await sleep(delay);
|
|
221
|
+
}
|
|
222
|
+
// Let an in-flight run finish reporting before we hand back metrics.
|
|
223
|
+
await queue;
|
|
224
|
+
if (fatal !== undefined) {
|
|
225
|
+
log(`${timestamp()} ✗ ${fatal}`);
|
|
226
|
+
metrics.fatal = fatal;
|
|
227
|
+
}
|
|
228
|
+
return metrics;
|
|
229
|
+
/** One socket lifetime. Resolves ``true`` when the session ever authed. */
|
|
230
|
+
function connectOnce() {
|
|
231
|
+
const ws = new Ctor(url);
|
|
232
|
+
let authed = false;
|
|
233
|
+
return new Promise((resolve) => {
|
|
234
|
+
const close = () => {
|
|
235
|
+
try {
|
|
236
|
+
ws.close();
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
/* already closing */
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
if (options.signal) {
|
|
243
|
+
if (options.signal.aborted)
|
|
244
|
+
close();
|
|
245
|
+
else
|
|
246
|
+
options.signal.addEventListener('abort', close, { once: true });
|
|
247
|
+
}
|
|
248
|
+
ws.on('open', () => {
|
|
249
|
+
// `subscription_id` is a presence signal, not a scoping one: the server
|
|
250
|
+
// derives which subscription's events this socket receives from the
|
|
251
|
+
// connection itself. Sending it stamps `connected_from` with this
|
|
252
|
+
// machine's name, which is what the UI shows and what the delivery
|
|
253
|
+
// path treats as "someone is listening".
|
|
254
|
+
ws.send(JSON.stringify({
|
|
255
|
+
type: 'auth',
|
|
256
|
+
api_key: options.credentials.api_key,
|
|
257
|
+
machine,
|
|
258
|
+
...(options.credentials.subscription_id
|
|
259
|
+
? { subscription_id: options.credentials.subscription_id }
|
|
260
|
+
: {}),
|
|
261
|
+
}));
|
|
262
|
+
});
|
|
263
|
+
ws.on('message', (data) => {
|
|
264
|
+
let frame;
|
|
265
|
+
try {
|
|
266
|
+
frame = JSON.parse(data.toString());
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (!frame || typeof frame !== 'object')
|
|
272
|
+
return;
|
|
273
|
+
const f = frame;
|
|
274
|
+
if (f.type === 'auth_ok') {
|
|
275
|
+
authed = true;
|
|
276
|
+
const name = options.credentials.agent?.name ?? 'agent';
|
|
277
|
+
log(`${timestamp()} ● connected as "${name}" — waiting for mentions (exec: ${options.exec})`);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (f.type === 'auth_error') {
|
|
281
|
+
const reason = String(f.reason ?? 'unknown');
|
|
282
|
+
const hint = reason === 'insufficient_scope'
|
|
283
|
+
? 'This credential is not allowed to stream events — ask your ProhostAI ' +
|
|
284
|
+
'admin to re-issue the pairing with event-stream access.'
|
|
285
|
+
: reason === 'stream_subscription_unavailable'
|
|
286
|
+
? 'This connection has no event subscription, so there is nothing to ' +
|
|
287
|
+
'stream. Re-pair: prohost agent pair --code <code>'
|
|
288
|
+
: 'Re-pair with: prohost agent pair --code <code>';
|
|
289
|
+
fatal = `auth_error: ${reason}. ${hint}`;
|
|
290
|
+
close();
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (f.type === 'ping') {
|
|
294
|
+
ws.send(JSON.stringify({ type: 'pong' }));
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
if (!isWebhookEvent(frame))
|
|
298
|
+
return;
|
|
299
|
+
metrics.eventsReceived += 1;
|
|
300
|
+
const ack = (status) => {
|
|
301
|
+
try {
|
|
302
|
+
ws.send(JSON.stringify({ type: 'ack', delivery_id: frame.delivery_id, status }));
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
/* best-effort */
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
if (frame.event !== EVENT_AGENT_RUN_REQUESTED) {
|
|
309
|
+
if (frame.event === EVENT_MENTION_CREATED || frame.event === EVENT_MESSAGE_TEAM_CHAT) {
|
|
310
|
+
log(`${timestamp()} ◦ ${frame.event} (no action — runs are triggered by mentions)`);
|
|
311
|
+
}
|
|
312
|
+
metrics.ignored += 1;
|
|
313
|
+
ack(ACK_IGNORED);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
const verified = verifySignature(options.credentials.webhook_secret, frame.payload, frame.signature);
|
|
317
|
+
if (!verified && !options.allowUnverified) {
|
|
318
|
+
metrics.signatureRejections += 1;
|
|
319
|
+
log(`${timestamp()} ✗ rejected ${EVENT_AGENT_RUN_REQUESTED} — signature did not verify. ` +
|
|
320
|
+
'Not running the command. Re-pair to refresh the webhook secret, or pass ' +
|
|
321
|
+
'--allow-unverified if you accept unauthenticated triggers.');
|
|
322
|
+
ack(ACK_REJECTED);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
const run = parseRunRequest(eventData(frame.payload));
|
|
326
|
+
if (run === null) {
|
|
327
|
+
log(`${timestamp()} ◦ ${EVENT_AGENT_RUN_REQUESTED} without a run_id — ignoring`);
|
|
328
|
+
metrics.ignored += 1;
|
|
329
|
+
ack(ACK_IGNORED);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
const previous = runs.get(run.run_id);
|
|
333
|
+
if (previous?.completed || handledThisSession.has(run.run_id)) {
|
|
334
|
+
metrics.duplicates += 1;
|
|
335
|
+
ack(ACK_ACCEPTED);
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
ack(ACK_ACCEPTED);
|
|
339
|
+
handledThisSession.add(run.run_id);
|
|
340
|
+
if (previous) {
|
|
341
|
+
// We ran this one before but never got a terminal state accepted —
|
|
342
|
+
// most likely the process died between the command and the
|
|
343
|
+
// completion. Retry only the completion: re-running the agent could
|
|
344
|
+
// repeat whatever it did with its credential the first time.
|
|
345
|
+
metrics.duplicates += 1;
|
|
346
|
+
queue = queue.then(async () => {
|
|
347
|
+
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
348
|
+
'reporting it now without re-running the agent');
|
|
349
|
+
const recovered = await completeWithRetry(api, previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
350
|
+
if (recovered.ok)
|
|
351
|
+
runs.markCompleted(run.run_id);
|
|
352
|
+
});
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
// Recorded BEFORE the command runs — a crash mid-run must not let a
|
|
356
|
+
// redelivery execute the agent a second time.
|
|
357
|
+
runs.markStarted(run.run_id, run.completion_path);
|
|
358
|
+
metrics.runsStarted += 1;
|
|
359
|
+
// Serialize execution but never block the socket reader: the ack and
|
|
360
|
+
// the heartbeat have to keep flowing while the agent thinks.
|
|
361
|
+
queue = queue.then(async () => {
|
|
362
|
+
// Shutdown was requested while this sat in the queue. Don't start a
|
|
363
|
+
// fresh command the operator is waiting to be rid of — but do close
|
|
364
|
+
// the run out, so the server isn't left waiting on it either.
|
|
365
|
+
if (options.signal?.aborted) {
|
|
366
|
+
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
|
|
367
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
|
|
368
|
+
if (closed.ok)
|
|
369
|
+
runs.markCompleted(run.run_id);
|
|
370
|
+
metrics.runsFailed += 1;
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const outcome = await handleRun(run, options, api, log, sleep).catch((err) => {
|
|
374
|
+
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
375
|
+
return 'failed';
|
|
376
|
+
});
|
|
377
|
+
// Both terminal outcomes reached the completion endpoint; only then
|
|
378
|
+
// is the run settled and safe to skip on redelivery.
|
|
379
|
+
if (outcome === 'succeeded')
|
|
380
|
+
metrics.runsSucceeded += 1;
|
|
381
|
+
if (outcome === 'failed' || outcome === 'unreported')
|
|
382
|
+
metrics.runsFailed += 1;
|
|
383
|
+
// Only a run the server acknowledged as terminal is settled; an
|
|
384
|
+
// unreported one stays open so a redelivery can report it.
|
|
385
|
+
if (outcome === 'succeeded' || outcome === 'failed')
|
|
386
|
+
runs.markCompleted(run.run_id);
|
|
387
|
+
});
|
|
388
|
+
});
|
|
389
|
+
ws.on('close', () => {
|
|
390
|
+
if (options.signal)
|
|
391
|
+
options.signal.removeEventListener('abort', close);
|
|
392
|
+
resolve(authed);
|
|
393
|
+
});
|
|
394
|
+
ws.on('error', (err) => {
|
|
395
|
+
if (!authed)
|
|
396
|
+
log(`${timestamp()} ✗ connection error: ${err.message}`);
|
|
397
|
+
});
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small on-disk ledger of runs this machine has already started.
|
|
3
|
+
*
|
|
4
|
+
* In-memory dedupe is not enough. Webhook deliveries retry, delivery acks are
|
|
5
|
+
* not persisted server-side, and the CLI is a long-lived process that people
|
|
6
|
+
* restart — so a redelivery arriving after a restart would run the agent
|
|
7
|
+
* command a second time. That is not just a duplicate message (the reply send
|
|
8
|
+
* is idempotency-keyed): the agent holds a credential and may have created a
|
|
9
|
+
* task, sent an email, or moved a deal. Executing twice is the expensive
|
|
10
|
+
* mistake, so a run is recorded **before** the command starts and is never
|
|
11
|
+
* executed twice.
|
|
12
|
+
*
|
|
13
|
+
* The completion is tracked separately, because the two failure modes want
|
|
14
|
+
* opposite treatment: a run whose command ran but whose completion never landed
|
|
15
|
+
* should have its completion retried on redelivery — without re-running
|
|
16
|
+
* anything.
|
|
17
|
+
*/
|
|
18
|
+
export interface RunRecord {
|
|
19
|
+
run_id: string;
|
|
20
|
+
/** Whether the server acknowledged a terminal state for this run. */
|
|
21
|
+
completed: boolean;
|
|
22
|
+
/** Path to complete at, kept so a recovery pass needs nothing else. */
|
|
23
|
+
completion_path: string;
|
|
24
|
+
started_at: string;
|
|
25
|
+
}
|
|
26
|
+
/** Cap on retained records. Oldest are dropped first (insertion order). */
|
|
27
|
+
export declare const RUN_LEDGER_LIMIT = 500;
|
|
28
|
+
export declare function runLedgerPath(env?: NodeJS.ProcessEnv): string;
|
|
29
|
+
/**
|
|
30
|
+
* Records of started runs, backed by a JSON file.
|
|
31
|
+
*
|
|
32
|
+
* Every persistence failure degrades to memory-only rather than throwing: a
|
|
33
|
+
* read-only home directory should cost dedupe-across-restarts, not the ability
|
|
34
|
+
* to answer a mention.
|
|
35
|
+
*/
|
|
36
|
+
export declare class RunStore {
|
|
37
|
+
private readonly file;
|
|
38
|
+
private readonly onWarn;
|
|
39
|
+
private records;
|
|
40
|
+
private constructor();
|
|
41
|
+
/** Read the ledger from disk. A missing or corrupt file starts empty. */
|
|
42
|
+
static load(env?: NodeJS.ProcessEnv, onWarn?: (line: string) => void): RunStore;
|
|
43
|
+
get(runId: string): RunRecord | undefined;
|
|
44
|
+
/** Record a run as started. Call this BEFORE running the command. */
|
|
45
|
+
markStarted(runId: string, completionPath: string): void;
|
|
46
|
+
/** Record that the server accepted a terminal state for this run. */
|
|
47
|
+
markCompleted(runId: string): void;
|
|
48
|
+
private trim;
|
|
49
|
+
private persist;
|
|
50
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small on-disk ledger of runs this machine has already started.
|
|
3
|
+
*
|
|
4
|
+
* In-memory dedupe is not enough. Webhook deliveries retry, delivery acks are
|
|
5
|
+
* not persisted server-side, and the CLI is a long-lived process that people
|
|
6
|
+
* restart — so a redelivery arriving after a restart would run the agent
|
|
7
|
+
* command a second time. That is not just a duplicate message (the reply send
|
|
8
|
+
* is idempotency-keyed): the agent holds a credential and may have created a
|
|
9
|
+
* task, sent an email, or moved a deal. Executing twice is the expensive
|
|
10
|
+
* mistake, so a run is recorded **before** the command starts and is never
|
|
11
|
+
* executed twice.
|
|
12
|
+
*
|
|
13
|
+
* The completion is tracked separately, because the two failure modes want
|
|
14
|
+
* opposite treatment: a run whose command ran but whose completion never landed
|
|
15
|
+
* should have its completion retried on redelivery — without re-running
|
|
16
|
+
* anything.
|
|
17
|
+
*/
|
|
18
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { prohostHome } from './credentials.js';
|
|
21
|
+
/** Cap on retained records. Oldest are dropped first (insertion order). */
|
|
22
|
+
export const RUN_LEDGER_LIMIT = 500;
|
|
23
|
+
export function runLedgerPath(env = process.env) {
|
|
24
|
+
return path.join(prohostHome(env), 'runs.json');
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Records of started runs, backed by a JSON file.
|
|
28
|
+
*
|
|
29
|
+
* Every persistence failure degrades to memory-only rather than throwing: a
|
|
30
|
+
* read-only home directory should cost dedupe-across-restarts, not the ability
|
|
31
|
+
* to answer a mention.
|
|
32
|
+
*/
|
|
33
|
+
export class RunStore {
|
|
34
|
+
file;
|
|
35
|
+
onWarn;
|
|
36
|
+
records = new Map();
|
|
37
|
+
constructor(file, onWarn) {
|
|
38
|
+
this.file = file;
|
|
39
|
+
this.onWarn = onWarn;
|
|
40
|
+
}
|
|
41
|
+
/** Read the ledger from disk. A missing or corrupt file starts empty. */
|
|
42
|
+
static load(env = process.env, onWarn = () => { }) {
|
|
43
|
+
const store = new RunStore(runLedgerPath(env), onWarn);
|
|
44
|
+
try {
|
|
45
|
+
const parsed = JSON.parse(readFileSync(store.file, 'utf8'));
|
|
46
|
+
if (Array.isArray(parsed)) {
|
|
47
|
+
for (const entry of parsed) {
|
|
48
|
+
const record = entry;
|
|
49
|
+
if (typeof record?.run_id === 'string' && record.run_id) {
|
|
50
|
+
store.records.set(record.run_id, {
|
|
51
|
+
run_id: record.run_id,
|
|
52
|
+
completed: record.completed === true,
|
|
53
|
+
completion_path: typeof record.completion_path === 'string' ? record.completion_path : '',
|
|
54
|
+
started_at: typeof record.started_at === 'string' ? record.started_at : '',
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// No ledger yet, or an unreadable one — either way, start clean.
|
|
62
|
+
}
|
|
63
|
+
return store;
|
|
64
|
+
}
|
|
65
|
+
get(runId) {
|
|
66
|
+
return this.records.get(runId);
|
|
67
|
+
}
|
|
68
|
+
/** Record a run as started. Call this BEFORE running the command. */
|
|
69
|
+
markStarted(runId, completionPath) {
|
|
70
|
+
this.records.set(runId, {
|
|
71
|
+
run_id: runId,
|
|
72
|
+
completed: false,
|
|
73
|
+
completion_path: completionPath,
|
|
74
|
+
started_at: new Date().toISOString(),
|
|
75
|
+
});
|
|
76
|
+
this.trim();
|
|
77
|
+
this.persist();
|
|
78
|
+
}
|
|
79
|
+
/** Record that the server accepted a terminal state for this run. */
|
|
80
|
+
markCompleted(runId) {
|
|
81
|
+
const existing = this.records.get(runId);
|
|
82
|
+
if (!existing)
|
|
83
|
+
return;
|
|
84
|
+
existing.completed = true;
|
|
85
|
+
this.persist();
|
|
86
|
+
}
|
|
87
|
+
trim() {
|
|
88
|
+
while (this.records.size > RUN_LEDGER_LIMIT) {
|
|
89
|
+
const oldest = this.records.keys().next().value;
|
|
90
|
+
if (oldest === undefined)
|
|
91
|
+
return;
|
|
92
|
+
this.records.delete(oldest);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
persist() {
|
|
96
|
+
try {
|
|
97
|
+
mkdirSync(path.dirname(this.file), { recursive: true, mode: 0o700 });
|
|
98
|
+
writeFileSync(this.file, `${JSON.stringify([...this.records.values()], null, 2)}\n`, {
|
|
99
|
+
mode: 0o600,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
this.onWarn(` ! could not write the run ledger (${err instanceof Error ? err.message : String(err)}); ` +
|
|
104
|
+
'dedupe is memory-only until this is fixed');
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HMAC verification for events arriving over the websocket.
|
|
3
|
+
*
|
|
4
|
+
* A run event makes this CLI execute a local command, so the frame has to be
|
|
5
|
+
* proven to come from ProhostAI before it is acted on. HTTPS webhook receivers
|
|
6
|
+
* verify the HMAC over the raw request body; the websocket carries the payload
|
|
7
|
+
* as parsed JSON instead, so we re-serialize it in the exact form the server
|
|
8
|
+
* signed and compare.
|
|
9
|
+
*
|
|
10
|
+
* The server signs ``json.dumps(envelope, sort_keys=True, default=str)``
|
|
11
|
+
* (`shared/webhooks_outbound/signing.py::canonical_body`), which means:
|
|
12
|
+
* recursively sorted keys, Python's ``", "`` / ``": "`` separators, and
|
|
13
|
+
* ``ensure_ascii=True`` (every character outside ``0x20``–``0x7e`` escaped as
|
|
14
|
+
* ``\uXXXX``, non-BMP as a UTF-16 surrogate pair — which is what JS strings
|
|
15
|
+
* already hold). :func:`canonicalJson` reproduces that byte-for-byte.
|
|
16
|
+
*
|
|
17
|
+
* Known limit: JSON has one number type but Python does not. A float that
|
|
18
|
+
* round-trips through ``JSON.parse`` as an integer (``1.0`` → ``1``)
|
|
19
|
+
* re-serializes as ``1`` and fails verification. ProhostAI event payloads carry
|
|
20
|
+
* ids, timestamps, and text — no bare floats — so this has no practical effect;
|
|
21
|
+
* it is called out because the failure mode would otherwise look like tampering.
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* Re-create the exact JSON bytes the server signed.
|
|
25
|
+
*
|
|
26
|
+
* Mirrors ``json.dumps(value, sort_keys=True, default=str)``: sorted keys,
|
|
27
|
+
* a space after every ``:`` and ``,``, and ASCII-escaped output.
|
|
28
|
+
*/
|
|
29
|
+
export declare function canonicalJson(value: unknown): string;
|
|
30
|
+
/**
|
|
31
|
+
* Constant-time check that ``signature`` is the server's HMAC-SHA256 over
|
|
32
|
+
* ``payload``. Returns ``false`` for an empty signature or secret rather than
|
|
33
|
+
* throwing — the caller decides what an unverifiable frame means.
|
|
34
|
+
*/
|
|
35
|
+
export declare function verifySignature(secret: string | null | undefined, payload: unknown, signature: string | null | undefined): boolean;
|