@controlflow-ai/daemon 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -24
- package/package.json +16 -3
- package/src/agent-avatar.ts +30 -0
- package/src/agent-key.ts +28 -0
- package/src/agent-permissions.ts +359 -0
- package/src/agent-runtime.ts +810 -28
- package/src/agent-workspace.ts +183 -0
- package/src/app.ts +2183 -79
- package/src/args.ts +54 -7
- package/src/cli.ts +873 -14
- package/src/client.ts +482 -12
- package/src/coco.ts +9 -40
- package/src/codex.ts +33 -5
- package/src/config.ts +28 -4
- package/src/console.ts +460 -26
- package/src/daemon-client.ts +116 -3
- package/src/daemon.ts +958 -101
- package/src/db.ts +3216 -113
- package/src/delivery-ws.ts +269 -0
- package/src/format.ts +4 -1
- package/src/lark/app-registration.ts +141 -0
- package/src/lark/cli.ts +7 -137
- package/src/lark/credentials.ts +36 -3
- package/src/lark/event-router.ts +61 -5
- package/src/lark/inbound-events.ts +156 -3
- package/src/lark/server-integration.ts +659 -111
- package/src/lark/setup.ts +74 -5
- package/src/lark/ws-daemon.ts +136 -10
- package/src/local-api.ts +611 -14
- package/src/local-auth.ts +36 -3
- package/src/message-attachments.ts +71 -0
- package/src/messaging-cli.ts +741 -0
- package/src/messaging-status.ts +669 -0
- package/src/migrations/023_projects.ts +65 -0
- package/src/migrations/024_agents_model.ts +10 -0
- package/src/migrations/025_room_archive.ts +44 -0
- package/src/migrations/026_project_archive.ts +44 -0
- package/src/migrations/027_agent_permission_profiles.ts +16 -0
- package/src/migrations/028_lark_websocket_restart_state.ts +16 -0
- package/src/migrations/029_held_message_drafts.ts +32 -0
- package/src/migrations/030_agent_room_read_state.ts +25 -0
- package/src/migrations/031_room_tasks.ts +29 -0
- package/src/migrations/032_room_reminders.ts +29 -0
- package/src/migrations/033_room_saved_messages.ts +25 -0
- package/src/migrations/034_agent_activity_events.ts +27 -0
- package/src/migrations/035_agent_avatars.ts +17 -0
- package/src/migrations/036_project_agent_defaults.ts +21 -0
- package/src/migrations/037_message_attachments.ts +36 -0
- package/src/migrations/038_agent_activity_room_scope.ts +64 -0
- package/src/migrations/039_message_attachments_path.ts +34 -0
- package/src/migrations/040_message_attachments_file_schema.ts +80 -0
- package/src/migrations/041_room_system_events.ts +30 -0
- package/src/migrations/042_message_attachment_file_kind.ts +52 -0
- package/src/migrations/043_room_mode_skill_registry.ts +92 -0
- package/src/migrations/044_workflow_runtime.ts +69 -0
- package/src/migrations/045_skill_repository_ownership.ts +64 -0
- package/src/migrations.ts +70 -1
- package/src/neeko.ts +40 -4
- package/src/runtime-env.ts +179 -0
- package/src/runtime-registry.ts +83 -13
- package/src/server.ts +244 -4
- package/src/token-file.ts +13 -6
- package/src/types.ts +394 -0
- package/src/workflow-runtime.ts +275 -0
- package/src/web.ts +0 -904
package/src/daemon.ts
CHANGED
|
@@ -6,10 +6,14 @@ import { DEFAULT_DAEMON_PORT, DEFAULT_HOST, agentHomePath, defaultDaemonToken, d
|
|
|
6
6
|
import { writeFailureMessage } from './failure-message.js';
|
|
7
7
|
import { formatMessage } from './format.js';
|
|
8
8
|
import { startLocalApi } from './local-api.js';
|
|
9
|
+
import type { LocalApiAgentPrincipal } from './local-auth.js';
|
|
9
10
|
import { loadOrCreateDaemonToken } from './token-file.js';
|
|
10
11
|
import type { ComputerAgentAssignment, Message, MessageDelivery } from './types.js';
|
|
11
12
|
import type { AgentRuntime } from './agent-runtime.js';
|
|
12
13
|
import { knownRuntimeNames, resolveRuntimeDriver } from './runtime-registry.js';
|
|
14
|
+
import { buildRuntimeLaunchContext, type AgentPermissionProfile } from './agent-permissions.js';
|
|
15
|
+
import { hydrateRuntimeEnv, logEnvReport } from './runtime-env.js';
|
|
16
|
+
import type { RuntimeAttachment } from './agent-runtime.js';
|
|
13
17
|
|
|
14
18
|
interface DaemonState {
|
|
15
19
|
daemonId: string;
|
|
@@ -23,11 +27,136 @@ interface DaemonState {
|
|
|
23
27
|
interface ManagedAgent {
|
|
24
28
|
agent: string;
|
|
25
29
|
runtimeProvider: string;
|
|
30
|
+
runtimeModel: string | null;
|
|
26
31
|
agentUuid: string;
|
|
27
32
|
agentHome: string;
|
|
28
33
|
projectCwd: string;
|
|
29
34
|
}
|
|
30
35
|
|
|
36
|
+
export interface SteeredDelivery {
|
|
37
|
+
id: string;
|
|
38
|
+
claimToken: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface ActiveRunHandle {
|
|
42
|
+
key: string;
|
|
43
|
+
agent: string;
|
|
44
|
+
chatId: string;
|
|
45
|
+
runId: string;
|
|
46
|
+
sessionId: string;
|
|
47
|
+
runtimeSessionId: string | null;
|
|
48
|
+
supportsSteer: boolean;
|
|
49
|
+
steeredDeliveries: SteeredDelivery[];
|
|
50
|
+
steer(message: Message): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type ActiveRunMap = Map<string, ActiveRunHandle>;
|
|
54
|
+
|
|
55
|
+
export class RuntimeLocalTokenRegistry {
|
|
56
|
+
private readonly tokens = new Map<string, LocalApiAgentPrincipal>();
|
|
57
|
+
|
|
58
|
+
create(input: { agent: string; runId: string; chatId: string }): string {
|
|
59
|
+
const token = `palrt_${crypto.randomUUID()}`;
|
|
60
|
+
this.tokens.set(token, { kind: 'agent', agent: input.agent, runId: input.runId, chatId: input.chatId });
|
|
61
|
+
return token;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
lookup(token: string): LocalApiAgentPrincipal | null {
|
|
65
|
+
return this.tokens.get(token) ?? null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
revoke(token: string): void {
|
|
69
|
+
this.tokens.delete(token);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function activeRunKey(agent: string, chatId: string): string {
|
|
74
|
+
return `${agent}\0${chatId}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function shouldStopDaemonForHeartbeatError(error: unknown): boolean {
|
|
78
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
79
|
+
return message.includes('CONNECTION_REVOKED')
|
|
80
|
+
|| message.includes('computer connection is not active')
|
|
81
|
+
|| message.includes('connection auth is required')
|
|
82
|
+
|| message.includes('invalid computer credential');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function shouldRetryDaemonConnectError(error: unknown): boolean {
|
|
86
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
87
|
+
return message.includes('fetch failed')
|
|
88
|
+
|| message.includes('Unable to connect')
|
|
89
|
+
|| message.includes('ECONNREFUSED')
|
|
90
|
+
|| message.includes('ECONNRESET')
|
|
91
|
+
|| message.includes('EPIPE')
|
|
92
|
+
|| message.includes('UND_ERR')
|
|
93
|
+
|| message.includes('Unexpected end of JSON input')
|
|
94
|
+
|| message.includes('request failed: 502')
|
|
95
|
+
|| message.includes('request failed: 503')
|
|
96
|
+
|| message.includes('request failed: 504');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function isDeliveryConnectionSuperseded(currentConnectionId: string | null | undefined, runConnectionId: string | null | undefined): boolean {
|
|
100
|
+
return Boolean(runConnectionId && currentConnectionId && runConnectionId !== currentConnectionId);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface RepeatedWarningState {
|
|
104
|
+
lastKey: string | null;
|
|
105
|
+
lastLoggedAtMs: number;
|
|
106
|
+
suppressed: number;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function defaultRepeatedWarningState(): RepeatedWarningState {
|
|
110
|
+
return { lastKey: null, lastLoggedAtMs: 0, suppressed: 0 };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function consumeRepeatedWarning(state: RepeatedWarningState, input: {
|
|
114
|
+
key: string;
|
|
115
|
+
nowMs: number;
|
|
116
|
+
minIntervalMs: number;
|
|
117
|
+
}): { shouldLog: boolean; suppressed: number } {
|
|
118
|
+
const keyChanged = state.lastKey !== input.key;
|
|
119
|
+
const intervalElapsed = input.nowMs - state.lastLoggedAtMs >= input.minIntervalMs;
|
|
120
|
+
if (keyChanged || intervalElapsed) {
|
|
121
|
+
const suppressed = state.suppressed;
|
|
122
|
+
state.lastKey = input.key;
|
|
123
|
+
state.lastLoggedAtMs = input.nowMs;
|
|
124
|
+
state.suppressed = 0;
|
|
125
|
+
return { shouldLog: true, suppressed };
|
|
126
|
+
}
|
|
127
|
+
state.suppressed += 1;
|
|
128
|
+
return { shouldLog: false, suppressed: state.suppressed };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export type PendingDeliveryAction =
|
|
132
|
+
| { kind: 'start'; delivery: MessageDelivery; key: string }
|
|
133
|
+
| { kind: 'steer'; delivery: MessageDelivery; key: string; active: ActiveRunHandle }
|
|
134
|
+
| { kind: 'skip'; delivery: MessageDelivery; key: string; reason: 'starting' | 'active-no-steer' };
|
|
135
|
+
|
|
136
|
+
const PENDING_DELIVERY_CANDIDATE_LIMIT = 100;
|
|
137
|
+
|
|
138
|
+
export function planPendingDeliveryActions(agent: string, deliveries: MessageDelivery[], activeRuns: ActiveRunMap): PendingDeliveryAction[] {
|
|
139
|
+
const actions: PendingDeliveryAction[] = [];
|
|
140
|
+
const startingKeys = new Set<string>();
|
|
141
|
+
for (const delivery of deliveries) {
|
|
142
|
+
const key = activeRunKey(agent, delivery.chat_id);
|
|
143
|
+
const active = activeRuns.get(key);
|
|
144
|
+
if (active) {
|
|
145
|
+
actions.push(active.supportsSteer
|
|
146
|
+
? { kind: 'steer', delivery, key, active }
|
|
147
|
+
: { kind: 'skip', delivery, key, reason: 'active-no-steer' });
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (startingKeys.has(key)) {
|
|
151
|
+
actions.push({ kind: 'skip', delivery, key, reason: 'starting' });
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
startingKeys.add(key);
|
|
155
|
+
actions.push({ kind: 'start', delivery, key });
|
|
156
|
+
}
|
|
157
|
+
return actions;
|
|
158
|
+
}
|
|
159
|
+
|
|
31
160
|
function usage(): string {
|
|
32
161
|
return `pal daemon
|
|
33
162
|
|
|
@@ -37,6 +166,7 @@ Usage:
|
|
|
37
166
|
|
|
38
167
|
The daemon connects one computer to the server, claims deliveries for assigned agents, and starts one agent run for each claimed delivery.
|
|
39
168
|
Runs have no default timeout; use the web control page or API to kill/restart a run.
|
|
169
|
+
Use --verbose or PAL_DAEMON_VERBOSE_TICKS=1 to log empty reconcile ticks.
|
|
40
170
|
`;
|
|
41
171
|
}
|
|
42
172
|
|
|
@@ -75,18 +205,224 @@ function writeState(path: string, state: DaemonState): void {
|
|
|
75
205
|
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
|
|
76
206
|
}
|
|
77
207
|
|
|
208
|
+
function runtimeAttachmentsForMessage(input: {
|
|
209
|
+
message: Message;
|
|
210
|
+
}): RuntimeAttachment[] {
|
|
211
|
+
const attachments = input.message.attachments ?? [];
|
|
212
|
+
if (attachments.length === 0) return [];
|
|
213
|
+
return attachments.map((attachment) => ({
|
|
214
|
+
id: attachment.id,
|
|
215
|
+
messageId: input.message.id,
|
|
216
|
+
kind: attachment.kind,
|
|
217
|
+
mimeType: attachment.mime_type,
|
|
218
|
+
filename: attachment.filename,
|
|
219
|
+
path: attachment.path,
|
|
220
|
+
}));
|
|
221
|
+
}
|
|
222
|
+
|
|
78
223
|
function sleep(ms: number): Promise<void> {
|
|
79
224
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
80
225
|
}
|
|
81
226
|
|
|
82
|
-
|
|
227
|
+
function deliveryWebSocketUrl(serverUrl: string): string {
|
|
228
|
+
const url = new URL(serverUrl);
|
|
229
|
+
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
230
|
+
url.pathname = '/api/daemon/ws';
|
|
231
|
+
url.search = '';
|
|
232
|
+
return url.toString();
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export interface DeliverySocketHandle {
|
|
236
|
+
stop(): void;
|
|
237
|
+
isOpen(): boolean;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const DELIVERY_WS_PING_INTERVAL_MS = 30_000;
|
|
241
|
+
const DELIVERY_WS_PONG_TIMEOUT_MS = 10_000;
|
|
242
|
+
|
|
243
|
+
export class DeliveryWakeQueue {
|
|
244
|
+
private pendingAgents = new Set<string>();
|
|
245
|
+
private pendingAll = false;
|
|
246
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
247
|
+
private readonly delayMs: number;
|
|
248
|
+
|
|
249
|
+
constructor(
|
|
250
|
+
private readonly flush: (agent?: string) => void,
|
|
251
|
+
options: { delayMs?: number } = {},
|
|
252
|
+
) {
|
|
253
|
+
this.delayMs = Math.max(0, options.delayMs ?? 0);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
enqueue(agent?: string): void {
|
|
257
|
+
if (agent) {
|
|
258
|
+
if (!this.pendingAll) this.pendingAgents.add(agent);
|
|
259
|
+
} else {
|
|
260
|
+
this.pendingAll = true;
|
|
261
|
+
this.pendingAgents.clear();
|
|
262
|
+
}
|
|
263
|
+
if (this.timer) return;
|
|
264
|
+
this.timer = setTimeout(() => this.drain(), this.delayMs);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
drainNow(): void {
|
|
268
|
+
if (this.timer) clearTimeout(this.timer);
|
|
269
|
+
this.drain();
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
stop(): void {
|
|
273
|
+
if (this.timer) clearTimeout(this.timer);
|
|
274
|
+
this.timer = null;
|
|
275
|
+
this.pendingAll = false;
|
|
276
|
+
this.pendingAgents.clear();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private drain(): void {
|
|
280
|
+
this.timer = null;
|
|
281
|
+
if (this.pendingAll) {
|
|
282
|
+
this.pendingAll = false;
|
|
283
|
+
this.pendingAgents.clear();
|
|
284
|
+
this.flush();
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const agents = Array.from(this.pendingAgents);
|
|
288
|
+
this.pendingAgents.clear();
|
|
289
|
+
for (const agent of agents) this.flush(agent);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export function startDeliveryWebSocket(input: {
|
|
294
|
+
serverUrl: string;
|
|
295
|
+
computerId: string;
|
|
296
|
+
connectionId: string;
|
|
297
|
+
token: string;
|
|
298
|
+
onDelivery(agent?: string): void;
|
|
299
|
+
pingIntervalMs?: number;
|
|
300
|
+
pongTimeoutMs?: number;
|
|
301
|
+
}): DeliverySocketHandle {
|
|
302
|
+
let stopped = false;
|
|
303
|
+
let ws: WebSocket | null = null;
|
|
304
|
+
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
305
|
+
let pingTimer: ReturnType<typeof setInterval> | null = null;
|
|
306
|
+
let pongTimer: ReturnType<typeof setTimeout> | null = null;
|
|
307
|
+
let attempt = 0;
|
|
308
|
+
const pingIntervalMs = input.pingIntervalMs ?? DELIVERY_WS_PING_INTERVAL_MS;
|
|
309
|
+
const pongTimeoutMs = input.pongTimeoutMs ?? DELIVERY_WS_PONG_TIMEOUT_MS;
|
|
310
|
+
|
|
311
|
+
const clearKeepalive = () => {
|
|
312
|
+
if (pingTimer) clearInterval(pingTimer);
|
|
313
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
314
|
+
pingTimer = null;
|
|
315
|
+
pongTimer = null;
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
const closeUnhealthySocket = (reason: string) => {
|
|
319
|
+
const current = ws;
|
|
320
|
+
if (!current || current.readyState === WebSocket.CLOSED || current.readyState === WebSocket.CLOSING) return;
|
|
321
|
+
console.warn(`[daemon] delivery websocket ${reason}; reconnecting`);
|
|
322
|
+
current.close(4000, reason);
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
const sendPing = () => {
|
|
326
|
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
327
|
+
try {
|
|
328
|
+
ws.send('ping');
|
|
329
|
+
} catch {
|
|
330
|
+
closeUnhealthySocket('ping send failed');
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
334
|
+
pongTimer = setTimeout(() => closeUnhealthySocket('pong timeout'), pongTimeoutMs);
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
const connect = () => {
|
|
338
|
+
if (stopped) return;
|
|
339
|
+
const url = deliveryWebSocketUrl(input.serverUrl);
|
|
340
|
+
try {
|
|
341
|
+
ws = new (WebSocket as unknown as { new(url: string, options: Bun.WebSocketOptions): WebSocket })(url, {
|
|
342
|
+
headers: {
|
|
343
|
+
'x-pal-computer-id': input.computerId,
|
|
344
|
+
'x-pal-connection-id': input.connectionId,
|
|
345
|
+
'x-pal-connection-token': input.token,
|
|
346
|
+
},
|
|
347
|
+
});
|
|
348
|
+
} catch (error) {
|
|
349
|
+
scheduleReconnect(error);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
ws.addEventListener('open', () => {
|
|
354
|
+
attempt = 0;
|
|
355
|
+
console.log(`[daemon] delivery websocket connected ${url}`);
|
|
356
|
+
clearKeepalive();
|
|
357
|
+
if (pingIntervalMs > 0) {
|
|
358
|
+
pingTimer = setInterval(sendPing, pingIntervalMs);
|
|
359
|
+
sendPing();
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
ws.addEventListener('message', (event) => {
|
|
363
|
+
let frame: { type?: string; agent?: string; delivery?: { agent?: string } };
|
|
364
|
+
try {
|
|
365
|
+
frame = JSON.parse(String(event.data));
|
|
366
|
+
} catch {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (frame.type === 'delivery') {
|
|
370
|
+
input.onDelivery(frame.delivery?.agent);
|
|
371
|
+
} else if (frame.type === 'pending') {
|
|
372
|
+
input.onDelivery(frame.agent);
|
|
373
|
+
} else if (frame.type === 'pong') {
|
|
374
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
375
|
+
pongTimer = null;
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
ws.addEventListener('close', () => {
|
|
379
|
+
clearKeepalive();
|
|
380
|
+
if (!stopped) scheduleReconnect();
|
|
381
|
+
});
|
|
382
|
+
ws.addEventListener('error', (event) => {
|
|
383
|
+
if (!stopped) console.log(`[daemon] delivery websocket error: ${String(event.type)}`);
|
|
384
|
+
});
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const scheduleReconnect = (error?: unknown) => {
|
|
388
|
+
if (stopped || reconnectTimer) return;
|
|
389
|
+
if (error) {
|
|
390
|
+
console.warn(`[daemon] delivery websocket connect failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
391
|
+
}
|
|
392
|
+
attempt += 1;
|
|
393
|
+
const delay = Math.min(10_000, 250 * 2 ** Math.min(attempt, 5));
|
|
394
|
+
reconnectTimer = setTimeout(() => {
|
|
395
|
+
reconnectTimer = null;
|
|
396
|
+
connect();
|
|
397
|
+
}, delay);
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
connect();
|
|
401
|
+
return {
|
|
402
|
+
stop() {
|
|
403
|
+
stopped = true;
|
|
404
|
+
if (reconnectTimer) clearTimeout(reconnectTimer);
|
|
405
|
+
reconnectTimer = null;
|
|
406
|
+
clearKeepalive();
|
|
407
|
+
ws?.close(1000, 'daemon stopping');
|
|
408
|
+
ws = null;
|
|
409
|
+
},
|
|
410
|
+
isOpen() {
|
|
411
|
+
return ws?.readyState === WebSocket.OPEN;
|
|
412
|
+
},
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async function resolveRuntime(agent: string, serverUrl: string, agentUuid: string): Promise<{ runtime: AgentRuntime; runtimeModel: string | null }> {
|
|
83
417
|
let configuredRuntime: string | null | undefined;
|
|
418
|
+
let configuredModel: string | null | undefined;
|
|
84
419
|
try {
|
|
85
420
|
const response = await fetch(`${serverUrl.replace(/\/$/, '')}/api/agents`);
|
|
86
421
|
if (response.ok) {
|
|
87
|
-
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; runtime: string | null }> } };
|
|
422
|
+
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; runtime: string | null; model?: string | null }> } };
|
|
88
423
|
const configured = payload.data?.agents.find((a) => a.agent_key === agent);
|
|
89
424
|
configuredRuntime = configured?.runtime;
|
|
425
|
+
configuredModel = configured?.model;
|
|
90
426
|
}
|
|
91
427
|
} catch (error) {
|
|
92
428
|
throw new Error(`Could not resolve runtime for agent '${agent}' from ${serverUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -95,7 +431,114 @@ async function resolveRuntime(agent: string, serverUrl: string, agentUuid: strin
|
|
|
95
431
|
if (!configuredRuntime) {
|
|
96
432
|
throw new Error(`Agent '${agent}' has no runtime configured. Configure it via POST /api/agents or PATCH /api/agents/<key>. Supported runtimes: ${knownRuntimeNames().join(', ')}.`);
|
|
97
433
|
}
|
|
98
|
-
return
|
|
434
|
+
return {
|
|
435
|
+
runtime: await resolveRuntimeDriver(configuredRuntime, agentUuid, configuredModel),
|
|
436
|
+
runtimeModel: configuredModel?.trim() || null,
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function fetchConfiguredRuntimeModel(agent: string, serverUrl: string): Promise<string | null> {
|
|
441
|
+
try {
|
|
442
|
+
const response = await fetch(`${serverUrl.replace(/\/$/, '')}/api/agents`);
|
|
443
|
+
if (!response.ok) return null;
|
|
444
|
+
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; model?: string | null }> } };
|
|
445
|
+
return payload.data?.agents.find((item) => item.agent_key === agent)?.model?.trim() || null;
|
|
446
|
+
} catch {
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
async function loadPermissionProfile(client: LockClient, agent: string): Promise<AgentPermissionProfile> {
|
|
452
|
+
const profile = await client.fetchPermissionProfile(agent);
|
|
453
|
+
return {
|
|
454
|
+
filesystemMode: profile.filesystem_mode,
|
|
455
|
+
extraWritableRoots: profile.extra_writable_roots,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
async function ackSteeredDeliveries(
|
|
460
|
+
client: LockClient,
|
|
461
|
+
active: ActiveRunHandle,
|
|
462
|
+
options: { daemonId: string; connectionId?: string | null },
|
|
463
|
+
runId: string,
|
|
464
|
+
): Promise<void> {
|
|
465
|
+
for (const steered of active.steeredDeliveries) {
|
|
466
|
+
await client.ackDelivery(steered.id, {
|
|
467
|
+
daemon_id: options.daemonId,
|
|
468
|
+
connection_id: options.connectionId,
|
|
469
|
+
claim_token: steered.claimToken,
|
|
470
|
+
run_id: runId,
|
|
471
|
+
}).catch((error) => {
|
|
472
|
+
console.warn(`[daemon] failed to ack steered delivery ${steered.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async function failSteeredDeliveries(
|
|
478
|
+
client: LockClient,
|
|
479
|
+
active: ActiveRunHandle,
|
|
480
|
+
options: { daemonId: string; connectionId?: string | null },
|
|
481
|
+
runId: string,
|
|
482
|
+
error: string,
|
|
483
|
+
): Promise<void> {
|
|
484
|
+
for (const steered of active.steeredDeliveries) {
|
|
485
|
+
await client.failDelivery(steered.id, {
|
|
486
|
+
daemon_id: options.daemonId,
|
|
487
|
+
connection_id: options.connectionId,
|
|
488
|
+
claim_token: steered.claimToken,
|
|
489
|
+
run_id: runId,
|
|
490
|
+
error,
|
|
491
|
+
}).catch((failError) => {
|
|
492
|
+
console.warn(`[daemon] failed to fail steered delivery ${steered.id}: ${failError instanceof Error ? failError.message : String(failError)}`);
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function ackCoveredPendingDeliveries(
|
|
498
|
+
client: LockClient,
|
|
499
|
+
input: {
|
|
500
|
+
agent: string;
|
|
501
|
+
chatId: string;
|
|
502
|
+
triggerMessageId: number;
|
|
503
|
+
maxCoveredMessageId: number;
|
|
504
|
+
daemonId: string;
|
|
505
|
+
connectionId?: string | null;
|
|
506
|
+
computerId?: string | null;
|
|
507
|
+
runId: string;
|
|
508
|
+
},
|
|
509
|
+
): Promise<void> {
|
|
510
|
+
if (input.maxCoveredMessageId <= input.triggerMessageId) return;
|
|
511
|
+
const pending = await client.listDeliveries(input.agent, 'pending', PENDING_DELIVERY_CANDIDATE_LIMIT);
|
|
512
|
+
const covered = pending.filter((delivery) => (
|
|
513
|
+
delivery.chat_id === input.chatId
|
|
514
|
+
&& delivery.message_id > input.triggerMessageId
|
|
515
|
+
&& delivery.message_id <= input.maxCoveredMessageId
|
|
516
|
+
));
|
|
517
|
+
for (const delivery of covered) {
|
|
518
|
+
try {
|
|
519
|
+
console.log(`[daemon] ack covered pending delivery=${delivery.id} run=${input.runId} message=${delivery.message_id}`);
|
|
520
|
+
const claimed = await client.claimDelivery(delivery.id, {
|
|
521
|
+
daemon_id: input.daemonId,
|
|
522
|
+
connection_id: input.connectionId,
|
|
523
|
+
computer_id: input.computerId,
|
|
524
|
+
steer_run_id: input.runId,
|
|
525
|
+
});
|
|
526
|
+
if (!claimed.claim_token) throw new Error(`covered delivery ${delivery.id} was claimed without a claim token`);
|
|
527
|
+
await client.ackDelivery(claimed.id, {
|
|
528
|
+
daemon_id: input.daemonId,
|
|
529
|
+
connection_id: input.connectionId,
|
|
530
|
+
claim_token: claimed.claim_token,
|
|
531
|
+
run_id: input.runId,
|
|
532
|
+
});
|
|
533
|
+
} catch (error) {
|
|
534
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
535
|
+
if (message.includes('pending delivery') && message.includes('was not found')) {
|
|
536
|
+
console.log(`[daemon] covered pending delivery already claimed delivery=${delivery.id}`);
|
|
537
|
+
} else {
|
|
538
|
+
console.warn(`[daemon] failed to ack covered pending delivery ${delivery.id}: ${message}`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
99
542
|
}
|
|
100
543
|
|
|
101
544
|
async function executeRun(client: LockClient, delivery: MessageDelivery, message: Message, options: {
|
|
@@ -114,12 +557,18 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
114
557
|
computerId?: string | null;
|
|
115
558
|
connectionId?: string | null;
|
|
116
559
|
runtimeProvider: string;
|
|
560
|
+
runtimeModel?: string | null;
|
|
561
|
+
permissionProfile: AgentPermissionProfile;
|
|
562
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
117
563
|
isConnectionRevoked?: () => boolean;
|
|
118
564
|
abortSignal?: AbortSignal;
|
|
565
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
566
|
+
activeRuns?: ActiveRunMap;
|
|
119
567
|
}): Promise<'done' | 'restart'> {
|
|
120
568
|
if (!delivery.claim_token) throw new Error(`delivery ${delivery.id} is not claimed`);
|
|
121
569
|
|
|
122
|
-
const
|
|
570
|
+
const runtimeModel = options.runtimeModel ?? await fetchConfiguredRuntimeModel(options.agent, options.serverUrl);
|
|
571
|
+
const runtime = await resolveRuntimeDriver(options.runtimeProvider, options.agentUuid, runtimeModel);
|
|
123
572
|
const logPrefix = `[${runtime.name}]`;
|
|
124
573
|
|
|
125
574
|
console.log(`${logPrefix} session getOrCreate chat=${message.chat_id} agent=${options.agent} daemon=${options.daemonId} agentHome=${options.agentHome} projectCwd=${options.projectCwd}`);
|
|
@@ -151,24 +600,83 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
151
600
|
delivery_id: delivery.id,
|
|
152
601
|
});
|
|
153
602
|
console.log(`${logPrefix} run=${run.id} status=${run.status}`);
|
|
603
|
+
const deliveryContext = await client.getDeliveryContext({
|
|
604
|
+
agent: options.agent,
|
|
605
|
+
chatId: message.chat_id,
|
|
606
|
+
messageId: message.id,
|
|
607
|
+
limit: 50,
|
|
608
|
+
}).catch((error) => {
|
|
609
|
+
console.warn(`${logPrefix} delivery context unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
610
|
+
return null;
|
|
611
|
+
});
|
|
612
|
+
const runtimeLocalToken = options.runtimeTokens?.create({ agent: options.agent, runId: run.id, chatId: message.chat_id });
|
|
613
|
+
const key = activeRunKey(options.agent, message.chat_id);
|
|
614
|
+
const activeHandle: ActiveRunHandle = {
|
|
615
|
+
key,
|
|
616
|
+
agent: options.agent,
|
|
617
|
+
chatId: message.chat_id,
|
|
618
|
+
runId: run.id,
|
|
619
|
+
sessionId: session.id,
|
|
620
|
+
runtimeSessionId: session.runtime_session_id,
|
|
621
|
+
supportsSteer: runtime.capabilities.supportsSteer && typeof runtime.steerActiveRun === 'function',
|
|
622
|
+
steeredDeliveries: [],
|
|
623
|
+
steer: async (steerMessage) => {
|
|
624
|
+
if (!runtime.capabilities.supportsSteer || !runtime.steerActiveRun) {
|
|
625
|
+
throw new Error(`runtime ${runtime.name} does not support active steer`);
|
|
626
|
+
}
|
|
627
|
+
await runtime.steerActiveRun({
|
|
628
|
+
agent: options.agent,
|
|
629
|
+
serverUrl: options.serverUrl,
|
|
630
|
+
runId: run.id,
|
|
631
|
+
sessionId: session.id,
|
|
632
|
+
runtimeSessionId: activeHandle.runtimeSessionId,
|
|
633
|
+
message: steerMessage,
|
|
634
|
+
cwd: options.agentHome,
|
|
635
|
+
agentHome: options.agentHome,
|
|
636
|
+
projectCwd: options.projectCwd,
|
|
637
|
+
localDaemonUrl: options.localDaemonUrl,
|
|
638
|
+
localDaemonToken: runtimeLocalToken ?? options.localDaemonToken,
|
|
639
|
+
privateCliBinDir: options.privateCliBinDir,
|
|
640
|
+
palCliCommand: 'pal',
|
|
641
|
+
runtimeEnv: options.runtimeEnv,
|
|
642
|
+
});
|
|
643
|
+
},
|
|
644
|
+
};
|
|
645
|
+
options.activeRuns?.set(key, activeHandle);
|
|
646
|
+
console.log(`${logPrefix} active run key=${key.replace('\0', ':')} supportsSteer=${activeHandle.supportsSteer}`);
|
|
154
647
|
|
|
155
648
|
const { runAgentRuntime } = await import('./agent-runtime.js');
|
|
156
|
-
const
|
|
157
|
-
.then((result) => result
|
|
649
|
+
const roomSnapshot = await client.listRoomMembers(message.chat_id, options.agent)
|
|
650
|
+
.then((result) => result)
|
|
158
651
|
.catch((error) => {
|
|
159
652
|
console.warn(`${logPrefix} room participant snapshot unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
160
|
-
return
|
|
653
|
+
return null;
|
|
161
654
|
});
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
655
|
+
const roomParticipants = roomSnapshot?.participants ?? [];
|
|
656
|
+
const roomAgentSubscriptions = roomSnapshot?.agent_subscriptions ?? [];
|
|
657
|
+
const enabledSkills = roomSnapshot?.enabled_skills ?? [];
|
|
658
|
+
const roomProject = roomSnapshot?.room.project_id ? {
|
|
659
|
+
id: roomSnapshot.room.project_id,
|
|
660
|
+
name: roomSnapshot.room.project_name ?? roomSnapshot.room.project_id,
|
|
661
|
+
rootPath: roomSnapshot.room.project_root_path ?? '',
|
|
662
|
+
computerId: roomSnapshot.room.project_computer_id ?? '',
|
|
663
|
+
computerName: roomSnapshot.room.project_computer_name,
|
|
664
|
+
} : null;
|
|
665
|
+
const projectAccessible = Boolean(roomProject && options.computerId && roomProject.computerId === options.computerId);
|
|
666
|
+
const effectiveProjectCwd = projectAccessible && roomProject?.rootPath ? roomProject.rootPath : options.projectCwd;
|
|
667
|
+
const launchContext = buildRuntimeLaunchContext({
|
|
668
|
+
agent: options.agent,
|
|
669
|
+
serverUrl: options.serverUrl,
|
|
670
|
+
message,
|
|
671
|
+
cwd: options.agentHome,
|
|
672
|
+
agentHome: options.agentHome,
|
|
673
|
+
projectCwd: projectAccessible ? effectiveProjectCwd : undefined,
|
|
674
|
+
extraArgs: options.extraArgs,
|
|
169
675
|
});
|
|
676
|
+
const maxCoveredMessageId = deliveryContext?.messages.reduce((max, item) => Math.max(max, item.id), message.id) ?? message.id;
|
|
170
677
|
|
|
171
678
|
try {
|
|
679
|
+
const runtimeAttachments = runtimeAttachmentsForMessage({ message });
|
|
172
680
|
console.log(`${logPrefix} spawn agent=${options.agent} chat=${message.chat_name} message=${message.id} dryRun=${options.dryRun}`);
|
|
173
681
|
const result = await runAgentRuntime(runtime, {
|
|
174
682
|
agent: options.agent,
|
|
@@ -176,21 +684,36 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
176
684
|
message,
|
|
177
685
|
cwd: options.agentHome,
|
|
178
686
|
agentHome: options.agentHome,
|
|
179
|
-
projectCwd:
|
|
687
|
+
projectCwd: effectiveProjectCwd,
|
|
688
|
+
launchContext,
|
|
689
|
+
permissionProfile: options.permissionProfile,
|
|
690
|
+
projectContext: roomProject ? {
|
|
691
|
+
...roomProject,
|
|
692
|
+
accessible: projectAccessible,
|
|
693
|
+
currentComputerId: options.computerId,
|
|
694
|
+
} : undefined,
|
|
180
695
|
extraArgs: options.extraArgs,
|
|
181
696
|
localDaemonUrl: options.localDaemonUrl,
|
|
182
|
-
localDaemonToken: options.localDaemonToken,
|
|
697
|
+
localDaemonToken: runtimeLocalToken ?? options.localDaemonToken,
|
|
183
698
|
privateCliBinDir: options.privateCliBinDir,
|
|
184
699
|
palCliCommand: 'pal',
|
|
700
|
+
runtimeEnv: options.runtimeEnv,
|
|
185
701
|
runtimeSessionId: session.runtime_session_id,
|
|
186
702
|
roomParticipants,
|
|
187
|
-
|
|
703
|
+
roomAgentSubscriptions,
|
|
704
|
+
roomMode: roomSnapshot?.room.mode ?? 'standard',
|
|
705
|
+
enabledSkills,
|
|
706
|
+
deliveryContext: deliveryContext ?? undefined,
|
|
707
|
+
runtimeAttachments,
|
|
188
708
|
dryRun: options.dryRun,
|
|
189
709
|
signal: options.abortSignal,
|
|
190
710
|
onStart: async (pid) => {
|
|
191
711
|
console.log(`${logPrefix} run=${run.id} pid=${pid}`);
|
|
192
712
|
await client.updateRunPid(run.id, pid);
|
|
193
713
|
},
|
|
714
|
+
onActivity: async (event) => {
|
|
715
|
+
await client.recordRunActivity(run.id, event);
|
|
716
|
+
},
|
|
194
717
|
getAction: async () => {
|
|
195
718
|
if (options.isConnectionRevoked?.()) return 'kill';
|
|
196
719
|
const latest = await client.getRun(run.id);
|
|
@@ -204,6 +727,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
204
727
|
if (result.runtimeSessionId && result.runtimeSessionId !== session.runtime_session_id) {
|
|
205
728
|
console.log(`${logPrefix} session=${session.id} runtime_session_id=${result.runtimeSessionId}`);
|
|
206
729
|
await client.updateSessionRuntimeSessionId(session.id, { runtime_session_id: result.runtimeSessionId });
|
|
730
|
+
activeHandle.runtimeSessionId = result.runtimeSessionId;
|
|
207
731
|
}
|
|
208
732
|
|
|
209
733
|
console.log(`${logPrefix} run=${run.id} exitCode=${result.exitCode} stoppedByAction=${result.stoppedByAction ?? '-'} outputLen=${result.output.length}`);
|
|
@@ -226,6 +750,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
226
750
|
if (result.stoppedByAction === 'restart') {
|
|
227
751
|
console.log(`${logPrefix} run=${run.id} finishing with restart`);
|
|
228
752
|
await client.finishRun(run.id, { status: 'restarted', exit_code: result.exitCode, output: result.output });
|
|
753
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, 'active run restarted before completion');
|
|
229
754
|
return 'restart';
|
|
230
755
|
}
|
|
231
756
|
|
|
@@ -233,6 +758,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
233
758
|
console.log(`${logPrefix} run=${run.id} finishing with killed`);
|
|
234
759
|
await client.finishRun(run.id, { status: 'killed', exit_code: result.exitCode, output: result.output });
|
|
235
760
|
await client.ackDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id });
|
|
761
|
+
await ackSteeredDeliveries(client, activeHandle, options, run.id);
|
|
236
762
|
return 'done';
|
|
237
763
|
}
|
|
238
764
|
|
|
@@ -240,6 +766,17 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
240
766
|
console.log(`${logPrefix} run=${run.id} finishing with completed`);
|
|
241
767
|
await client.finishRun(run.id, { status: 'completed', exit_code: result.exitCode, output: result.output });
|
|
242
768
|
await client.ackDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id });
|
|
769
|
+
await ackCoveredPendingDeliveries(client, {
|
|
770
|
+
agent: options.agent,
|
|
771
|
+
chatId: message.chat_id,
|
|
772
|
+
triggerMessageId: message.id,
|
|
773
|
+
maxCoveredMessageId,
|
|
774
|
+
daemonId: options.daemonId,
|
|
775
|
+
connectionId: options.connectionId,
|
|
776
|
+
computerId: options.computerId,
|
|
777
|
+
runId: run.id,
|
|
778
|
+
});
|
|
779
|
+
await ackSteeredDeliveries(client, activeHandle, options, run.id);
|
|
243
780
|
return 'done';
|
|
244
781
|
}
|
|
245
782
|
|
|
@@ -247,6 +784,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
247
784
|
console.log(`${logPrefix} run=${run.id} finishing with failed: ${output.slice(0, 200)}`);
|
|
248
785
|
await client.finishRun(run.id, { status: 'failed', exit_code: result.exitCode, output });
|
|
249
786
|
await client.failDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id, error: output });
|
|
787
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, output);
|
|
250
788
|
await writeFailureMessage(client, message, options.agent, output);
|
|
251
789
|
return 'done';
|
|
252
790
|
} catch (error) {
|
|
@@ -258,8 +796,15 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
258
796
|
}
|
|
259
797
|
await client.finishRun(run.id, { status: 'failed', output });
|
|
260
798
|
await client.failDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id, error: output });
|
|
799
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, output);
|
|
261
800
|
await writeFailureMessage(client, message, options.agent, output);
|
|
262
801
|
return 'done';
|
|
802
|
+
} finally {
|
|
803
|
+
if (runtimeLocalToken) options.runtimeTokens?.revoke(runtimeLocalToken);
|
|
804
|
+
if (options.activeRuns?.get(key) === activeHandle) {
|
|
805
|
+
options.activeRuns.delete(key);
|
|
806
|
+
console.log(`${logPrefix} inactive run key=${key.replace('\0', ':')}`);
|
|
807
|
+
}
|
|
263
808
|
}
|
|
264
809
|
}
|
|
265
810
|
|
|
@@ -278,8 +823,13 @@ async function handleDelivery(client: LockClient, delivery: MessageDelivery, opt
|
|
|
278
823
|
computerId?: string | null;
|
|
279
824
|
connectionId?: string | null;
|
|
280
825
|
runtimeProvider: string;
|
|
826
|
+
runtimeModel?: string | null;
|
|
827
|
+
permissionProfile: AgentPermissionProfile;
|
|
828
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
281
829
|
isConnectionRevoked?: () => boolean;
|
|
282
830
|
abortSignal?: AbortSignal;
|
|
831
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
832
|
+
activeRuns?: ActiveRunMap;
|
|
283
833
|
}): Promise<void> {
|
|
284
834
|
console.log(`[daemon] handleDelivery delivery=${delivery.id} message=${delivery.message_id} attempts=${delivery.attempts}`);
|
|
285
835
|
const message = await client.getMessage(delivery.message_id);
|
|
@@ -307,7 +857,54 @@ async function discoverDeliveries(client: LockClient, state: DaemonState, agent:
|
|
|
307
857
|
}
|
|
308
858
|
}
|
|
309
859
|
|
|
310
|
-
async function
|
|
860
|
+
async function steerDeliveryToActiveRun(
|
|
861
|
+
client: LockClient,
|
|
862
|
+
delivery: MessageDelivery,
|
|
863
|
+
active: ActiveRunHandle,
|
|
864
|
+
options: {
|
|
865
|
+
daemonId: string;
|
|
866
|
+
connectionId?: string | null;
|
|
867
|
+
computerId?: string | null;
|
|
868
|
+
},
|
|
869
|
+
): Promise<void> {
|
|
870
|
+
let claimed: MessageDelivery | null = null;
|
|
871
|
+
try {
|
|
872
|
+
console.log(`[daemon] steerDelivery claim delivery=${delivery.id} activeRun=${active.runId}`);
|
|
873
|
+
claimed = await client.claimDelivery(delivery.id, {
|
|
874
|
+
daemon_id: options.daemonId,
|
|
875
|
+
connection_id: options.connectionId,
|
|
876
|
+
computer_id: options.computerId,
|
|
877
|
+
steer_run_id: active.runId,
|
|
878
|
+
});
|
|
879
|
+
if (!claimed.claim_token) throw new Error(`steered delivery ${delivery.id} was claimed without a claim token`);
|
|
880
|
+
const message = await client.getMessage(claimed.message_id);
|
|
881
|
+
await active.steer(message);
|
|
882
|
+
await client.markDeliveryProcessingCompleted(claimed.id, {
|
|
883
|
+
daemon_id: options.daemonId,
|
|
884
|
+
connection_id: options.connectionId,
|
|
885
|
+
claim_token: claimed.claim_token,
|
|
886
|
+
run_id: active.runId,
|
|
887
|
+
});
|
|
888
|
+
active.steeredDeliveries.push({ id: claimed.id, claimToken: claimed.claim_token });
|
|
889
|
+
console.log(`[daemon] steerDelivery accepted delivery=${delivery.id} activeRun=${active.runId}`);
|
|
890
|
+
} catch (error) {
|
|
891
|
+
const output = error instanceof Error ? error.message : String(error);
|
|
892
|
+
console.warn(`[daemon] steerDelivery failed delivery=${delivery.id}: ${output}`);
|
|
893
|
+
if (claimed?.claim_token) {
|
|
894
|
+
await client.failDelivery(claimed.id, {
|
|
895
|
+
daemon_id: options.daemonId,
|
|
896
|
+
connection_id: options.connectionId,
|
|
897
|
+
claim_token: claimed.claim_token,
|
|
898
|
+
run_id: active.runId,
|
|
899
|
+
error: output,
|
|
900
|
+
}).catch((failError) => {
|
|
901
|
+
console.warn(`[daemon] failed to mark steered delivery ${claimed?.id} failed: ${failError instanceof Error ? failError.message : String(failError)}`);
|
|
902
|
+
});
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
export async function processPendingDeliveries(client: LockClient, options: {
|
|
311
908
|
agent: string;
|
|
312
909
|
daemonId: string;
|
|
313
910
|
serverUrl: string;
|
|
@@ -322,37 +919,85 @@ async function processPendingDeliveries(client: LockClient, options: {
|
|
|
322
919
|
computerId?: string | null;
|
|
323
920
|
connectionId?: string | null;
|
|
324
921
|
runtimeProvider: string;
|
|
922
|
+
runtimeModel?: string | null;
|
|
923
|
+
permissionProfile: AgentPermissionProfile;
|
|
924
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
325
925
|
isConnectionRevoked?: () => boolean;
|
|
326
926
|
abortSignal?: AbortSignal;
|
|
327
|
-
|
|
927
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
928
|
+
activeRuns: ActiveRunMap;
|
|
929
|
+
}): Promise<Array<Promise<void>>> {
|
|
328
930
|
console.log(`[daemon] processPendingDeliveries agent=${options.agent} daemon=${options.daemonId}`);
|
|
329
|
-
const deliveries = await client.listDeliveries(options.agent, 'pending',
|
|
931
|
+
const deliveries = await client.listDeliveries(options.agent, 'pending', PENDING_DELIVERY_CANDIDATE_LIMIT, {
|
|
932
|
+
distinctChat: true,
|
|
933
|
+
excludeRunningComputerId: options.computerId,
|
|
934
|
+
connectionId: options.connectionId,
|
|
935
|
+
});
|
|
330
936
|
console.log(`[daemon] processPendingDeliveries found ${deliveries.length} pending deliveries`);
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
await handleDelivery(client, claimed, options);
|
|
337
|
-
} catch (error) {
|
|
338
|
-
console.warn(`[daemon] claimDelivery failed delivery=${delivery.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
937
|
+
const started: Array<Promise<void>> = [];
|
|
938
|
+
for (const action of planPendingDeliveryActions(options.agent, deliveries, options.activeRuns)) {
|
|
939
|
+
if (action.kind === 'steer') {
|
|
940
|
+
await steerDeliveryToActiveRun(client, action.delivery, action.active, options);
|
|
941
|
+
continue;
|
|
339
942
|
}
|
|
943
|
+
if (action.kind === 'skip') {
|
|
944
|
+
const reason = action.reason === 'starting' ? 'run already starting' : 'active run does not support steer';
|
|
945
|
+
console.log(`[daemon] delivery=${action.delivery.id} remains pending; ${reason} key=${action.key.replace('\0', ':')}`);
|
|
946
|
+
continue;
|
|
947
|
+
}
|
|
948
|
+
const delivery = action.delivery;
|
|
949
|
+
const reservation: ActiveRunHandle = {
|
|
950
|
+
key: action.key,
|
|
951
|
+
agent: options.agent,
|
|
952
|
+
chatId: delivery.chat_id,
|
|
953
|
+
runId: `starting:${delivery.id}`,
|
|
954
|
+
sessionId: '',
|
|
955
|
+
runtimeSessionId: null,
|
|
956
|
+
supportsSteer: false,
|
|
957
|
+
steeredDeliveries: [],
|
|
958
|
+
steer: async () => {
|
|
959
|
+
throw new Error('run is still starting');
|
|
960
|
+
},
|
|
961
|
+
};
|
|
962
|
+
options.activeRuns.set(action.key, reservation);
|
|
963
|
+
const task = (async () => {
|
|
964
|
+
try {
|
|
965
|
+
console.log(`[daemon] claimDelivery delivery=${delivery.id}`);
|
|
966
|
+
const claimed = await client.claimDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, computer_id: options.computerId });
|
|
967
|
+
console.log(`[daemon] claimDelivery success delivery=${claimed.id} token=${claimed.claim_token}`);
|
|
968
|
+
await handleDelivery(client, claimed, options);
|
|
969
|
+
} catch (error) {
|
|
970
|
+
console.warn(`[daemon] claimDelivery failed delivery=${delivery.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
971
|
+
} finally {
|
|
972
|
+
if (options.activeRuns.get(action.key) === reservation) {
|
|
973
|
+
options.activeRuns.delete(action.key);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
})();
|
|
977
|
+
started.push(task);
|
|
340
978
|
}
|
|
979
|
+
return started;
|
|
341
980
|
}
|
|
342
981
|
|
|
343
982
|
async function buildManagedAgent(input: {
|
|
344
|
-
assignment: Pick<ComputerAgentAssignment, 'agent' | 'runtime' | 'cwd'>;
|
|
983
|
+
assignment: Pick<ComputerAgentAssignment, 'agent' | 'runtime' | 'model' | 'cwd'>;
|
|
345
984
|
state: DaemonState;
|
|
346
985
|
serverUrl: string;
|
|
347
986
|
defaultCwd: string;
|
|
348
987
|
}): Promise<ManagedAgent> {
|
|
349
988
|
const agentUuid = ensureAgentUuid(input.state, input.assignment.agent);
|
|
350
|
-
const
|
|
989
|
+
const resolved = input.assignment.runtime
|
|
990
|
+
? {
|
|
991
|
+
runtime: await resolveRuntimeDriver(input.assignment.runtime, agentUuid, input.assignment.model),
|
|
992
|
+
runtimeModel: input.assignment.model?.trim() || null,
|
|
993
|
+
}
|
|
994
|
+
: await resolveRuntime(input.assignment.agent, input.serverUrl, agentUuid);
|
|
351
995
|
const agentHome = agentHomePath(agentUuid);
|
|
352
|
-
ensureAgentHome(agentHome, input.assignment.agent, runtime.name);
|
|
996
|
+
ensureAgentHome(agentHome, input.assignment.agent, resolved.runtime.name);
|
|
353
997
|
return {
|
|
354
998
|
agent: input.assignment.agent,
|
|
355
|
-
runtimeProvider: runtime.name,
|
|
999
|
+
runtimeProvider: resolved.runtime.name,
|
|
1000
|
+
runtimeModel: resolved.runtimeModel,
|
|
356
1001
|
agentUuid,
|
|
357
1002
|
agentHome,
|
|
358
1003
|
projectCwd: input.assignment.cwd || input.defaultCwd,
|
|
@@ -363,7 +1008,7 @@ function daemonAgentPayload(managedAgents: Map<string, ManagedAgent>): Array<{ a
|
|
|
363
1008
|
return Array.from(managedAgents.values()).map((managed) => ({
|
|
364
1009
|
agent: managed.agent,
|
|
365
1010
|
cwd: managed.agentHome,
|
|
366
|
-
capabilities: { runtime: managed.runtimeProvider, agentHome: managed.agentHome, projectCwd: managed.projectCwd },
|
|
1011
|
+
capabilities: { runtime: managed.runtimeProvider, model: managed.runtimeModel, agentHome: managed.agentHome, projectCwd: managed.projectCwd },
|
|
367
1012
|
}));
|
|
368
1013
|
}
|
|
369
1014
|
|
|
@@ -377,8 +1022,10 @@ async function reconcileManagedAgents(input: {
|
|
|
377
1022
|
computerId: string;
|
|
378
1023
|
serverUrl: string;
|
|
379
1024
|
defaultCwd: string;
|
|
380
|
-
|
|
1025
|
+
localUrl?: string;
|
|
1026
|
+
}): Promise<string[]> {
|
|
381
1027
|
const desiredAssignedAgents = new Set(input.assignments.map((assignment) => assignment.agent));
|
|
1028
|
+
const activatedAgents: string[] = [];
|
|
382
1029
|
|
|
383
1030
|
for (const assignment of input.assignments) {
|
|
384
1031
|
if (!assignment.runtime) {
|
|
@@ -387,10 +1034,11 @@ async function reconcileManagedAgents(input: {
|
|
|
387
1034
|
}
|
|
388
1035
|
const existing = input.managedAgents.get(assignment.agent);
|
|
389
1036
|
const projectCwd = assignment.cwd || input.defaultCwd;
|
|
390
|
-
if (existing && existing.runtimeProvider === assignment.runtime && existing.projectCwd === projectCwd) continue;
|
|
1037
|
+
if (existing && existing.runtimeProvider === assignment.runtime && existing.runtimeModel === assignment.model && existing.projectCwd === projectCwd) continue;
|
|
391
1038
|
const managed = await buildManagedAgent({ assignment, state: input.state, serverUrl: input.serverUrl, defaultCwd: input.defaultCwd });
|
|
392
1039
|
input.managedAgents.set(assignment.agent, managed);
|
|
393
|
-
|
|
1040
|
+
activatedAgents.push(managed.agent);
|
|
1041
|
+
console.log(`[daemon] agent active agent=${managed.agent} runtime=${managed.runtimeProvider} model=${managed.runtimeModel ?? '-'} projectCwd=${managed.projectCwd}`);
|
|
394
1042
|
}
|
|
395
1043
|
|
|
396
1044
|
for (const agent of Array.from(input.managedAgents.keys())) {
|
|
@@ -403,9 +1051,11 @@ async function reconcileManagedAgents(input: {
|
|
|
403
1051
|
await input.client.registerDaemon({
|
|
404
1052
|
id: input.daemonId,
|
|
405
1053
|
name: input.computerId,
|
|
1054
|
+
local_url: input.localUrl,
|
|
406
1055
|
server_url: input.serverUrl,
|
|
407
1056
|
agents: daemonAgentPayload(input.managedAgents),
|
|
408
1057
|
});
|
|
1058
|
+
return activatedAgents;
|
|
409
1059
|
}
|
|
410
1060
|
|
|
411
1061
|
async function main(): Promise<void> {
|
|
@@ -420,7 +1070,15 @@ async function main(): Promise<void> {
|
|
|
420
1070
|
return;
|
|
421
1071
|
}
|
|
422
1072
|
|
|
423
|
-
const
|
|
1073
|
+
const runtimeEnvReport = hydrateRuntimeEnv();
|
|
1074
|
+
logEnvReport(runtimeEnvReport);
|
|
1075
|
+
if (runtimeEnvReport.missing.length) {
|
|
1076
|
+
const message = `[daemon] runtime env warnings: missing=[${runtimeEnvReport.missing.join(', ')}], runtime auth may fail`;
|
|
1077
|
+
if (process.env.PAL_DAEMON_STRICT_ENV === '1') throw new Error(message);
|
|
1078
|
+
console.log(message);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const explicitAgent = flag(args.flags, 'agent');
|
|
424
1082
|
const serverUrl = flag(args.flags, 'server-url') ?? flag(args.flags, 'server') ?? defaultServerUrl();
|
|
425
1083
|
const cwd = flag(args.flags, 'cwd') ?? process.cwd();
|
|
426
1084
|
const interval = numberFlag(args.flags, 'interval', 1500)!;
|
|
@@ -434,6 +1092,7 @@ async function main(): Promise<void> {
|
|
|
434
1092
|
const statePath = flag(args.flags, 'state') ?? defaultStatePath(explicitAgent ?? 'computer');
|
|
435
1093
|
const once = boolFlag(args.flags, 'once');
|
|
436
1094
|
const dryRun = boolFlag(args.flags, 'dry-run');
|
|
1095
|
+
const verboseTicks = boolFlag(args.flags, 'verbose') || process.env.PAL_DAEMON_VERBOSE_TICKS === '1';
|
|
437
1096
|
const extraArgsRaw = flag(args.flags, 'neeko-args') ?? flag(args.flags, 'agent-args') ?? '';
|
|
438
1097
|
const extraArgs = extraArgsRaw ? extraArgsRaw.split(' ').filter(Boolean) : [];
|
|
439
1098
|
const privateCliBinDir = ensurePrivatePalCliBin();
|
|
@@ -442,16 +1101,19 @@ async function main(): Promise<void> {
|
|
|
442
1101
|
|
|
443
1102
|
const apiKey = flag(args.flags, 'api-key') ?? process.env.PAL_API_KEY;
|
|
444
1103
|
const computerId = flag(args.flags, 'computer-id') ?? process.env.PAL_COMPUTER_ID;
|
|
1104
|
+
const computerName = flag(args.flags, 'name') ?? flag(args.flags, 'computer-name') ?? process.env.PAL_COMPUTER_NAME;
|
|
445
1105
|
const computerSecret = flag(args.flags, 'computer-secret') ?? process.env.PAL_COMPUTER_SECRET;
|
|
446
1106
|
if (!apiKey?.trim() && !computerId?.trim()) throw new Error('--api-key, --computer-id, or PAL_COMPUTER_ID is required');
|
|
447
1107
|
if (!apiKey?.trim() && !computerSecret?.trim()) throw new Error('--api-key, --computer-secret, or PAL_COMPUTER_SECRET is required');
|
|
448
1108
|
|
|
449
1109
|
const explicitAgents = new Set<string>();
|
|
450
1110
|
const managedAgents = new Map<string, ManagedAgent>();
|
|
1111
|
+
const activeRuns: ActiveRunMap = new Map();
|
|
1112
|
+
const runtimeTokens = new RuntimeLocalTokenRegistry();
|
|
451
1113
|
if (explicitAgent) {
|
|
452
1114
|
explicitAgents.add(explicitAgent);
|
|
453
1115
|
const managed = await buildManagedAgent({
|
|
454
|
-
assignment: { agent: explicitAgent, runtime: null, cwd },
|
|
1116
|
+
assignment: { agent: explicitAgent, runtime: null, model: null, cwd },
|
|
455
1117
|
state,
|
|
456
1118
|
serverUrl,
|
|
457
1119
|
defaultCwd: cwd,
|
|
@@ -459,20 +1121,78 @@ async function main(): Promise<void> {
|
|
|
459
1121
|
managedAgents.set(explicitAgent, managed);
|
|
460
1122
|
}
|
|
461
1123
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
1124
|
+
type ConnectedComputer = Awaited<ReturnType<LockClient['connectComputer']>>;
|
|
1125
|
+
let connected!: ConnectedComputer;
|
|
1126
|
+
let daemonAuth!: { computer_id: string; connection_id: string; token: string };
|
|
1127
|
+
let client!: LockClient;
|
|
1128
|
+
let deliverySocket!: DeliverySocketHandle;
|
|
1129
|
+
let reconnecting = false;
|
|
1130
|
+
let onDeliveryFrame: (agent?: string) => void = () => {};
|
|
1131
|
+
|
|
1132
|
+
const connectAndInstall = async (reason: 'startup' | 'reconnect'): Promise<void> => {
|
|
1133
|
+
const previousConnectionId = reason === 'reconnect' ? connected.connection.id : null;
|
|
1134
|
+
const next = await bootstrapClient.connectComputer({
|
|
1135
|
+
computer_id: computerId,
|
|
1136
|
+
secret: computerSecret,
|
|
1137
|
+
api_key: apiKey,
|
|
1138
|
+
name: computerName,
|
|
1139
|
+
server_url: serverUrl,
|
|
1140
|
+
agents: daemonAgentPayload(managedAgents),
|
|
1141
|
+
});
|
|
1142
|
+
connected = next;
|
|
1143
|
+
state.computerId = next.computer.id;
|
|
1144
|
+
state.connectionId = next.connection.id;
|
|
1145
|
+
state.daemonId = next.connection.id;
|
|
1146
|
+
daemonAuth = { computer_id: next.computer.id, connection_id: next.connection.id, token: next.token };
|
|
1147
|
+
client = new LockClient(serverUrl, daemonAuth);
|
|
1148
|
+
deliverySocket?.stop();
|
|
1149
|
+
deliverySocket = startDeliveryWebSocket({
|
|
1150
|
+
serverUrl,
|
|
1151
|
+
computerId: next.computer.id,
|
|
1152
|
+
connectionId: next.connection.id,
|
|
1153
|
+
token: next.token,
|
|
1154
|
+
onDelivery: (agent) => onDeliveryFrame(agent),
|
|
1155
|
+
});
|
|
1156
|
+
writeState(statePath, state);
|
|
1157
|
+
if (reason === 'reconnect') {
|
|
1158
|
+
console.log(`[daemon] reconnected computer=${next.computer.id} connection=${next.connection.id}${previousConnectionId ? ` previous=${previousConnectionId}` : ''}`);
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
473
1161
|
|
|
474
|
-
|
|
475
|
-
const
|
|
1162
|
+
let running = true;
|
|
1163
|
+
const runtimeAbortController = new AbortController();
|
|
1164
|
+
process.on('SIGINT', () => { running = false; runtimeAbortController.abort(); });
|
|
1165
|
+
process.on('SIGTERM', () => { running = false; runtimeAbortController.abort(); });
|
|
1166
|
+
|
|
1167
|
+
const connectAndInstallWithRetry = async (reason: 'startup' | 'reconnect'): Promise<boolean> => {
|
|
1168
|
+
let attempt = 0;
|
|
1169
|
+
while (running) {
|
|
1170
|
+
try {
|
|
1171
|
+
await connectAndInstall(reason);
|
|
1172
|
+
return true;
|
|
1173
|
+
} catch (error) {
|
|
1174
|
+
if (!shouldRetryDaemonConnectError(error)) throw error;
|
|
1175
|
+
attempt += 1;
|
|
1176
|
+
const delayMs = Math.min(10_000, 500 * 2 ** Math.min(attempt, 5));
|
|
1177
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1178
|
+
console.warn(`[daemon] ${reason} connection failed, will retry in ${delayMs}ms: ${message}`);
|
|
1179
|
+
await sleep(delayMs);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
return false;
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
if (!await connectAndInstallWithRetry('startup')) return;
|
|
1186
|
+
const localServer = startLocalApi({
|
|
1187
|
+
host: localHost,
|
|
1188
|
+
port: localPort,
|
|
1189
|
+
serverUrl,
|
|
1190
|
+
token: localToken,
|
|
1191
|
+
controlToken: () => connected.local_control_token,
|
|
1192
|
+
daemonAuth: () => daemonAuth,
|
|
1193
|
+
runtimeTokenLookup: (token) => runtimeTokens.lookup(token),
|
|
1194
|
+
});
|
|
1195
|
+
const localUrl = `http://${localServer.hostname}:${localServer.port}`;
|
|
476
1196
|
await reconcileManagedAgents({
|
|
477
1197
|
assignments: connected.agents ?? [],
|
|
478
1198
|
explicitAgents,
|
|
@@ -483,40 +1203,159 @@ async function main(): Promise<void> {
|
|
|
483
1203
|
computerId: connected.computer.id,
|
|
484
1204
|
serverUrl,
|
|
485
1205
|
defaultCwd: cwd,
|
|
1206
|
+
localUrl,
|
|
486
1207
|
});
|
|
487
1208
|
if (managedAgents.size === 0) {
|
|
488
1209
|
console.log(`[daemon] no agents currently assigned to computer ${connected.computer.id}; waiting for assignments`);
|
|
489
1210
|
}
|
|
490
1211
|
writeState(statePath, state);
|
|
491
|
-
let connectionRevoked = false;
|
|
492
1212
|
const heartbeatMs = numberFlag(args.flags, 'heartbeat-interval', 5000)!;
|
|
1213
|
+
const heartbeatWarningState = defaultRepeatedWarningState();
|
|
1214
|
+
const heartbeatWarningIntervalMs = 30_000;
|
|
1215
|
+
let heartbeatInFlight = false;
|
|
493
1216
|
|
|
494
|
-
const
|
|
1217
|
+
const warnHeartbeatRetry = (message: string) => {
|
|
1218
|
+
const decision = consumeRepeatedWarning(heartbeatWarningState, {
|
|
1219
|
+
key: `${connected.connection.id}:${message}`,
|
|
1220
|
+
nowMs: Date.now(),
|
|
1221
|
+
minIntervalMs: heartbeatWarningIntervalMs,
|
|
1222
|
+
});
|
|
1223
|
+
if (!decision.shouldLog) return;
|
|
1224
|
+
const suppressed = decision.suppressed > 0 ? ` suppressed_repeats=${decision.suppressed}` : '';
|
|
1225
|
+
console.warn(`[daemon] heartbeat failed, will retry connection=${connected.connection.id}: ${message}${suppressed}`);
|
|
1226
|
+
};
|
|
1227
|
+
const runHeartbeat = async () => {
|
|
1228
|
+
if (heartbeatInFlight) return null;
|
|
1229
|
+
heartbeatInFlight = true;
|
|
1230
|
+
try {
|
|
1231
|
+
return await client.heartbeatComputer(connected.computer.id);
|
|
1232
|
+
} finally {
|
|
1233
|
+
heartbeatInFlight = false;
|
|
1234
|
+
}
|
|
1235
|
+
};
|
|
495
1236
|
|
|
496
1237
|
console.log(`pal daemon computer=${connected.computer.id} connection=${connected.connection.id} agents=${Array.from(managedAgents.values()).map((managed) => `${managed.agent}:${managed.runtimeProvider}`).join(',') || 'none'} server=${serverUrl}`);
|
|
497
1238
|
console.log(`local api=http://${localServer.hostname}:${localServer.port} token=${localToken ? 'set' : 'missing'}`);
|
|
498
1239
|
console.log(`state=${statePath} lastSeenId=${state.lastSeenId}`);
|
|
499
1240
|
console.log(`private cli bin=${privateCliBinDir}`);
|
|
500
1241
|
|
|
501
|
-
let running = true;
|
|
502
|
-
const runtimeAbortController = new AbortController();
|
|
503
|
-
process.on('SIGINT', () => { running = false; runtimeAbortController.abort(); });
|
|
504
|
-
process.on('SIGTERM', () => { running = false; runtimeAbortController.abort(); });
|
|
505
1242
|
const heartbeatTimer = setInterval(() => {
|
|
506
|
-
void
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
1243
|
+
void runHeartbeat().catch((error) => {
|
|
1244
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1245
|
+
if (shouldStopDaemonForHeartbeatError(error)) {
|
|
1246
|
+
void reconnectConnection(message);
|
|
1247
|
+
} else {
|
|
1248
|
+
warnHeartbeatRetry(message);
|
|
1249
|
+
}
|
|
511
1250
|
});
|
|
512
1251
|
}, heartbeatMs);
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
1252
|
+
const processingAgents = new Set<string>();
|
|
1253
|
+
const queuedProcessingAgents = new Set<string>();
|
|
1254
|
+
const activeDeliveryTasks = new Set<Promise<void>>();
|
|
1255
|
+
const scheduleDeliveryProcessing = (agent?: string) => {
|
|
1256
|
+
for (const managed of managedAgents.values()) {
|
|
1257
|
+
if (agent && managed.agent !== agent) continue;
|
|
1258
|
+
if (processingAgents.has(managed.agent)) {
|
|
1259
|
+
queuedProcessingAgents.add(managed.agent);
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
processingAgents.add(managed.agent);
|
|
1263
|
+
const taskClient = client;
|
|
1264
|
+
const taskConnection = connected;
|
|
1265
|
+
const task = (async () => {
|
|
1266
|
+
try {
|
|
1267
|
+
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
1268
|
+
await discoverDeliveries(taskClient, state, managed.agent);
|
|
1269
|
+
writeState(statePath, state);
|
|
1270
|
+
}
|
|
1271
|
+
const started = await processPendingDeliveries(taskClient, {
|
|
1272
|
+
agent: managed.agent,
|
|
1273
|
+
daemonId: taskConnection.connection.id,
|
|
1274
|
+
serverUrl,
|
|
1275
|
+
agentHome: managed.agentHome,
|
|
1276
|
+
projectCwd: managed.projectCwd,
|
|
1277
|
+
extraArgs,
|
|
1278
|
+
dryRun,
|
|
1279
|
+
agentUuid: managed.agentUuid,
|
|
1280
|
+
computerId: taskConnection.computer.id,
|
|
1281
|
+
connectionId: taskConnection.connection.id,
|
|
1282
|
+
runtimeProvider: managed.runtimeProvider,
|
|
1283
|
+
runtimeModel: managed.runtimeModel,
|
|
1284
|
+
permissionProfile: await loadPermissionProfile(taskClient, managed.agent),
|
|
1285
|
+
runtimeTokens,
|
|
1286
|
+
isConnectionRevoked: () => isDeliveryConnectionSuperseded(connected.connection.id, taskConnection.connection.id),
|
|
1287
|
+
abortSignal: runtimeAbortController.signal,
|
|
1288
|
+
localDaemonUrl: `http://${localServer.hostname}:${localServer.port}`,
|
|
1289
|
+
localDaemonToken: localToken,
|
|
1290
|
+
privateCliBinDir,
|
|
1291
|
+
runtimeEnv: runtimeEnvReport.effectiveEnv,
|
|
1292
|
+
activeRuns,
|
|
1293
|
+
});
|
|
1294
|
+
for (const runTask of started) {
|
|
1295
|
+
activeDeliveryTasks.add(runTask);
|
|
1296
|
+
runTask.finally(() => activeDeliveryTasks.delete(runTask));
|
|
1297
|
+
}
|
|
1298
|
+
} catch (error) {
|
|
1299
|
+
console.warn(`[daemon] delivery processing failed for agent=${managed.agent}; will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
1300
|
+
} finally {
|
|
1301
|
+
processingAgents.delete(managed.agent);
|
|
1302
|
+
if (queuedProcessingAgents.delete(managed.agent)) {
|
|
1303
|
+
setTimeout(() => scheduleDeliveryProcessing(managed.agent), 0);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
})();
|
|
1307
|
+
activeDeliveryTasks.add(task);
|
|
1308
|
+
task.finally(() => activeDeliveryTasks.delete(task));
|
|
1309
|
+
}
|
|
1310
|
+
};
|
|
1311
|
+
const schedulePendingDeliveryProcessing = async () => {
|
|
1312
|
+
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
1313
|
+
scheduleDeliveryProcessing();
|
|
1314
|
+
return;
|
|
1315
|
+
}
|
|
516
1316
|
try {
|
|
517
|
-
const
|
|
1317
|
+
const pendingAgents = await client.listPendingDeliveryAgents();
|
|
1318
|
+
for (const { agent } of pendingAgents) scheduleDeliveryProcessing(agent);
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
console.warn(`[daemon] pending delivery agent probe failed; falling back to all agents: ${error instanceof Error ? error.message : String(error)}`);
|
|
1321
|
+
scheduleDeliveryProcessing();
|
|
1322
|
+
}
|
|
1323
|
+
};
|
|
1324
|
+
const reconcileFromHeartbeat = async (): Promise<void> => {
|
|
1325
|
+
const heartbeat = await runHeartbeat();
|
|
1326
|
+
if (!heartbeat) return;
|
|
1327
|
+
await reconcileManagedAgents({
|
|
1328
|
+
assignments: heartbeat.agents ?? [],
|
|
1329
|
+
explicitAgents,
|
|
1330
|
+
managedAgents,
|
|
1331
|
+
state,
|
|
1332
|
+
client,
|
|
1333
|
+
daemonId: connected.connection.id,
|
|
1334
|
+
computerId: connected.computer.id,
|
|
1335
|
+
serverUrl,
|
|
1336
|
+
defaultCwd: cwd,
|
|
1337
|
+
localUrl,
|
|
1338
|
+
});
|
|
1339
|
+
writeState(statePath, state);
|
|
1340
|
+
};
|
|
1341
|
+
const handleDeliveryWake = async (agent?: string): Promise<void> => {
|
|
1342
|
+
if (agent && !managedAgents.has(agent)) {
|
|
1343
|
+
await reconcileFromHeartbeat().catch((error) => {
|
|
1344
|
+
console.warn(`[daemon] delivery wake reconcile failed for agent=${agent}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1345
|
+
});
|
|
1346
|
+
}
|
|
1347
|
+
scheduleDeliveryProcessing(agent);
|
|
1348
|
+
};
|
|
1349
|
+
const deliveryWakeQueue = new DeliveryWakeQueue((agent) => { void handleDeliveryWake(agent); });
|
|
1350
|
+
onDeliveryFrame = (agent) => deliveryWakeQueue.enqueue(agent);
|
|
1351
|
+
const reconnectConnection = async (message: string): Promise<boolean> => {
|
|
1352
|
+
if (reconnecting) return false;
|
|
1353
|
+
reconnecting = true;
|
|
1354
|
+
console.warn(`[daemon] connection ${connected.connection.id} is no longer active; reconnecting: ${message}`);
|
|
1355
|
+
try {
|
|
1356
|
+
if (!await connectAndInstallWithRetry('reconnect')) return false;
|
|
518
1357
|
await reconcileManagedAgents({
|
|
519
|
-
assignments:
|
|
1358
|
+
assignments: connected.agents ?? [],
|
|
520
1359
|
explicitAgents,
|
|
521
1360
|
managedAgents,
|
|
522
1361
|
state,
|
|
@@ -525,56 +1364,74 @@ async function main(): Promise<void> {
|
|
|
525
1364
|
computerId: connected.computer.id,
|
|
526
1365
|
serverUrl,
|
|
527
1366
|
defaultCwd: cwd,
|
|
1367
|
+
localUrl,
|
|
528
1368
|
});
|
|
1369
|
+
await schedulePendingDeliveryProcessing();
|
|
1370
|
+
return true;
|
|
1371
|
+
} catch (error) {
|
|
1372
|
+
console.warn(`[daemon] reconnect failed, will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
1373
|
+
return false;
|
|
1374
|
+
} finally {
|
|
1375
|
+
reconnecting = false;
|
|
1376
|
+
}
|
|
1377
|
+
};
|
|
1378
|
+
|
|
1379
|
+
while (running) {
|
|
1380
|
+
if (verboseTicks) {
|
|
1381
|
+
console.log(`[daemon] tick reconcile interval=${interval}ms once=${once} ws=${deliverySocket.isOpen() ? 'open' : 'closed'}`);
|
|
1382
|
+
}
|
|
1383
|
+
try {
|
|
1384
|
+
const heartbeat = await runHeartbeat();
|
|
1385
|
+
if (heartbeat) {
|
|
1386
|
+
await reconcileManagedAgents({
|
|
1387
|
+
assignments: heartbeat.agents ?? [],
|
|
1388
|
+
explicitAgents,
|
|
1389
|
+
managedAgents,
|
|
1390
|
+
state,
|
|
1391
|
+
client,
|
|
1392
|
+
daemonId: connected.connection.id,
|
|
1393
|
+
computerId: connected.computer.id,
|
|
1394
|
+
serverUrl,
|
|
1395
|
+
defaultCwd: cwd,
|
|
1396
|
+
localUrl,
|
|
1397
|
+
});
|
|
1398
|
+
}
|
|
1399
|
+
await schedulePendingDeliveryProcessing();
|
|
529
1400
|
writeState(statePath, state);
|
|
530
1401
|
} catch (error) {
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
1402
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1403
|
+
if (shouldStopDaemonForHeartbeatError(error)) {
|
|
1404
|
+
await reconnectConnection(message);
|
|
1405
|
+
} else {
|
|
1406
|
+
warnHeartbeatRetry(message);
|
|
1407
|
+
}
|
|
536
1408
|
}
|
|
537
1409
|
if (managedAgents.size === 0) {
|
|
538
1410
|
console.log(`[daemon] no assigned agents; idle`);
|
|
539
1411
|
}
|
|
540
|
-
for (const managed of managedAgents.values()) {
|
|
541
|
-
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
542
|
-
await discoverDeliveries(client, state, managed.agent);
|
|
543
|
-
writeState(statePath, state);
|
|
544
|
-
}
|
|
545
|
-
await processPendingDeliveries(client, {
|
|
546
|
-
agent: managed.agent,
|
|
547
|
-
daemonId: connected.connection.id,
|
|
548
|
-
serverUrl,
|
|
549
|
-
agentHome: managed.agentHome,
|
|
550
|
-
projectCwd: managed.projectCwd,
|
|
551
|
-
extraArgs,
|
|
552
|
-
dryRun,
|
|
553
|
-
agentUuid: managed.agentUuid,
|
|
554
|
-
computerId: connected.computer.id,
|
|
555
|
-
connectionId: connected.connection.id,
|
|
556
|
-
runtimeProvider: managed.runtimeProvider,
|
|
557
|
-
isConnectionRevoked: () => connectionRevoked,
|
|
558
|
-
abortSignal: runtimeAbortController.signal,
|
|
559
|
-
localDaemonUrl: `http://${localServer.hostname}:${localServer.port}`,
|
|
560
|
-
localDaemonToken: localToken,
|
|
561
|
-
privateCliBinDir,
|
|
562
|
-
});
|
|
563
|
-
}
|
|
564
1412
|
|
|
565
1413
|
if (once) {
|
|
1414
|
+
if (activeDeliveryTasks.size) {
|
|
1415
|
+
console.log(`[daemon] --once waiting for ${activeDeliveryTasks.size} active delivery task(s)`);
|
|
1416
|
+
await Promise.allSettled(Array.from(activeDeliveryTasks));
|
|
1417
|
+
}
|
|
566
1418
|
console.log(`[daemon] --once set, exiting loop`);
|
|
567
1419
|
break;
|
|
568
1420
|
}
|
|
569
|
-
console.log(`[daemon] sleep ${interval}ms`);
|
|
1421
|
+
if (verboseTicks) console.log(`[daemon] sleep ${interval}ms`);
|
|
570
1422
|
await sleep(interval);
|
|
571
1423
|
}
|
|
572
1424
|
|
|
573
1425
|
clearInterval(heartbeatTimer);
|
|
1426
|
+
deliveryWakeQueue.stop();
|
|
1427
|
+
deliverySocket.stop();
|
|
1428
|
+
if (activeDeliveryTasks.size) await Promise.allSettled(Array.from(activeDeliveryTasks));
|
|
574
1429
|
localServer.stop(true);
|
|
575
1430
|
}
|
|
576
1431
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
1432
|
+
if (import.meta.main) {
|
|
1433
|
+
main().catch((error) => {
|
|
1434
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
1435
|
+
process.exit(1);
|
|
1436
|
+
});
|
|
1437
|
+
}
|