@controlflow-ai/daemon 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -6
- package/package.json +3 -1
- package/src/agent-avatar.ts +30 -0
- package/src/agent-key.ts +28 -0
- package/src/agent-permissions.ts +359 -0
- package/src/agent-runtime.ts +795 -28
- package/src/agent-workspace.ts +183 -0
- package/src/app.ts +1970 -79
- package/src/args.ts +54 -7
- package/src/cli.ts +873 -14
- package/src/client.ts +472 -10
- package/src/coco.ts +9 -40
- package/src/codex.ts +33 -5
- package/src/config.ts +28 -4
- package/src/console.ts +230 -20
- package/src/daemon-client.ts +116 -3
- package/src/daemon.ts +936 -98
- package/src/db.ts +3128 -122
- package/src/delivery-ws.ts +269 -0
- package/src/format.ts +4 -1
- package/src/lark/cli.ts +3 -3
- package/src/lark/event-router.ts +60 -4
- package/src/lark/inbound-events.ts +156 -3
- package/src/lark/server-integration.ts +659 -111
- package/src/lark/ws-daemon.ts +136 -10
- package/src/local-api.ts +545 -15
- package/src/local-auth.ts +33 -1
- package/src/message-attachments.ts +71 -0
- package/src/messaging-cli.ts +741 -0
- package/src/messaging-status.ts +669 -0
- package/src/migrations/024_agents_model.ts +10 -0
- package/src/migrations/025_room_archive.ts +44 -0
- package/src/migrations/026_project_archive.ts +44 -0
- package/src/migrations/027_agent_permission_profiles.ts +16 -0
- package/src/migrations/028_lark_websocket_restart_state.ts +16 -0
- package/src/migrations/029_held_message_drafts.ts +32 -0
- package/src/migrations/030_agent_room_read_state.ts +25 -0
- package/src/migrations/031_room_tasks.ts +29 -0
- package/src/migrations/032_room_reminders.ts +29 -0
- package/src/migrations/033_room_saved_messages.ts +25 -0
- package/src/migrations/034_agent_activity_events.ts +27 -0
- package/src/migrations/035_agent_avatars.ts +17 -0
- package/src/migrations/036_project_agent_defaults.ts +21 -0
- package/src/migrations/037_message_attachments.ts +36 -0
- package/src/migrations/038_agent_activity_room_scope.ts +64 -0
- package/src/migrations/039_message_attachments_path.ts +34 -0
- package/src/migrations/040_message_attachments_file_schema.ts +80 -0
- package/src/migrations/041_room_system_events.ts +30 -0
- package/src/migrations/042_message_attachment_file_kind.ts +52 -0
- package/src/migrations/043_room_mode_skill_registry.ts +92 -0
- package/src/migrations/044_workflow_runtime.ts +69 -0
- package/src/migrations/045_skill_repository_ownership.ts +64 -0
- package/src/migrations.ts +69 -1
- package/src/neeko.ts +40 -4
- package/src/runtime-env.ts +179 -0
- package/src/runtime-registry.ts +83 -13
- package/src/server.ts +244 -4
- package/src/token-file.ts +13 -6
- package/src/types.ts +362 -0
- package/src/workflow-runtime.ts +275 -0
- package/src/web.ts +0 -904
package/src/daemon.ts
CHANGED
|
@@ -6,10 +6,14 @@ import { DEFAULT_DAEMON_PORT, DEFAULT_HOST, agentHomePath, defaultDaemonToken, d
|
|
|
6
6
|
import { writeFailureMessage } from './failure-message.js';
|
|
7
7
|
import { formatMessage } from './format.js';
|
|
8
8
|
import { startLocalApi } from './local-api.js';
|
|
9
|
+
import type { LocalApiAgentPrincipal } from './local-auth.js';
|
|
9
10
|
import { loadOrCreateDaemonToken } from './token-file.js';
|
|
10
11
|
import type { ComputerAgentAssignment, Message, MessageDelivery } from './types.js';
|
|
11
12
|
import type { AgentRuntime } from './agent-runtime.js';
|
|
12
13
|
import { knownRuntimeNames, resolveRuntimeDriver } from './runtime-registry.js';
|
|
14
|
+
import { buildRuntimeLaunchContext, type AgentPermissionProfile } from './agent-permissions.js';
|
|
15
|
+
import { hydrateRuntimeEnv, logEnvReport } from './runtime-env.js';
|
|
16
|
+
import type { RuntimeAttachment } from './agent-runtime.js';
|
|
13
17
|
|
|
14
18
|
interface DaemonState {
|
|
15
19
|
daemonId: string;
|
|
@@ -23,11 +27,136 @@ interface DaemonState {
|
|
|
23
27
|
interface ManagedAgent {
|
|
24
28
|
agent: string;
|
|
25
29
|
runtimeProvider: string;
|
|
30
|
+
runtimeModel: string | null;
|
|
26
31
|
agentUuid: string;
|
|
27
32
|
agentHome: string;
|
|
28
33
|
projectCwd: string;
|
|
29
34
|
}
|
|
30
35
|
|
|
36
|
+
export interface SteeredDelivery {
|
|
37
|
+
id: string;
|
|
38
|
+
claimToken: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface ActiveRunHandle {
|
|
42
|
+
key: string;
|
|
43
|
+
agent: string;
|
|
44
|
+
chatId: string;
|
|
45
|
+
runId: string;
|
|
46
|
+
sessionId: string;
|
|
47
|
+
runtimeSessionId: string | null;
|
|
48
|
+
supportsSteer: boolean;
|
|
49
|
+
steeredDeliveries: SteeredDelivery[];
|
|
50
|
+
steer(message: Message): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type ActiveRunMap = Map<string, ActiveRunHandle>;
|
|
54
|
+
|
|
55
|
+
export class RuntimeLocalTokenRegistry {
|
|
56
|
+
private readonly tokens = new Map<string, LocalApiAgentPrincipal>();
|
|
57
|
+
|
|
58
|
+
create(input: { agent: string; runId: string; chatId: string }): string {
|
|
59
|
+
const token = `palrt_${crypto.randomUUID()}`;
|
|
60
|
+
this.tokens.set(token, { kind: 'agent', agent: input.agent, runId: input.runId, chatId: input.chatId });
|
|
61
|
+
return token;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
lookup(token: string): LocalApiAgentPrincipal | null {
|
|
65
|
+
return this.tokens.get(token) ?? null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
revoke(token: string): void {
|
|
69
|
+
this.tokens.delete(token);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function activeRunKey(agent: string, chatId: string): string {
|
|
74
|
+
return `${agent}\0${chatId}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function shouldStopDaemonForHeartbeatError(error: unknown): boolean {
|
|
78
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
79
|
+
return message.includes('CONNECTION_REVOKED')
|
|
80
|
+
|| message.includes('computer connection is not active')
|
|
81
|
+
|| message.includes('connection auth is required')
|
|
82
|
+
|| message.includes('invalid computer credential');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function shouldRetryDaemonConnectError(error: unknown): boolean {
|
|
86
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
87
|
+
return message.includes('fetch failed')
|
|
88
|
+
|| message.includes('Unable to connect')
|
|
89
|
+
|| message.includes('ECONNREFUSED')
|
|
90
|
+
|| message.includes('ECONNRESET')
|
|
91
|
+
|| message.includes('EPIPE')
|
|
92
|
+
|| message.includes('UND_ERR')
|
|
93
|
+
|| message.includes('Unexpected end of JSON input')
|
|
94
|
+
|| message.includes('request failed: 502')
|
|
95
|
+
|| message.includes('request failed: 503')
|
|
96
|
+
|| message.includes('request failed: 504');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function isDeliveryConnectionSuperseded(currentConnectionId: string | null | undefined, runConnectionId: string | null | undefined): boolean {
|
|
100
|
+
return Boolean(runConnectionId && currentConnectionId && runConnectionId !== currentConnectionId);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface RepeatedWarningState {
|
|
104
|
+
lastKey: string | null;
|
|
105
|
+
lastLoggedAtMs: number;
|
|
106
|
+
suppressed: number;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function defaultRepeatedWarningState(): RepeatedWarningState {
|
|
110
|
+
return { lastKey: null, lastLoggedAtMs: 0, suppressed: 0 };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function consumeRepeatedWarning(state: RepeatedWarningState, input: {
|
|
114
|
+
key: string;
|
|
115
|
+
nowMs: number;
|
|
116
|
+
minIntervalMs: number;
|
|
117
|
+
}): { shouldLog: boolean; suppressed: number } {
|
|
118
|
+
const keyChanged = state.lastKey !== input.key;
|
|
119
|
+
const intervalElapsed = input.nowMs - state.lastLoggedAtMs >= input.minIntervalMs;
|
|
120
|
+
if (keyChanged || intervalElapsed) {
|
|
121
|
+
const suppressed = state.suppressed;
|
|
122
|
+
state.lastKey = input.key;
|
|
123
|
+
state.lastLoggedAtMs = input.nowMs;
|
|
124
|
+
state.suppressed = 0;
|
|
125
|
+
return { shouldLog: true, suppressed };
|
|
126
|
+
}
|
|
127
|
+
state.suppressed += 1;
|
|
128
|
+
return { shouldLog: false, suppressed: state.suppressed };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export type PendingDeliveryAction =
|
|
132
|
+
| { kind: 'start'; delivery: MessageDelivery; key: string }
|
|
133
|
+
| { kind: 'steer'; delivery: MessageDelivery; key: string; active: ActiveRunHandle }
|
|
134
|
+
| { kind: 'skip'; delivery: MessageDelivery; key: string; reason: 'starting' | 'active-no-steer' };
|
|
135
|
+
|
|
136
|
+
const PENDING_DELIVERY_CANDIDATE_LIMIT = 100;
|
|
137
|
+
|
|
138
|
+
export function planPendingDeliveryActions(agent: string, deliveries: MessageDelivery[], activeRuns: ActiveRunMap): PendingDeliveryAction[] {
|
|
139
|
+
const actions: PendingDeliveryAction[] = [];
|
|
140
|
+
const startingKeys = new Set<string>();
|
|
141
|
+
for (const delivery of deliveries) {
|
|
142
|
+
const key = activeRunKey(agent, delivery.chat_id);
|
|
143
|
+
const active = activeRuns.get(key);
|
|
144
|
+
if (active) {
|
|
145
|
+
actions.push(active.supportsSteer
|
|
146
|
+
? { kind: 'steer', delivery, key, active }
|
|
147
|
+
: { kind: 'skip', delivery, key, reason: 'active-no-steer' });
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (startingKeys.has(key)) {
|
|
151
|
+
actions.push({ kind: 'skip', delivery, key, reason: 'starting' });
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
startingKeys.add(key);
|
|
155
|
+
actions.push({ kind: 'start', delivery, key });
|
|
156
|
+
}
|
|
157
|
+
return actions;
|
|
158
|
+
}
|
|
159
|
+
|
|
31
160
|
function usage(): string {
|
|
32
161
|
return `pal daemon
|
|
33
162
|
|
|
@@ -37,6 +166,7 @@ Usage:
|
|
|
37
166
|
|
|
38
167
|
The daemon connects one computer to the server, claims deliveries for assigned agents, and starts one agent run for each claimed delivery.
|
|
39
168
|
Runs have no default timeout; use the web control page or API to kill/restart a run.
|
|
169
|
+
Use --verbose or PAL_DAEMON_VERBOSE_TICKS=1 to log empty reconcile ticks.
|
|
40
170
|
`;
|
|
41
171
|
}
|
|
42
172
|
|
|
@@ -75,18 +205,224 @@ function writeState(path: string, state: DaemonState): void {
|
|
|
75
205
|
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
|
|
76
206
|
}
|
|
77
207
|
|
|
208
|
+
function runtimeAttachmentsForMessage(input: {
|
|
209
|
+
message: Message;
|
|
210
|
+
}): RuntimeAttachment[] {
|
|
211
|
+
const attachments = input.message.attachments ?? [];
|
|
212
|
+
if (attachments.length === 0) return [];
|
|
213
|
+
return attachments.map((attachment) => ({
|
|
214
|
+
id: attachment.id,
|
|
215
|
+
messageId: input.message.id,
|
|
216
|
+
kind: attachment.kind,
|
|
217
|
+
mimeType: attachment.mime_type,
|
|
218
|
+
filename: attachment.filename,
|
|
219
|
+
path: attachment.path,
|
|
220
|
+
}));
|
|
221
|
+
}
|
|
222
|
+
|
|
78
223
|
function sleep(ms: number): Promise<void> {
|
|
79
224
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
80
225
|
}
|
|
81
226
|
|
|
82
|
-
|
|
227
|
+
function deliveryWebSocketUrl(serverUrl: string): string {
|
|
228
|
+
const url = new URL(serverUrl);
|
|
229
|
+
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
230
|
+
url.pathname = '/api/daemon/ws';
|
|
231
|
+
url.search = '';
|
|
232
|
+
return url.toString();
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export interface DeliverySocketHandle {
|
|
236
|
+
stop(): void;
|
|
237
|
+
isOpen(): boolean;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const DELIVERY_WS_PING_INTERVAL_MS = 30_000;
|
|
241
|
+
const DELIVERY_WS_PONG_TIMEOUT_MS = 10_000;
|
|
242
|
+
|
|
243
|
+
export class DeliveryWakeQueue {
|
|
244
|
+
private pendingAgents = new Set<string>();
|
|
245
|
+
private pendingAll = false;
|
|
246
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
247
|
+
private readonly delayMs: number;
|
|
248
|
+
|
|
249
|
+
constructor(
|
|
250
|
+
private readonly flush: (agent?: string) => void,
|
|
251
|
+
options: { delayMs?: number } = {},
|
|
252
|
+
) {
|
|
253
|
+
this.delayMs = Math.max(0, options.delayMs ?? 0);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
enqueue(agent?: string): void {
|
|
257
|
+
if (agent) {
|
|
258
|
+
if (!this.pendingAll) this.pendingAgents.add(agent);
|
|
259
|
+
} else {
|
|
260
|
+
this.pendingAll = true;
|
|
261
|
+
this.pendingAgents.clear();
|
|
262
|
+
}
|
|
263
|
+
if (this.timer) return;
|
|
264
|
+
this.timer = setTimeout(() => this.drain(), this.delayMs);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
drainNow(): void {
|
|
268
|
+
if (this.timer) clearTimeout(this.timer);
|
|
269
|
+
this.drain();
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
stop(): void {
|
|
273
|
+
if (this.timer) clearTimeout(this.timer);
|
|
274
|
+
this.timer = null;
|
|
275
|
+
this.pendingAll = false;
|
|
276
|
+
this.pendingAgents.clear();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private drain(): void {
|
|
280
|
+
this.timer = null;
|
|
281
|
+
if (this.pendingAll) {
|
|
282
|
+
this.pendingAll = false;
|
|
283
|
+
this.pendingAgents.clear();
|
|
284
|
+
this.flush();
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const agents = Array.from(this.pendingAgents);
|
|
288
|
+
this.pendingAgents.clear();
|
|
289
|
+
for (const agent of agents) this.flush(agent);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export function startDeliveryWebSocket(input: {
|
|
294
|
+
serverUrl: string;
|
|
295
|
+
computerId: string;
|
|
296
|
+
connectionId: string;
|
|
297
|
+
token: string;
|
|
298
|
+
onDelivery(agent?: string): void;
|
|
299
|
+
pingIntervalMs?: number;
|
|
300
|
+
pongTimeoutMs?: number;
|
|
301
|
+
}): DeliverySocketHandle {
|
|
302
|
+
let stopped = false;
|
|
303
|
+
let ws: WebSocket | null = null;
|
|
304
|
+
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
305
|
+
let pingTimer: ReturnType<typeof setInterval> | null = null;
|
|
306
|
+
let pongTimer: ReturnType<typeof setTimeout> | null = null;
|
|
307
|
+
let attempt = 0;
|
|
308
|
+
const pingIntervalMs = input.pingIntervalMs ?? DELIVERY_WS_PING_INTERVAL_MS;
|
|
309
|
+
const pongTimeoutMs = input.pongTimeoutMs ?? DELIVERY_WS_PONG_TIMEOUT_MS;
|
|
310
|
+
|
|
311
|
+
const clearKeepalive = () => {
|
|
312
|
+
if (pingTimer) clearInterval(pingTimer);
|
|
313
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
314
|
+
pingTimer = null;
|
|
315
|
+
pongTimer = null;
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
const closeUnhealthySocket = (reason: string) => {
|
|
319
|
+
const current = ws;
|
|
320
|
+
if (!current || current.readyState === WebSocket.CLOSED || current.readyState === WebSocket.CLOSING) return;
|
|
321
|
+
console.warn(`[daemon] delivery websocket ${reason}; reconnecting`);
|
|
322
|
+
current.close(4000, reason);
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
const sendPing = () => {
|
|
326
|
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
327
|
+
try {
|
|
328
|
+
ws.send('ping');
|
|
329
|
+
} catch {
|
|
330
|
+
closeUnhealthySocket('ping send failed');
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
334
|
+
pongTimer = setTimeout(() => closeUnhealthySocket('pong timeout'), pongTimeoutMs);
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
const connect = () => {
|
|
338
|
+
if (stopped) return;
|
|
339
|
+
const url = deliveryWebSocketUrl(input.serverUrl);
|
|
340
|
+
try {
|
|
341
|
+
ws = new (WebSocket as unknown as { new(url: string, options: Bun.WebSocketOptions): WebSocket })(url, {
|
|
342
|
+
headers: {
|
|
343
|
+
'x-pal-computer-id': input.computerId,
|
|
344
|
+
'x-pal-connection-id': input.connectionId,
|
|
345
|
+
'x-pal-connection-token': input.token,
|
|
346
|
+
},
|
|
347
|
+
});
|
|
348
|
+
} catch (error) {
|
|
349
|
+
scheduleReconnect(error);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
ws.addEventListener('open', () => {
|
|
354
|
+
attempt = 0;
|
|
355
|
+
console.log(`[daemon] delivery websocket connected ${url}`);
|
|
356
|
+
clearKeepalive();
|
|
357
|
+
if (pingIntervalMs > 0) {
|
|
358
|
+
pingTimer = setInterval(sendPing, pingIntervalMs);
|
|
359
|
+
sendPing();
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
ws.addEventListener('message', (event) => {
|
|
363
|
+
let frame: { type?: string; agent?: string; delivery?: { agent?: string } };
|
|
364
|
+
try {
|
|
365
|
+
frame = JSON.parse(String(event.data));
|
|
366
|
+
} catch {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (frame.type === 'delivery') {
|
|
370
|
+
input.onDelivery(frame.delivery?.agent);
|
|
371
|
+
} else if (frame.type === 'pending') {
|
|
372
|
+
input.onDelivery(frame.agent);
|
|
373
|
+
} else if (frame.type === 'pong') {
|
|
374
|
+
if (pongTimer) clearTimeout(pongTimer);
|
|
375
|
+
pongTimer = null;
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
ws.addEventListener('close', () => {
|
|
379
|
+
clearKeepalive();
|
|
380
|
+
if (!stopped) scheduleReconnect();
|
|
381
|
+
});
|
|
382
|
+
ws.addEventListener('error', (event) => {
|
|
383
|
+
if (!stopped) console.log(`[daemon] delivery websocket error: ${String(event.type)}`);
|
|
384
|
+
});
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const scheduleReconnect = (error?: unknown) => {
|
|
388
|
+
if (stopped || reconnectTimer) return;
|
|
389
|
+
if (error) {
|
|
390
|
+
console.warn(`[daemon] delivery websocket connect failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
391
|
+
}
|
|
392
|
+
attempt += 1;
|
|
393
|
+
const delay = Math.min(10_000, 250 * 2 ** Math.min(attempt, 5));
|
|
394
|
+
reconnectTimer = setTimeout(() => {
|
|
395
|
+
reconnectTimer = null;
|
|
396
|
+
connect();
|
|
397
|
+
}, delay);
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
connect();
|
|
401
|
+
return {
|
|
402
|
+
stop() {
|
|
403
|
+
stopped = true;
|
|
404
|
+
if (reconnectTimer) clearTimeout(reconnectTimer);
|
|
405
|
+
reconnectTimer = null;
|
|
406
|
+
clearKeepalive();
|
|
407
|
+
ws?.close(1000, 'daemon stopping');
|
|
408
|
+
ws = null;
|
|
409
|
+
},
|
|
410
|
+
isOpen() {
|
|
411
|
+
return ws?.readyState === WebSocket.OPEN;
|
|
412
|
+
},
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async function resolveRuntime(agent: string, serverUrl: string, agentUuid: string): Promise<{ runtime: AgentRuntime; runtimeModel: string | null }> {
|
|
83
417
|
let configuredRuntime: string | null | undefined;
|
|
418
|
+
let configuredModel: string | null | undefined;
|
|
84
419
|
try {
|
|
85
420
|
const response = await fetch(`${serverUrl.replace(/\/$/, '')}/api/agents`);
|
|
86
421
|
if (response.ok) {
|
|
87
|
-
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; runtime: string | null }> } };
|
|
422
|
+
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; runtime: string | null; model?: string | null }> } };
|
|
88
423
|
const configured = payload.data?.agents.find((a) => a.agent_key === agent);
|
|
89
424
|
configuredRuntime = configured?.runtime;
|
|
425
|
+
configuredModel = configured?.model;
|
|
90
426
|
}
|
|
91
427
|
} catch (error) {
|
|
92
428
|
throw new Error(`Could not resolve runtime for agent '${agent}' from ${serverUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -95,7 +431,114 @@ async function resolveRuntime(agent: string, serverUrl: string, agentUuid: strin
|
|
|
95
431
|
if (!configuredRuntime) {
|
|
96
432
|
throw new Error(`Agent '${agent}' has no runtime configured. Configure it via POST /api/agents or PATCH /api/agents/<key>. Supported runtimes: ${knownRuntimeNames().join(', ')}.`);
|
|
97
433
|
}
|
|
98
|
-
return
|
|
434
|
+
return {
|
|
435
|
+
runtime: await resolveRuntimeDriver(configuredRuntime, agentUuid, configuredModel),
|
|
436
|
+
runtimeModel: configuredModel?.trim() || null,
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function fetchConfiguredRuntimeModel(agent: string, serverUrl: string): Promise<string | null> {
|
|
441
|
+
try {
|
|
442
|
+
const response = await fetch(`${serverUrl.replace(/\/$/, '')}/api/agents`);
|
|
443
|
+
if (!response.ok) return null;
|
|
444
|
+
const payload = await response.json() as { data?: { agents: Array<{ agent_key: string; model?: string | null }> } };
|
|
445
|
+
return payload.data?.agents.find((item) => item.agent_key === agent)?.model?.trim() || null;
|
|
446
|
+
} catch {
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
async function loadPermissionProfile(client: LockClient, agent: string): Promise<AgentPermissionProfile> {
|
|
452
|
+
const profile = await client.fetchPermissionProfile(agent);
|
|
453
|
+
return {
|
|
454
|
+
filesystemMode: profile.filesystem_mode,
|
|
455
|
+
extraWritableRoots: profile.extra_writable_roots,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
async function ackSteeredDeliveries(
|
|
460
|
+
client: LockClient,
|
|
461
|
+
active: ActiveRunHandle,
|
|
462
|
+
options: { daemonId: string; connectionId?: string | null },
|
|
463
|
+
runId: string,
|
|
464
|
+
): Promise<void> {
|
|
465
|
+
for (const steered of active.steeredDeliveries) {
|
|
466
|
+
await client.ackDelivery(steered.id, {
|
|
467
|
+
daemon_id: options.daemonId,
|
|
468
|
+
connection_id: options.connectionId,
|
|
469
|
+
claim_token: steered.claimToken,
|
|
470
|
+
run_id: runId,
|
|
471
|
+
}).catch((error) => {
|
|
472
|
+
console.warn(`[daemon] failed to ack steered delivery ${steered.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async function failSteeredDeliveries(
|
|
478
|
+
client: LockClient,
|
|
479
|
+
active: ActiveRunHandle,
|
|
480
|
+
options: { daemonId: string; connectionId?: string | null },
|
|
481
|
+
runId: string,
|
|
482
|
+
error: string,
|
|
483
|
+
): Promise<void> {
|
|
484
|
+
for (const steered of active.steeredDeliveries) {
|
|
485
|
+
await client.failDelivery(steered.id, {
|
|
486
|
+
daemon_id: options.daemonId,
|
|
487
|
+
connection_id: options.connectionId,
|
|
488
|
+
claim_token: steered.claimToken,
|
|
489
|
+
run_id: runId,
|
|
490
|
+
error,
|
|
491
|
+
}).catch((failError) => {
|
|
492
|
+
console.warn(`[daemon] failed to fail steered delivery ${steered.id}: ${failError instanceof Error ? failError.message : String(failError)}`);
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function ackCoveredPendingDeliveries(
|
|
498
|
+
client: LockClient,
|
|
499
|
+
input: {
|
|
500
|
+
agent: string;
|
|
501
|
+
chatId: string;
|
|
502
|
+
triggerMessageId: number;
|
|
503
|
+
maxCoveredMessageId: number;
|
|
504
|
+
daemonId: string;
|
|
505
|
+
connectionId?: string | null;
|
|
506
|
+
computerId?: string | null;
|
|
507
|
+
runId: string;
|
|
508
|
+
},
|
|
509
|
+
): Promise<void> {
|
|
510
|
+
if (input.maxCoveredMessageId <= input.triggerMessageId) return;
|
|
511
|
+
const pending = await client.listDeliveries(input.agent, 'pending', PENDING_DELIVERY_CANDIDATE_LIMIT);
|
|
512
|
+
const covered = pending.filter((delivery) => (
|
|
513
|
+
delivery.chat_id === input.chatId
|
|
514
|
+
&& delivery.message_id > input.triggerMessageId
|
|
515
|
+
&& delivery.message_id <= input.maxCoveredMessageId
|
|
516
|
+
));
|
|
517
|
+
for (const delivery of covered) {
|
|
518
|
+
try {
|
|
519
|
+
console.log(`[daemon] ack covered pending delivery=${delivery.id} run=${input.runId} message=${delivery.message_id}`);
|
|
520
|
+
const claimed = await client.claimDelivery(delivery.id, {
|
|
521
|
+
daemon_id: input.daemonId,
|
|
522
|
+
connection_id: input.connectionId,
|
|
523
|
+
computer_id: input.computerId,
|
|
524
|
+
steer_run_id: input.runId,
|
|
525
|
+
});
|
|
526
|
+
if (!claimed.claim_token) throw new Error(`covered delivery ${delivery.id} was claimed without a claim token`);
|
|
527
|
+
await client.ackDelivery(claimed.id, {
|
|
528
|
+
daemon_id: input.daemonId,
|
|
529
|
+
connection_id: input.connectionId,
|
|
530
|
+
claim_token: claimed.claim_token,
|
|
531
|
+
run_id: input.runId,
|
|
532
|
+
});
|
|
533
|
+
} catch (error) {
|
|
534
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
535
|
+
if (message.includes('pending delivery') && message.includes('was not found')) {
|
|
536
|
+
console.log(`[daemon] covered pending delivery already claimed delivery=${delivery.id}`);
|
|
537
|
+
} else {
|
|
538
|
+
console.warn(`[daemon] failed to ack covered pending delivery ${delivery.id}: ${message}`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
99
542
|
}
|
|
100
543
|
|
|
101
544
|
async function executeRun(client: LockClient, delivery: MessageDelivery, message: Message, options: {
|
|
@@ -114,12 +557,18 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
114
557
|
computerId?: string | null;
|
|
115
558
|
connectionId?: string | null;
|
|
116
559
|
runtimeProvider: string;
|
|
560
|
+
runtimeModel?: string | null;
|
|
561
|
+
permissionProfile: AgentPermissionProfile;
|
|
562
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
117
563
|
isConnectionRevoked?: () => boolean;
|
|
118
564
|
abortSignal?: AbortSignal;
|
|
565
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
566
|
+
activeRuns?: ActiveRunMap;
|
|
119
567
|
}): Promise<'done' | 'restart'> {
|
|
120
568
|
if (!delivery.claim_token) throw new Error(`delivery ${delivery.id} is not claimed`);
|
|
121
569
|
|
|
122
|
-
const
|
|
570
|
+
const runtimeModel = options.runtimeModel ?? await fetchConfiguredRuntimeModel(options.agent, options.serverUrl);
|
|
571
|
+
const runtime = await resolveRuntimeDriver(options.runtimeProvider, options.agentUuid, runtimeModel);
|
|
123
572
|
const logPrefix = `[${runtime.name}]`;
|
|
124
573
|
|
|
125
574
|
console.log(`${logPrefix} session getOrCreate chat=${message.chat_id} agent=${options.agent} daemon=${options.daemonId} agentHome=${options.agentHome} projectCwd=${options.projectCwd}`);
|
|
@@ -151,15 +600,61 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
151
600
|
delivery_id: delivery.id,
|
|
152
601
|
});
|
|
153
602
|
console.log(`${logPrefix} run=${run.id} status=${run.status}`);
|
|
603
|
+
const deliveryContext = await client.getDeliveryContext({
|
|
604
|
+
agent: options.agent,
|
|
605
|
+
chatId: message.chat_id,
|
|
606
|
+
messageId: message.id,
|
|
607
|
+
limit: 50,
|
|
608
|
+
}).catch((error) => {
|
|
609
|
+
console.warn(`${logPrefix} delivery context unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
610
|
+
return null;
|
|
611
|
+
});
|
|
612
|
+
const runtimeLocalToken = options.runtimeTokens?.create({ agent: options.agent, runId: run.id, chatId: message.chat_id });
|
|
613
|
+
const key = activeRunKey(options.agent, message.chat_id);
|
|
614
|
+
const activeHandle: ActiveRunHandle = {
|
|
615
|
+
key,
|
|
616
|
+
agent: options.agent,
|
|
617
|
+
chatId: message.chat_id,
|
|
618
|
+
runId: run.id,
|
|
619
|
+
sessionId: session.id,
|
|
620
|
+
runtimeSessionId: session.runtime_session_id,
|
|
621
|
+
supportsSteer: runtime.capabilities.supportsSteer && typeof runtime.steerActiveRun === 'function',
|
|
622
|
+
steeredDeliveries: [],
|
|
623
|
+
steer: async (steerMessage) => {
|
|
624
|
+
if (!runtime.capabilities.supportsSteer || !runtime.steerActiveRun) {
|
|
625
|
+
throw new Error(`runtime ${runtime.name} does not support active steer`);
|
|
626
|
+
}
|
|
627
|
+
await runtime.steerActiveRun({
|
|
628
|
+
agent: options.agent,
|
|
629
|
+
serverUrl: options.serverUrl,
|
|
630
|
+
runId: run.id,
|
|
631
|
+
sessionId: session.id,
|
|
632
|
+
runtimeSessionId: activeHandle.runtimeSessionId,
|
|
633
|
+
message: steerMessage,
|
|
634
|
+
cwd: options.agentHome,
|
|
635
|
+
agentHome: options.agentHome,
|
|
636
|
+
projectCwd: options.projectCwd,
|
|
637
|
+
localDaemonUrl: options.localDaemonUrl,
|
|
638
|
+
localDaemonToken: runtimeLocalToken ?? options.localDaemonToken,
|
|
639
|
+
privateCliBinDir: options.privateCliBinDir,
|
|
640
|
+
palCliCommand: 'pal',
|
|
641
|
+
runtimeEnv: options.runtimeEnv,
|
|
642
|
+
});
|
|
643
|
+
},
|
|
644
|
+
};
|
|
645
|
+
options.activeRuns?.set(key, activeHandle);
|
|
646
|
+
console.log(`${logPrefix} active run key=${key.replace('\0', ':')} supportsSteer=${activeHandle.supportsSteer}`);
|
|
154
647
|
|
|
155
648
|
const { runAgentRuntime } = await import('./agent-runtime.js');
|
|
156
|
-
const roomSnapshot = await client.listRoomMembers(message.chat_id)
|
|
649
|
+
const roomSnapshot = await client.listRoomMembers(message.chat_id, options.agent)
|
|
157
650
|
.then((result) => result)
|
|
158
651
|
.catch((error) => {
|
|
159
652
|
console.warn(`${logPrefix} room participant snapshot unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
160
653
|
return null;
|
|
161
654
|
});
|
|
162
655
|
const roomParticipants = roomSnapshot?.participants ?? [];
|
|
656
|
+
const roomAgentSubscriptions = roomSnapshot?.agent_subscriptions ?? [];
|
|
657
|
+
const enabledSkills = roomSnapshot?.enabled_skills ?? [];
|
|
163
658
|
const roomProject = roomSnapshot?.room.project_id ? {
|
|
164
659
|
id: roomSnapshot.room.project_id,
|
|
165
660
|
name: roomSnapshot.room.project_name ?? roomSnapshot.room.project_id,
|
|
@@ -169,16 +664,19 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
169
664
|
} : null;
|
|
170
665
|
const projectAccessible = Boolean(roomProject && options.computerId && roomProject.computerId === options.computerId);
|
|
171
666
|
const effectiveProjectCwd = projectAccessible && roomProject?.rootPath ? roomProject.rootPath : options.projectCwd;
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
667
|
+
const launchContext = buildRuntimeLaunchContext({
|
|
668
|
+
agent: options.agent,
|
|
669
|
+
serverUrl: options.serverUrl,
|
|
670
|
+
message,
|
|
671
|
+
cwd: options.agentHome,
|
|
672
|
+
agentHome: options.agentHome,
|
|
673
|
+
projectCwd: projectAccessible ? effectiveProjectCwd : undefined,
|
|
674
|
+
extraArgs: options.extraArgs,
|
|
179
675
|
});
|
|
676
|
+
const maxCoveredMessageId = deliveryContext?.messages.reduce((max, item) => Math.max(max, item.id), message.id) ?? message.id;
|
|
180
677
|
|
|
181
678
|
try {
|
|
679
|
+
const runtimeAttachments = runtimeAttachmentsForMessage({ message });
|
|
182
680
|
console.log(`${logPrefix} spawn agent=${options.agent} chat=${message.chat_name} message=${message.id} dryRun=${options.dryRun}`);
|
|
183
681
|
const result = await runAgentRuntime(runtime, {
|
|
184
682
|
agent: options.agent,
|
|
@@ -187,6 +685,8 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
187
685
|
cwd: options.agentHome,
|
|
188
686
|
agentHome: options.agentHome,
|
|
189
687
|
projectCwd: effectiveProjectCwd,
|
|
688
|
+
launchContext,
|
|
689
|
+
permissionProfile: options.permissionProfile,
|
|
190
690
|
projectContext: roomProject ? {
|
|
191
691
|
...roomProject,
|
|
192
692
|
accessible: projectAccessible,
|
|
@@ -194,18 +694,26 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
194
694
|
} : undefined,
|
|
195
695
|
extraArgs: options.extraArgs,
|
|
196
696
|
localDaemonUrl: options.localDaemonUrl,
|
|
197
|
-
localDaemonToken: options.localDaemonToken,
|
|
697
|
+
localDaemonToken: runtimeLocalToken ?? options.localDaemonToken,
|
|
198
698
|
privateCliBinDir: options.privateCliBinDir,
|
|
199
699
|
palCliCommand: 'pal',
|
|
700
|
+
runtimeEnv: options.runtimeEnv,
|
|
200
701
|
runtimeSessionId: session.runtime_session_id,
|
|
201
702
|
roomParticipants,
|
|
202
|
-
|
|
703
|
+
roomAgentSubscriptions,
|
|
704
|
+
roomMode: roomSnapshot?.room.mode ?? 'standard',
|
|
705
|
+
enabledSkills,
|
|
706
|
+
deliveryContext: deliveryContext ?? undefined,
|
|
707
|
+
runtimeAttachments,
|
|
203
708
|
dryRun: options.dryRun,
|
|
204
709
|
signal: options.abortSignal,
|
|
205
710
|
onStart: async (pid) => {
|
|
206
711
|
console.log(`${logPrefix} run=${run.id} pid=${pid}`);
|
|
207
712
|
await client.updateRunPid(run.id, pid);
|
|
208
713
|
},
|
|
714
|
+
onActivity: async (event) => {
|
|
715
|
+
await client.recordRunActivity(run.id, event);
|
|
716
|
+
},
|
|
209
717
|
getAction: async () => {
|
|
210
718
|
if (options.isConnectionRevoked?.()) return 'kill';
|
|
211
719
|
const latest = await client.getRun(run.id);
|
|
@@ -219,6 +727,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
219
727
|
if (result.runtimeSessionId && result.runtimeSessionId !== session.runtime_session_id) {
|
|
220
728
|
console.log(`${logPrefix} session=${session.id} runtime_session_id=${result.runtimeSessionId}`);
|
|
221
729
|
await client.updateSessionRuntimeSessionId(session.id, { runtime_session_id: result.runtimeSessionId });
|
|
730
|
+
activeHandle.runtimeSessionId = result.runtimeSessionId;
|
|
222
731
|
}
|
|
223
732
|
|
|
224
733
|
console.log(`${logPrefix} run=${run.id} exitCode=${result.exitCode} stoppedByAction=${result.stoppedByAction ?? '-'} outputLen=${result.output.length}`);
|
|
@@ -241,6 +750,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
241
750
|
if (result.stoppedByAction === 'restart') {
|
|
242
751
|
console.log(`${logPrefix} run=${run.id} finishing with restart`);
|
|
243
752
|
await client.finishRun(run.id, { status: 'restarted', exit_code: result.exitCode, output: result.output });
|
|
753
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, 'active run restarted before completion');
|
|
244
754
|
return 'restart';
|
|
245
755
|
}
|
|
246
756
|
|
|
@@ -248,6 +758,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
248
758
|
console.log(`${logPrefix} run=${run.id} finishing with killed`);
|
|
249
759
|
await client.finishRun(run.id, { status: 'killed', exit_code: result.exitCode, output: result.output });
|
|
250
760
|
await client.ackDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id });
|
|
761
|
+
await ackSteeredDeliveries(client, activeHandle, options, run.id);
|
|
251
762
|
return 'done';
|
|
252
763
|
}
|
|
253
764
|
|
|
@@ -255,6 +766,17 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
255
766
|
console.log(`${logPrefix} run=${run.id} finishing with completed`);
|
|
256
767
|
await client.finishRun(run.id, { status: 'completed', exit_code: result.exitCode, output: result.output });
|
|
257
768
|
await client.ackDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id });
|
|
769
|
+
await ackCoveredPendingDeliveries(client, {
|
|
770
|
+
agent: options.agent,
|
|
771
|
+
chatId: message.chat_id,
|
|
772
|
+
triggerMessageId: message.id,
|
|
773
|
+
maxCoveredMessageId,
|
|
774
|
+
daemonId: options.daemonId,
|
|
775
|
+
connectionId: options.connectionId,
|
|
776
|
+
computerId: options.computerId,
|
|
777
|
+
runId: run.id,
|
|
778
|
+
});
|
|
779
|
+
await ackSteeredDeliveries(client, activeHandle, options, run.id);
|
|
258
780
|
return 'done';
|
|
259
781
|
}
|
|
260
782
|
|
|
@@ -262,6 +784,7 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
262
784
|
console.log(`${logPrefix} run=${run.id} finishing with failed: ${output.slice(0, 200)}`);
|
|
263
785
|
await client.finishRun(run.id, { status: 'failed', exit_code: result.exitCode, output });
|
|
264
786
|
await client.failDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id, error: output });
|
|
787
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, output);
|
|
265
788
|
await writeFailureMessage(client, message, options.agent, output);
|
|
266
789
|
return 'done';
|
|
267
790
|
} catch (error) {
|
|
@@ -273,8 +796,15 @@ async function executeRun(client: LockClient, delivery: MessageDelivery, message
|
|
|
273
796
|
}
|
|
274
797
|
await client.finishRun(run.id, { status: 'failed', output });
|
|
275
798
|
await client.failDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, claim_token: delivery.claim_token, run_id: run.id, error: output });
|
|
799
|
+
await failSteeredDeliveries(client, activeHandle, options, run.id, output);
|
|
276
800
|
await writeFailureMessage(client, message, options.agent, output);
|
|
277
801
|
return 'done';
|
|
802
|
+
} finally {
|
|
803
|
+
if (runtimeLocalToken) options.runtimeTokens?.revoke(runtimeLocalToken);
|
|
804
|
+
if (options.activeRuns?.get(key) === activeHandle) {
|
|
805
|
+
options.activeRuns.delete(key);
|
|
806
|
+
console.log(`${logPrefix} inactive run key=${key.replace('\0', ':')}`);
|
|
807
|
+
}
|
|
278
808
|
}
|
|
279
809
|
}
|
|
280
810
|
|
|
@@ -293,8 +823,13 @@ async function handleDelivery(client: LockClient, delivery: MessageDelivery, opt
|
|
|
293
823
|
computerId?: string | null;
|
|
294
824
|
connectionId?: string | null;
|
|
295
825
|
runtimeProvider: string;
|
|
826
|
+
runtimeModel?: string | null;
|
|
827
|
+
permissionProfile: AgentPermissionProfile;
|
|
828
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
296
829
|
isConnectionRevoked?: () => boolean;
|
|
297
830
|
abortSignal?: AbortSignal;
|
|
831
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
832
|
+
activeRuns?: ActiveRunMap;
|
|
298
833
|
}): Promise<void> {
|
|
299
834
|
console.log(`[daemon] handleDelivery delivery=${delivery.id} message=${delivery.message_id} attempts=${delivery.attempts}`);
|
|
300
835
|
const message = await client.getMessage(delivery.message_id);
|
|
@@ -322,7 +857,54 @@ async function discoverDeliveries(client: LockClient, state: DaemonState, agent:
|
|
|
322
857
|
}
|
|
323
858
|
}
|
|
324
859
|
|
|
325
|
-
async function
|
|
860
|
+
async function steerDeliveryToActiveRun(
|
|
861
|
+
client: LockClient,
|
|
862
|
+
delivery: MessageDelivery,
|
|
863
|
+
active: ActiveRunHandle,
|
|
864
|
+
options: {
|
|
865
|
+
daemonId: string;
|
|
866
|
+
connectionId?: string | null;
|
|
867
|
+
computerId?: string | null;
|
|
868
|
+
},
|
|
869
|
+
): Promise<void> {
|
|
870
|
+
let claimed: MessageDelivery | null = null;
|
|
871
|
+
try {
|
|
872
|
+
console.log(`[daemon] steerDelivery claim delivery=${delivery.id} activeRun=${active.runId}`);
|
|
873
|
+
claimed = await client.claimDelivery(delivery.id, {
|
|
874
|
+
daemon_id: options.daemonId,
|
|
875
|
+
connection_id: options.connectionId,
|
|
876
|
+
computer_id: options.computerId,
|
|
877
|
+
steer_run_id: active.runId,
|
|
878
|
+
});
|
|
879
|
+
if (!claimed.claim_token) throw new Error(`steered delivery ${delivery.id} was claimed without a claim token`);
|
|
880
|
+
const message = await client.getMessage(claimed.message_id);
|
|
881
|
+
await active.steer(message);
|
|
882
|
+
await client.markDeliveryProcessingCompleted(claimed.id, {
|
|
883
|
+
daemon_id: options.daemonId,
|
|
884
|
+
connection_id: options.connectionId,
|
|
885
|
+
claim_token: claimed.claim_token,
|
|
886
|
+
run_id: active.runId,
|
|
887
|
+
});
|
|
888
|
+
active.steeredDeliveries.push({ id: claimed.id, claimToken: claimed.claim_token });
|
|
889
|
+
console.log(`[daemon] steerDelivery accepted delivery=${delivery.id} activeRun=${active.runId}`);
|
|
890
|
+
} catch (error) {
|
|
891
|
+
const output = error instanceof Error ? error.message : String(error);
|
|
892
|
+
console.warn(`[daemon] steerDelivery failed delivery=${delivery.id}: ${output}`);
|
|
893
|
+
if (claimed?.claim_token) {
|
|
894
|
+
await client.failDelivery(claimed.id, {
|
|
895
|
+
daemon_id: options.daemonId,
|
|
896
|
+
connection_id: options.connectionId,
|
|
897
|
+
claim_token: claimed.claim_token,
|
|
898
|
+
run_id: active.runId,
|
|
899
|
+
error: output,
|
|
900
|
+
}).catch((failError) => {
|
|
901
|
+
console.warn(`[daemon] failed to mark steered delivery ${claimed?.id} failed: ${failError instanceof Error ? failError.message : String(failError)}`);
|
|
902
|
+
});
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
export async function processPendingDeliveries(client: LockClient, options: {
|
|
326
908
|
agent: string;
|
|
327
909
|
daemonId: string;
|
|
328
910
|
serverUrl: string;
|
|
@@ -337,37 +919,85 @@ async function processPendingDeliveries(client: LockClient, options: {
|
|
|
337
919
|
computerId?: string | null;
|
|
338
920
|
connectionId?: string | null;
|
|
339
921
|
runtimeProvider: string;
|
|
922
|
+
runtimeModel?: string | null;
|
|
923
|
+
permissionProfile: AgentPermissionProfile;
|
|
924
|
+
runtimeTokens?: RuntimeLocalTokenRegistry;
|
|
340
925
|
isConnectionRevoked?: () => boolean;
|
|
341
926
|
abortSignal?: AbortSignal;
|
|
342
|
-
|
|
927
|
+
runtimeEnv?: NodeJS.ProcessEnv;
|
|
928
|
+
activeRuns: ActiveRunMap;
|
|
929
|
+
}): Promise<Array<Promise<void>>> {
|
|
343
930
|
console.log(`[daemon] processPendingDeliveries agent=${options.agent} daemon=${options.daemonId}`);
|
|
344
|
-
const deliveries = await client.listDeliveries(options.agent, 'pending',
|
|
931
|
+
const deliveries = await client.listDeliveries(options.agent, 'pending', PENDING_DELIVERY_CANDIDATE_LIMIT, {
|
|
932
|
+
distinctChat: true,
|
|
933
|
+
excludeRunningComputerId: options.computerId,
|
|
934
|
+
connectionId: options.connectionId,
|
|
935
|
+
});
|
|
345
936
|
console.log(`[daemon] processPendingDeliveries found ${deliveries.length} pending deliveries`);
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
937
|
+
const started: Array<Promise<void>> = [];
|
|
938
|
+
for (const action of planPendingDeliveryActions(options.agent, deliveries, options.activeRuns)) {
|
|
939
|
+
if (action.kind === 'steer') {
|
|
940
|
+
await steerDeliveryToActiveRun(client, action.delivery, action.active, options);
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
if (action.kind === 'skip') {
|
|
944
|
+
const reason = action.reason === 'starting' ? 'run already starting' : 'active run does not support steer';
|
|
945
|
+
console.log(`[daemon] delivery=${action.delivery.id} remains pending; ${reason} key=${action.key.replace('\0', ':')}`);
|
|
946
|
+
continue;
|
|
354
947
|
}
|
|
948
|
+
const delivery = action.delivery;
|
|
949
|
+
const reservation: ActiveRunHandle = {
|
|
950
|
+
key: action.key,
|
|
951
|
+
agent: options.agent,
|
|
952
|
+
chatId: delivery.chat_id,
|
|
953
|
+
runId: `starting:${delivery.id}`,
|
|
954
|
+
sessionId: '',
|
|
955
|
+
runtimeSessionId: null,
|
|
956
|
+
supportsSteer: false,
|
|
957
|
+
steeredDeliveries: [],
|
|
958
|
+
steer: async () => {
|
|
959
|
+
throw new Error('run is still starting');
|
|
960
|
+
},
|
|
961
|
+
};
|
|
962
|
+
options.activeRuns.set(action.key, reservation);
|
|
963
|
+
const task = (async () => {
|
|
964
|
+
try {
|
|
965
|
+
console.log(`[daemon] claimDelivery delivery=${delivery.id}`);
|
|
966
|
+
const claimed = await client.claimDelivery(delivery.id, { daemon_id: options.daemonId, connection_id: options.connectionId, computer_id: options.computerId });
|
|
967
|
+
console.log(`[daemon] claimDelivery success delivery=${claimed.id} token=${claimed.claim_token}`);
|
|
968
|
+
await handleDelivery(client, claimed, options);
|
|
969
|
+
} catch (error) {
|
|
970
|
+
console.warn(`[daemon] claimDelivery failed delivery=${delivery.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
971
|
+
} finally {
|
|
972
|
+
if (options.activeRuns.get(action.key) === reservation) {
|
|
973
|
+
options.activeRuns.delete(action.key);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
})();
|
|
977
|
+
started.push(task);
|
|
355
978
|
}
|
|
979
|
+
return started;
|
|
356
980
|
}
|
|
357
981
|
|
|
358
982
|
async function buildManagedAgent(input: {
|
|
359
|
-
assignment: Pick<ComputerAgentAssignment, 'agent' | 'runtime' | 'cwd'>;
|
|
983
|
+
assignment: Pick<ComputerAgentAssignment, 'agent' | 'runtime' | 'model' | 'cwd'>;
|
|
360
984
|
state: DaemonState;
|
|
361
985
|
serverUrl: string;
|
|
362
986
|
defaultCwd: string;
|
|
363
987
|
}): Promise<ManagedAgent> {
|
|
364
988
|
const agentUuid = ensureAgentUuid(input.state, input.assignment.agent);
|
|
365
|
-
const
|
|
989
|
+
const resolved = input.assignment.runtime
|
|
990
|
+
? {
|
|
991
|
+
runtime: await resolveRuntimeDriver(input.assignment.runtime, agentUuid, input.assignment.model),
|
|
992
|
+
runtimeModel: input.assignment.model?.trim() || null,
|
|
993
|
+
}
|
|
994
|
+
: await resolveRuntime(input.assignment.agent, input.serverUrl, agentUuid);
|
|
366
995
|
const agentHome = agentHomePath(agentUuid);
|
|
367
|
-
ensureAgentHome(agentHome, input.assignment.agent, runtime.name);
|
|
996
|
+
ensureAgentHome(agentHome, input.assignment.agent, resolved.runtime.name);
|
|
368
997
|
return {
|
|
369
998
|
agent: input.assignment.agent,
|
|
370
|
-
runtimeProvider: runtime.name,
|
|
999
|
+
runtimeProvider: resolved.runtime.name,
|
|
1000
|
+
runtimeModel: resolved.runtimeModel,
|
|
371
1001
|
agentUuid,
|
|
372
1002
|
agentHome,
|
|
373
1003
|
projectCwd: input.assignment.cwd || input.defaultCwd,
|
|
@@ -378,7 +1008,7 @@ function daemonAgentPayload(managedAgents: Map<string, ManagedAgent>): Array<{ a
|
|
|
378
1008
|
return Array.from(managedAgents.values()).map((managed) => ({
|
|
379
1009
|
agent: managed.agent,
|
|
380
1010
|
cwd: managed.agentHome,
|
|
381
|
-
capabilities: { runtime: managed.runtimeProvider, agentHome: managed.agentHome, projectCwd: managed.projectCwd },
|
|
1011
|
+
capabilities: { runtime: managed.runtimeProvider, model: managed.runtimeModel, agentHome: managed.agentHome, projectCwd: managed.projectCwd },
|
|
382
1012
|
}));
|
|
383
1013
|
}
|
|
384
1014
|
|
|
@@ -393,8 +1023,9 @@ async function reconcileManagedAgents(input: {
|
|
|
393
1023
|
serverUrl: string;
|
|
394
1024
|
defaultCwd: string;
|
|
395
1025
|
localUrl?: string;
|
|
396
|
-
}): Promise<
|
|
1026
|
+
}): Promise<string[]> {
|
|
397
1027
|
const desiredAssignedAgents = new Set(input.assignments.map((assignment) => assignment.agent));
|
|
1028
|
+
const activatedAgents: string[] = [];
|
|
398
1029
|
|
|
399
1030
|
for (const assignment of input.assignments) {
|
|
400
1031
|
if (!assignment.runtime) {
|
|
@@ -403,10 +1034,11 @@ async function reconcileManagedAgents(input: {
|
|
|
403
1034
|
}
|
|
404
1035
|
const existing = input.managedAgents.get(assignment.agent);
|
|
405
1036
|
const projectCwd = assignment.cwd || input.defaultCwd;
|
|
406
|
-
if (existing && existing.runtimeProvider === assignment.runtime && existing.projectCwd === projectCwd) continue;
|
|
1037
|
+
if (existing && existing.runtimeProvider === assignment.runtime && existing.runtimeModel === assignment.model && existing.projectCwd === projectCwd) continue;
|
|
407
1038
|
const managed = await buildManagedAgent({ assignment, state: input.state, serverUrl: input.serverUrl, defaultCwd: input.defaultCwd });
|
|
408
1039
|
input.managedAgents.set(assignment.agent, managed);
|
|
409
|
-
|
|
1040
|
+
activatedAgents.push(managed.agent);
|
|
1041
|
+
console.log(`[daemon] agent active agent=${managed.agent} runtime=${managed.runtimeProvider} model=${managed.runtimeModel ?? '-'} projectCwd=${managed.projectCwd}`);
|
|
410
1042
|
}
|
|
411
1043
|
|
|
412
1044
|
for (const agent of Array.from(input.managedAgents.keys())) {
|
|
@@ -423,6 +1055,7 @@ async function reconcileManagedAgents(input: {
|
|
|
423
1055
|
server_url: input.serverUrl,
|
|
424
1056
|
agents: daemonAgentPayload(input.managedAgents),
|
|
425
1057
|
});
|
|
1058
|
+
return activatedAgents;
|
|
426
1059
|
}
|
|
427
1060
|
|
|
428
1061
|
async function main(): Promise<void> {
|
|
@@ -437,7 +1070,15 @@ async function main(): Promise<void> {
|
|
|
437
1070
|
return;
|
|
438
1071
|
}
|
|
439
1072
|
|
|
440
|
-
const
|
|
1073
|
+
const runtimeEnvReport = hydrateRuntimeEnv();
|
|
1074
|
+
logEnvReport(runtimeEnvReport);
|
|
1075
|
+
if (runtimeEnvReport.missing.length) {
|
|
1076
|
+
const message = `[daemon] runtime env warnings: missing=[${runtimeEnvReport.missing.join(', ')}], runtime auth may fail`;
|
|
1077
|
+
if (process.env.PAL_DAEMON_STRICT_ENV === '1') throw new Error(message);
|
|
1078
|
+
console.log(message);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const explicitAgent = flag(args.flags, 'agent');
|
|
441
1082
|
const serverUrl = flag(args.flags, 'server-url') ?? flag(args.flags, 'server') ?? defaultServerUrl();
|
|
442
1083
|
const cwd = flag(args.flags, 'cwd') ?? process.cwd();
|
|
443
1084
|
const interval = numberFlag(args.flags, 'interval', 1500)!;
|
|
@@ -451,6 +1092,7 @@ async function main(): Promise<void> {
|
|
|
451
1092
|
const statePath = flag(args.flags, 'state') ?? defaultStatePath(explicitAgent ?? 'computer');
|
|
452
1093
|
const once = boolFlag(args.flags, 'once');
|
|
453
1094
|
const dryRun = boolFlag(args.flags, 'dry-run');
|
|
1095
|
+
const verboseTicks = boolFlag(args.flags, 'verbose') || process.env.PAL_DAEMON_VERBOSE_TICKS === '1';
|
|
454
1096
|
const extraArgsRaw = flag(args.flags, 'neeko-args') ?? flag(args.flags, 'agent-args') ?? '';
|
|
455
1097
|
const extraArgs = extraArgsRaw ? extraArgsRaw.split(' ').filter(Boolean) : [];
|
|
456
1098
|
const privateCliBinDir = ensurePrivatePalCliBin();
|
|
@@ -459,16 +1101,19 @@ async function main(): Promise<void> {
|
|
|
459
1101
|
|
|
460
1102
|
const apiKey = flag(args.flags, 'api-key') ?? process.env.PAL_API_KEY;
|
|
461
1103
|
const computerId = flag(args.flags, 'computer-id') ?? process.env.PAL_COMPUTER_ID;
|
|
1104
|
+
const computerName = flag(args.flags, 'name') ?? flag(args.flags, 'computer-name') ?? process.env.PAL_COMPUTER_NAME;
|
|
462
1105
|
const computerSecret = flag(args.flags, 'computer-secret') ?? process.env.PAL_COMPUTER_SECRET;
|
|
463
1106
|
if (!apiKey?.trim() && !computerId?.trim()) throw new Error('--api-key, --computer-id, or PAL_COMPUTER_ID is required');
|
|
464
1107
|
if (!apiKey?.trim() && !computerSecret?.trim()) throw new Error('--api-key, --computer-secret, or PAL_COMPUTER_SECRET is required');
|
|
465
1108
|
|
|
466
1109
|
const explicitAgents = new Set<string>();
|
|
467
1110
|
const managedAgents = new Map<string, ManagedAgent>();
|
|
1111
|
+
const activeRuns: ActiveRunMap = new Map();
|
|
1112
|
+
const runtimeTokens = new RuntimeLocalTokenRegistry();
|
|
468
1113
|
if (explicitAgent) {
|
|
469
1114
|
explicitAgents.add(explicitAgent);
|
|
470
1115
|
const managed = await buildManagedAgent({
|
|
471
|
-
assignment: { agent: explicitAgent, runtime: null, cwd },
|
|
1116
|
+
assignment: { agent: explicitAgent, runtime: null, model: null, cwd },
|
|
472
1117
|
state,
|
|
473
1118
|
serverUrl,
|
|
474
1119
|
defaultCwd: cwd,
|
|
@@ -476,21 +1121,77 @@ async function main(): Promise<void> {
|
|
|
476
1121
|
managedAgents.set(explicitAgent, managed);
|
|
477
1122
|
}
|
|
478
1123
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
1124
|
+
type ConnectedComputer = Awaited<ReturnType<LockClient['connectComputer']>>;
|
|
1125
|
+
let connected!: ConnectedComputer;
|
|
1126
|
+
let daemonAuth!: { computer_id: string; connection_id: string; token: string };
|
|
1127
|
+
let client!: LockClient;
|
|
1128
|
+
let deliverySocket!: DeliverySocketHandle;
|
|
1129
|
+
let reconnecting = false;
|
|
1130
|
+
let onDeliveryFrame: (agent?: string) => void = () => {};
|
|
1131
|
+
|
|
1132
|
+
const connectAndInstall = async (reason: 'startup' | 'reconnect'): Promise<void> => {
|
|
1133
|
+
const previousConnectionId = reason === 'reconnect' ? connected.connection.id : null;
|
|
1134
|
+
const next = await bootstrapClient.connectComputer({
|
|
1135
|
+
computer_id: computerId,
|
|
1136
|
+
secret: computerSecret,
|
|
1137
|
+
api_key: apiKey,
|
|
1138
|
+
name: computerName,
|
|
1139
|
+
server_url: serverUrl,
|
|
1140
|
+
agents: daemonAgentPayload(managedAgents),
|
|
1141
|
+
});
|
|
1142
|
+
connected = next;
|
|
1143
|
+
state.computerId = next.computer.id;
|
|
1144
|
+
state.connectionId = next.connection.id;
|
|
1145
|
+
state.daemonId = next.connection.id;
|
|
1146
|
+
daemonAuth = { computer_id: next.computer.id, connection_id: next.connection.id, token: next.token };
|
|
1147
|
+
client = new LockClient(serverUrl, daemonAuth);
|
|
1148
|
+
deliverySocket?.stop();
|
|
1149
|
+
deliverySocket = startDeliveryWebSocket({
|
|
1150
|
+
serverUrl,
|
|
1151
|
+
computerId: next.computer.id,
|
|
1152
|
+
connectionId: next.connection.id,
|
|
1153
|
+
token: next.token,
|
|
1154
|
+
onDelivery: (agent) => onDeliveryFrame(agent),
|
|
1155
|
+
});
|
|
1156
|
+
writeState(statePath, state);
|
|
1157
|
+
if (reason === 'reconnect') {
|
|
1158
|
+
console.log(`[daemon] reconnected computer=${next.computer.id} connection=${next.connection.id}${previousConnectionId ? ` previous=${previousConnectionId}` : ''}`);
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
|
|
1162
|
+
let running = true;
|
|
1163
|
+
const runtimeAbortController = new AbortController();
|
|
1164
|
+
process.on('SIGINT', () => { running = false; runtimeAbortController.abort(); });
|
|
1165
|
+
process.on('SIGTERM', () => { running = false; runtimeAbortController.abort(); });
|
|
490
1166
|
|
|
491
|
-
const
|
|
492
|
-
|
|
493
|
-
|
|
1167
|
+
const connectAndInstallWithRetry = async (reason: 'startup' | 'reconnect'): Promise<boolean> => {
|
|
1168
|
+
let attempt = 0;
|
|
1169
|
+
while (running) {
|
|
1170
|
+
try {
|
|
1171
|
+
await connectAndInstall(reason);
|
|
1172
|
+
return true;
|
|
1173
|
+
} catch (error) {
|
|
1174
|
+
if (!shouldRetryDaemonConnectError(error)) throw error;
|
|
1175
|
+
attempt += 1;
|
|
1176
|
+
const delayMs = Math.min(10_000, 500 * 2 ** Math.min(attempt, 5));
|
|
1177
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1178
|
+
console.warn(`[daemon] ${reason} connection failed, will retry in ${delayMs}ms: ${message}`);
|
|
1179
|
+
await sleep(delayMs);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
return false;
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
if (!await connectAndInstallWithRetry('startup')) return;
|
|
1186
|
+
const localServer = startLocalApi({
|
|
1187
|
+
host: localHost,
|
|
1188
|
+
port: localPort,
|
|
1189
|
+
serverUrl,
|
|
1190
|
+
token: localToken,
|
|
1191
|
+
controlToken: () => connected.local_control_token,
|
|
1192
|
+
daemonAuth: () => daemonAuth,
|
|
1193
|
+
runtimeTokenLookup: (token) => runtimeTokens.lookup(token),
|
|
1194
|
+
});
|
|
494
1195
|
const localUrl = `http://${localServer.hostname}:${localServer.port}`;
|
|
495
1196
|
await reconcileManagedAgents({
|
|
496
1197
|
assignments: connected.agents ?? [],
|
|
@@ -508,33 +1209,153 @@ async function main(): Promise<void> {
|
|
|
508
1209
|
console.log(`[daemon] no agents currently assigned to computer ${connected.computer.id}; waiting for assignments`);
|
|
509
1210
|
}
|
|
510
1211
|
writeState(statePath, state);
|
|
511
|
-
let connectionRevoked = false;
|
|
512
1212
|
const heartbeatMs = numberFlag(args.flags, 'heartbeat-interval', 5000)!;
|
|
1213
|
+
const heartbeatWarningState = defaultRepeatedWarningState();
|
|
1214
|
+
const heartbeatWarningIntervalMs = 30_000;
|
|
1215
|
+
let heartbeatInFlight = false;
|
|
1216
|
+
|
|
1217
|
+
const warnHeartbeatRetry = (message: string) => {
|
|
1218
|
+
const decision = consumeRepeatedWarning(heartbeatWarningState, {
|
|
1219
|
+
key: `${connected.connection.id}:${message}`,
|
|
1220
|
+
nowMs: Date.now(),
|
|
1221
|
+
minIntervalMs: heartbeatWarningIntervalMs,
|
|
1222
|
+
});
|
|
1223
|
+
if (!decision.shouldLog) return;
|
|
1224
|
+
const suppressed = decision.suppressed > 0 ? ` suppressed_repeats=${decision.suppressed}` : '';
|
|
1225
|
+
console.warn(`[daemon] heartbeat failed, will retry connection=${connected.connection.id}: ${message}${suppressed}`);
|
|
1226
|
+
};
|
|
1227
|
+
const runHeartbeat = async () => {
|
|
1228
|
+
if (heartbeatInFlight) return null;
|
|
1229
|
+
heartbeatInFlight = true;
|
|
1230
|
+
try {
|
|
1231
|
+
return await client.heartbeatComputer(connected.computer.id);
|
|
1232
|
+
} finally {
|
|
1233
|
+
heartbeatInFlight = false;
|
|
1234
|
+
}
|
|
1235
|
+
};
|
|
513
1236
|
|
|
514
1237
|
console.log(`pal daemon computer=${connected.computer.id} connection=${connected.connection.id} agents=${Array.from(managedAgents.values()).map((managed) => `${managed.agent}:${managed.runtimeProvider}`).join(',') || 'none'} server=${serverUrl}`);
|
|
515
1238
|
console.log(`local api=http://${localServer.hostname}:${localServer.port} token=${localToken ? 'set' : 'missing'}`);
|
|
516
1239
|
console.log(`state=${statePath} lastSeenId=${state.lastSeenId}`);
|
|
517
1240
|
console.log(`private cli bin=${privateCliBinDir}`);
|
|
518
1241
|
|
|
519
|
-
let running = true;
|
|
520
|
-
const runtimeAbortController = new AbortController();
|
|
521
|
-
process.on('SIGINT', () => { running = false; runtimeAbortController.abort(); });
|
|
522
|
-
process.on('SIGTERM', () => { running = false; runtimeAbortController.abort(); });
|
|
523
1242
|
const heartbeatTimer = setInterval(() => {
|
|
524
|
-
void
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
1243
|
+
void runHeartbeat().catch((error) => {
|
|
1244
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1245
|
+
if (shouldStopDaemonForHeartbeatError(error)) {
|
|
1246
|
+
void reconnectConnection(message);
|
|
1247
|
+
} else {
|
|
1248
|
+
warnHeartbeatRetry(message);
|
|
1249
|
+
}
|
|
529
1250
|
});
|
|
530
1251
|
}, heartbeatMs);
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
1252
|
+
const processingAgents = new Set<string>();
|
|
1253
|
+
const queuedProcessingAgents = new Set<string>();
|
|
1254
|
+
const activeDeliveryTasks = new Set<Promise<void>>();
|
|
1255
|
+
const scheduleDeliveryProcessing = (agent?: string) => {
|
|
1256
|
+
for (const managed of managedAgents.values()) {
|
|
1257
|
+
if (agent && managed.agent !== agent) continue;
|
|
1258
|
+
if (processingAgents.has(managed.agent)) {
|
|
1259
|
+
queuedProcessingAgents.add(managed.agent);
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
processingAgents.add(managed.agent);
|
|
1263
|
+
const taskClient = client;
|
|
1264
|
+
const taskConnection = connected;
|
|
1265
|
+
const task = (async () => {
|
|
1266
|
+
try {
|
|
1267
|
+
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
1268
|
+
await discoverDeliveries(taskClient, state, managed.agent);
|
|
1269
|
+
writeState(statePath, state);
|
|
1270
|
+
}
|
|
1271
|
+
const started = await processPendingDeliveries(taskClient, {
|
|
1272
|
+
agent: managed.agent,
|
|
1273
|
+
daemonId: taskConnection.connection.id,
|
|
1274
|
+
serverUrl,
|
|
1275
|
+
agentHome: managed.agentHome,
|
|
1276
|
+
projectCwd: managed.projectCwd,
|
|
1277
|
+
extraArgs,
|
|
1278
|
+
dryRun,
|
|
1279
|
+
agentUuid: managed.agentUuid,
|
|
1280
|
+
computerId: taskConnection.computer.id,
|
|
1281
|
+
connectionId: taskConnection.connection.id,
|
|
1282
|
+
runtimeProvider: managed.runtimeProvider,
|
|
1283
|
+
runtimeModel: managed.runtimeModel,
|
|
1284
|
+
permissionProfile: await loadPermissionProfile(taskClient, managed.agent),
|
|
1285
|
+
runtimeTokens,
|
|
1286
|
+
isConnectionRevoked: () => isDeliveryConnectionSuperseded(connected.connection.id, taskConnection.connection.id),
|
|
1287
|
+
abortSignal: runtimeAbortController.signal,
|
|
1288
|
+
localDaemonUrl: `http://${localServer.hostname}:${localServer.port}`,
|
|
1289
|
+
localDaemonToken: localToken,
|
|
1290
|
+
privateCliBinDir,
|
|
1291
|
+
runtimeEnv: runtimeEnvReport.effectiveEnv,
|
|
1292
|
+
activeRuns,
|
|
1293
|
+
});
|
|
1294
|
+
for (const runTask of started) {
|
|
1295
|
+
activeDeliveryTasks.add(runTask);
|
|
1296
|
+
runTask.finally(() => activeDeliveryTasks.delete(runTask));
|
|
1297
|
+
}
|
|
1298
|
+
} catch (error) {
|
|
1299
|
+
console.warn(`[daemon] delivery processing failed for agent=${managed.agent}; will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
1300
|
+
} finally {
|
|
1301
|
+
processingAgents.delete(managed.agent);
|
|
1302
|
+
if (queuedProcessingAgents.delete(managed.agent)) {
|
|
1303
|
+
setTimeout(() => scheduleDeliveryProcessing(managed.agent), 0);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
})();
|
|
1307
|
+
activeDeliveryTasks.add(task);
|
|
1308
|
+
task.finally(() => activeDeliveryTasks.delete(task));
|
|
1309
|
+
}
|
|
1310
|
+
};
|
|
1311
|
+
const schedulePendingDeliveryProcessing = async () => {
|
|
1312
|
+
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
1313
|
+
scheduleDeliveryProcessing();
|
|
1314
|
+
return;
|
|
1315
|
+
}
|
|
534
1316
|
try {
|
|
535
|
-
const
|
|
1317
|
+
const pendingAgents = await client.listPendingDeliveryAgents();
|
|
1318
|
+
for (const { agent } of pendingAgents) scheduleDeliveryProcessing(agent);
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
console.warn(`[daemon] pending delivery agent probe failed; falling back to all agents: ${error instanceof Error ? error.message : String(error)}`);
|
|
1321
|
+
scheduleDeliveryProcessing();
|
|
1322
|
+
}
|
|
1323
|
+
};
|
|
1324
|
+
const reconcileFromHeartbeat = async (): Promise<void> => {
|
|
1325
|
+
const heartbeat = await runHeartbeat();
|
|
1326
|
+
if (!heartbeat) return;
|
|
1327
|
+
await reconcileManagedAgents({
|
|
1328
|
+
assignments: heartbeat.agents ?? [],
|
|
1329
|
+
explicitAgents,
|
|
1330
|
+
managedAgents,
|
|
1331
|
+
state,
|
|
1332
|
+
client,
|
|
1333
|
+
daemonId: connected.connection.id,
|
|
1334
|
+
computerId: connected.computer.id,
|
|
1335
|
+
serverUrl,
|
|
1336
|
+
defaultCwd: cwd,
|
|
1337
|
+
localUrl,
|
|
1338
|
+
});
|
|
1339
|
+
writeState(statePath, state);
|
|
1340
|
+
};
|
|
1341
|
+
const handleDeliveryWake = async (agent?: string): Promise<void> => {
|
|
1342
|
+
if (agent && !managedAgents.has(agent)) {
|
|
1343
|
+
await reconcileFromHeartbeat().catch((error) => {
|
|
1344
|
+
console.warn(`[daemon] delivery wake reconcile failed for agent=${agent}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1345
|
+
});
|
|
1346
|
+
}
|
|
1347
|
+
scheduleDeliveryProcessing(agent);
|
|
1348
|
+
};
|
|
1349
|
+
const deliveryWakeQueue = new DeliveryWakeQueue((agent) => { void handleDeliveryWake(agent); });
|
|
1350
|
+
onDeliveryFrame = (agent) => deliveryWakeQueue.enqueue(agent);
|
|
1351
|
+
const reconnectConnection = async (message: string): Promise<boolean> => {
|
|
1352
|
+
if (reconnecting) return false;
|
|
1353
|
+
reconnecting = true;
|
|
1354
|
+
console.warn(`[daemon] connection ${connected.connection.id} is no longer active; reconnecting: ${message}`);
|
|
1355
|
+
try {
|
|
1356
|
+
if (!await connectAndInstallWithRetry('reconnect')) return false;
|
|
536
1357
|
await reconcileManagedAgents({
|
|
537
|
-
assignments:
|
|
1358
|
+
assignments: connected.agents ?? [],
|
|
538
1359
|
explicitAgents,
|
|
539
1360
|
managedAgents,
|
|
540
1361
|
state,
|
|
@@ -545,55 +1366,72 @@ async function main(): Promise<void> {
|
|
|
545
1366
|
defaultCwd: cwd,
|
|
546
1367
|
localUrl,
|
|
547
1368
|
});
|
|
1369
|
+
await schedulePendingDeliveryProcessing();
|
|
1370
|
+
return true;
|
|
1371
|
+
} catch (error) {
|
|
1372
|
+
console.warn(`[daemon] reconnect failed, will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
1373
|
+
return false;
|
|
1374
|
+
} finally {
|
|
1375
|
+
reconnecting = false;
|
|
1376
|
+
}
|
|
1377
|
+
};
|
|
1378
|
+
|
|
1379
|
+
while (running) {
|
|
1380
|
+
if (verboseTicks) {
|
|
1381
|
+
console.log(`[daemon] tick reconcile interval=${interval}ms once=${once} ws=${deliverySocket.isOpen() ? 'open' : 'closed'}`);
|
|
1382
|
+
}
|
|
1383
|
+
try {
|
|
1384
|
+
const heartbeat = await runHeartbeat();
|
|
1385
|
+
if (heartbeat) {
|
|
1386
|
+
await reconcileManagedAgents({
|
|
1387
|
+
assignments: heartbeat.agents ?? [],
|
|
1388
|
+
explicitAgents,
|
|
1389
|
+
managedAgents,
|
|
1390
|
+
state,
|
|
1391
|
+
client,
|
|
1392
|
+
daemonId: connected.connection.id,
|
|
1393
|
+
computerId: connected.computer.id,
|
|
1394
|
+
serverUrl,
|
|
1395
|
+
defaultCwd: cwd,
|
|
1396
|
+
localUrl,
|
|
1397
|
+
});
|
|
1398
|
+
}
|
|
1399
|
+
await schedulePendingDeliveryProcessing();
|
|
548
1400
|
writeState(statePath, state);
|
|
549
1401
|
} catch (error) {
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
1402
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1403
|
+
if (shouldStopDaemonForHeartbeatError(error)) {
|
|
1404
|
+
await reconnectConnection(message);
|
|
1405
|
+
} else {
|
|
1406
|
+
warnHeartbeatRetry(message);
|
|
1407
|
+
}
|
|
555
1408
|
}
|
|
556
1409
|
if (managedAgents.size === 0) {
|
|
557
1410
|
console.log(`[daemon] no assigned agents; idle`);
|
|
558
1411
|
}
|
|
559
|
-
for (const managed of managedAgents.values()) {
|
|
560
|
-
if (process.env.PAL_DAEMON_DISCOVER_INBOX === '1') {
|
|
561
|
-
await discoverDeliveries(client, state, managed.agent);
|
|
562
|
-
writeState(statePath, state);
|
|
563
|
-
}
|
|
564
|
-
await processPendingDeliveries(client, {
|
|
565
|
-
agent: managed.agent,
|
|
566
|
-
daemonId: connected.connection.id,
|
|
567
|
-
serverUrl,
|
|
568
|
-
agentHome: managed.agentHome,
|
|
569
|
-
projectCwd: managed.projectCwd,
|
|
570
|
-
extraArgs,
|
|
571
|
-
dryRun,
|
|
572
|
-
agentUuid: managed.agentUuid,
|
|
573
|
-
computerId: connected.computer.id,
|
|
574
|
-
connectionId: connected.connection.id,
|
|
575
|
-
runtimeProvider: managed.runtimeProvider,
|
|
576
|
-
isConnectionRevoked: () => connectionRevoked,
|
|
577
|
-
abortSignal: runtimeAbortController.signal,
|
|
578
|
-
localDaemonUrl: `http://${localServer.hostname}:${localServer.port}`,
|
|
579
|
-
localDaemonToken: localToken,
|
|
580
|
-
privateCliBinDir,
|
|
581
|
-
});
|
|
582
|
-
}
|
|
583
1412
|
|
|
584
1413
|
if (once) {
|
|
1414
|
+
if (activeDeliveryTasks.size) {
|
|
1415
|
+
console.log(`[daemon] --once waiting for ${activeDeliveryTasks.size} active delivery task(s)`);
|
|
1416
|
+
await Promise.allSettled(Array.from(activeDeliveryTasks));
|
|
1417
|
+
}
|
|
585
1418
|
console.log(`[daemon] --once set, exiting loop`);
|
|
586
1419
|
break;
|
|
587
1420
|
}
|
|
588
|
-
console.log(`[daemon] sleep ${interval}ms`);
|
|
1421
|
+
if (verboseTicks) console.log(`[daemon] sleep ${interval}ms`);
|
|
589
1422
|
await sleep(interval);
|
|
590
1423
|
}
|
|
591
1424
|
|
|
592
1425
|
clearInterval(heartbeatTimer);
|
|
1426
|
+
deliveryWakeQueue.stop();
|
|
1427
|
+
deliverySocket.stop();
|
|
1428
|
+
if (activeDeliveryTasks.size) await Promise.allSettled(Array.from(activeDeliveryTasks));
|
|
593
1429
|
localServer.stop(true);
|
|
594
1430
|
}
|
|
595
1431
|
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
1432
|
+
if (import.meta.main) {
|
|
1433
|
+
main().catch((error) => {
|
|
1434
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
1435
|
+
process.exit(1);
|
|
1436
|
+
});
|
|
1437
|
+
}
|