crewly 1.4.30 → 1.4.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/backend/backend/src/constants.d.ts +1 -1
  2. package/dist/backend/backend/src/constants.d.ts.map +1 -1
  3. package/dist/backend/backend/src/constants.js +2 -2
  4. package/dist/backend/backend/src/constants.js.map +1 -1
  5. package/dist/backend/backend/src/controllers/cloud/cloud-google-auth.controller.js +6 -6
  6. package/dist/backend/backend/src/controllers/cloud/cloud-google-auth.controller.js.map +1 -1
  7. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.d.ts +23 -0
  8. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.d.ts.map +1 -0
  9. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.js +48 -0
  10. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.js.map +1 -0
  11. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.d.ts.map +1 -1
  12. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.js +6 -0
  13. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.js.map +1 -1
  14. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.d.ts +45 -0
  15. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.d.ts.map +1 -0
  16. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.js +72 -0
  17. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.js.map +1 -0
  18. package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
  19. package/dist/backend/backend/src/services/agent/agent-registration.service.js +25 -2
  20. package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
  21. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.d.ts +85 -3
  22. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.d.ts.map +1 -1
  23. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.js +309 -8
  24. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.js.map +1 -1
  25. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.d.ts +86 -0
  26. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.d.ts.map +1 -0
  27. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.js +147 -0
  28. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.js.map +1 -0
  29. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.d.ts +123 -17
  30. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.d.ts.map +1 -1
  31. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.js +584 -47
  32. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.js.map +1 -1
  33. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.d.ts +41 -3
  34. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.d.ts.map +1 -1
  35. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.js +116 -3
  36. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.js.map +1 -1
  37. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.d.ts +17 -0
  38. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.d.ts.map +1 -1
  39. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.js +44 -3
  40. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.js.map +1 -1
  41. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.d.ts.map +1 -1
  42. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.js +134 -39
  43. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.js.map +1 -1
  44. package/dist/backend/backend/src/services/agent/crewly-agent/types.d.ts +28 -2
  45. package/dist/backend/backend/src/services/agent/crewly-agent/types.d.ts.map +1 -1
  46. package/dist/backend/backend/src/services/agent/crewly-agent/types.js +12 -2
  47. package/dist/backend/backend/src/services/agent/crewly-agent/types.js.map +1 -1
  48. package/dist/backend/backend/src/services/cloud/cloud-client.service.d.ts +2 -2
  49. package/dist/backend/backend/src/services/cloud/cloud-client.service.js +2 -2
  50. package/dist/backend/backend/src/services/core/env.config.js +1 -1
  51. package/dist/backend/backend/src/services/session/session-backend.interface.d.ts +9 -0
  52. package/dist/backend/backend/src/services/session/session-backend.interface.d.ts.map +1 -1
  53. package/dist/backend/backend/src/services/session/session-backend.interface.js.map +1 -1
  54. package/dist/backend/backend/src/websocket/terminal.gateway.d.ts.map +1 -1
  55. package/dist/backend/backend/src/websocket/terminal.gateway.js +17 -5
  56. package/dist/backend/backend/src/websocket/terminal.gateway.js.map +1 -1
  57. package/dist/cli/backend/src/constants.d.ts +1 -1
  58. package/dist/cli/backend/src/constants.d.ts.map +1 -1
  59. package/dist/cli/backend/src/constants.js +2 -2
  60. package/dist/cli/backend/src/constants.js.map +1 -1
  61. package/dist/cli/cli/src/commands/service.d.ts +4 -0
  62. package/dist/cli/cli/src/commands/service.d.ts.map +1 -1
  63. package/dist/cli/cli/src/commands/service.js +248 -2
  64. package/dist/cli/cli/src/commands/service.js.map +1 -1
  65. package/dist/cli/cli/src/index.js +5 -1
  66. package/dist/cli/cli/src/index.js.map +1 -1
  67. package/frontend/dist/assets/{index-c10b16b7.js → index-411a5785.js} +338 -337
  68. package/frontend/dist/assets/{index-2b76b01d.css → index-63a5cc28.css} +1 -1
  69. package/frontend/dist/index.html +2 -2
  70. package/package.json +1 -1
@@ -1,17 +1,18 @@
1
1
  /**
2
2
  * Crewly Agent Runtime Service
3
3
  *
4
- * Concrete RuntimeAgentService subclass for the in-process Crewly Agent.
5
- * Unlike PTY-based runtimes (Claude Code, Gemini CLI), this runtime runs
6
- * entirely inside the Node.js process using the Vercel AI SDK.
7
- *
8
- * No tmux session, no shell commands — messages are routed directly to
9
- * the AgentRunnerService.handleMessage() method.
4
+ * Concrete RuntimeAgentService subclass for the Crewly Agent.
5
+ * Supports two execution modes:
6
+ * - **In-process** (default): AgentRunner runs in the main Node.js process
7
+ * - **Worker process**: AgentRunner runs in a child process via fork(),
8
+ * enabling hot-reload and crash isolation
10
9
  *
11
10
  * @module services/agent/crewly-agent/crewly-agent-runtime.service
12
11
  */
13
12
  import { promises as fs } from 'fs';
14
13
  import * as path from 'path';
14
+ import { fork } from 'child_process';
15
+ // fileURLToPath used for ESM __dirname equivalent — lazy-loaded to avoid CJS issues
15
16
  import { RuntimeAgentService } from '../runtime-agent.service.abstract.js';
16
17
  import { AgentRunnerService } from './agent-runner.service.js';
17
18
  import { RUNTIME_TYPES, CREWLY_CONSTANTS, ADDON_CONSTANTS } from '../../../constants.js';
@@ -24,29 +25,51 @@ import { PtyActivityTrackerService } from '../pty-activity-tracker.service.js';
24
25
  import { TokenUsageService } from '../../monitoring/token-usage.service.js';
25
26
  import { getSettingsService } from '../../settings/settings.service.js';
26
27
  /**
27
- * In-process Crewly Agent runtime powered by AI SDK generateText.
28
+ * Crewly Agent runtime with optional worker process isolation.
28
29
  *
29
- * Key differences from PTY-based runtimes:
30
- * - No tmux session needed runs in-process
31
- * - Messages routed via handleMessage() instead of PTY write
32
- * - System prompt loaded from config/roles/orchestrator/prompt.md
33
- * - Ready immediately after initialization (no CLI startup wait)
30
+ * Supports two modes:
31
+ * - **In-process** (default): AgentRunner runs directly in the main process
32
+ * - **Worker process** (`useWorkerProcess: true`): AgentRunner runs in a
33
+ * forked child process, enabling hot-reload and crash isolation
34
34
  *
35
35
  * @example
36
36
  * ```typescript
37
+ * // In-process mode (default)
37
38
  * const runtime = new CrewlyAgentRuntimeService(sessionHelper, projectRoot);
38
39
  * await runtime.initializeInProcess('crewly-orc');
39
- * const result = await runtime.handleMessage('Check all team statuses');
40
+ *
41
+ * // Worker process mode
42
+ * const runtime = new CrewlyAgentRuntimeService(sessionHelper, projectRoot);
43
+ * await runtime.initializeInProcess('crewly-orc', { useWorkerProcess: true });
44
+ *
45
+ * // Hot-reload: restart worker with fresh code, preserving session
46
+ * await runtime.hotReload();
40
47
  * ```
41
48
  */
42
49
  export class CrewlyAgentRuntimeService extends RuntimeAgentService {
43
50
  agentRunner = null;
44
51
  initialized = false;
45
52
  currentSessionName = null;
53
+ currentMemberId;
46
54
  currentModelString = 'unknown';
47
55
  logBuffer;
48
56
  rateLimiter;
49
57
  heartbeatTimer = null;
58
+ /** AbortController for the currently executing message — enables external abort */
59
+ messageAbortController = null;
60
+ // ===== Worker process fields =====
61
+ /** Whether this instance uses a worker process instead of in-process execution */
62
+ useWorkerProcess = false;
63
+ /** The forked worker child process */
64
+ workerProcess = null;
65
+ /** Stored config for hot-reload — needed to re-init the worker */
66
+ storedConfig = null;
67
+ /** Pending promise resolver for the current worker run */
68
+ workerRunResolve = null;
69
+ /** Pending promise rejector for the current worker run */
70
+ workerRunReject = null;
71
+ /** Whether the worker is currently processing a message */
72
+ workerProcessing = false;
50
73
  constructor(sessionHelper, projectRoot) {
51
74
  super(sessionHelper, projectRoot);
52
75
  this.logBuffer = InProcessLogBuffer.getInstance();
@@ -69,6 +92,9 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
69
92
  * @returns True if the agent runner is initialized
70
93
  */
71
94
  async detectRuntimeSpecific(_sessionName) {
95
+ if (this.useWorkerProcess) {
96
+ return this.initialized && this.workerProcess !== null && this.workerProcess.connected;
97
+ }
72
98
  return this.initialized && this.agentRunner !== null && this.agentRunner.isInitialized();
73
99
  }
74
100
  /**
@@ -98,17 +124,19 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
98
124
  }
99
125
  // ===== In-process lifecycle methods =====
100
126
  /**
101
- * Initialize the in-process agent runtime.
127
+ * Initialize the agent runtime.
102
128
  *
103
129
  * Loads the system prompt from config/roles/orchestrator/prompt.md,
104
- * creates the AgentRunnerService, and initializes the model.
130
+ * creates the AgentRunnerService (in-process or worker), and initializes the model.
105
131
  *
106
132
  * @param sessionName - Session name for this agent instance
107
- * @param config - Optional partial config overrides
133
+ * @param config - Optional partial config overrides. Set `useWorkerProcess: true` to run in a child process.
108
134
  * @param roleName - Role name for system prompt lookup (default: 'orchestrator')
109
135
  */
110
136
  async initializeInProcess(sessionName, config, roleName) {
111
137
  this.currentSessionName = sessionName;
138
+ this.currentMemberId = config?.memberId;
139
+ this.useWorkerProcess = config?.useWorkerProcess ?? false;
112
140
  // Build enhanced system prompt with skills and addon awareness
113
141
  const systemPrompt = await this.buildEnhancedSystemPrompt(roleName || 'orchestrator');
114
142
  // Build full config with defaults
@@ -122,15 +150,22 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
122
150
  compactionThreshold: config?.compactionThreshold || CREWLY_AGENT_DEFAULTS.COMPACTION_THRESHOLD,
123
151
  projectPath: config?.projectPath,
124
152
  };
125
- this.agentRunner = new AgentRunnerService(fullConfig);
126
- try {
127
- await this.agentRunner.initialize();
153
+ // Store config for hot-reload
154
+ this.storedConfig = fullConfig;
155
+ if (this.useWorkerProcess) {
156
+ await this.initializeWorker(fullConfig);
128
157
  }
129
- catch (error) {
130
- // Clean up on initialization failure to prevent partial state
131
- this.agentRunner = null;
132
- this.currentSessionName = null;
133
- throw error;
158
+ else {
159
+ this.agentRunner = new AgentRunnerService(fullConfig);
160
+ try {
161
+ await this.agentRunner.initialize();
162
+ }
163
+ catch (error) {
164
+ // Clean up on initialization failure to prevent partial state
165
+ this.agentRunner = null;
166
+ this.currentSessionName = null;
167
+ throw error;
168
+ }
134
169
  }
135
170
  this.initialized = true;
136
171
  this.currentModelString = `${fullConfig.model.provider}/${fullConfig.model.modelId}`;
@@ -138,9 +173,11 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
138
173
  this.startHeartbeat(sessionName);
139
174
  // Register in-process session for frontend terminal visibility
140
175
  this.logBuffer.registerSession(sessionName);
141
- this.logBuffer.append(sessionName, 'info', `Crewly Agent initialized (${this.currentModelString})`);
176
+ const mode = this.useWorkerProcess ? 'worker' : 'in-process';
177
+ this.logBuffer.append(sessionName, 'info', `Crewly Agent initialized [${mode}] (${this.currentModelString})`);
142
178
  this.logger.info('Crewly Agent runtime initialized', {
143
179
  sessionName,
180
+ mode,
144
181
  model: `${fullConfig.model.provider}/${fullConfig.model.modelId}`,
145
182
  maxSteps: fullConfig.maxSteps,
146
183
  });
@@ -157,9 +194,15 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
157
194
  * @throws Error if the runtime is not initialized
158
195
  */
159
196
  async handleMessage(message, metadata) {
160
- if (!this.agentRunner || !this.initialized) {
197
+ if (!this.initialized) {
161
198
  throw new Error('Crewly Agent runtime not initialized. Call initializeInProcess() first.');
162
199
  }
200
+ if (!this.useWorkerProcess && !this.agentRunner) {
201
+ throw new Error('Crewly Agent runtime not initialized. Call initializeInProcess() first.');
202
+ }
203
+ if (this.useWorkerProcess && (!this.workerProcess || !this.workerProcess.connected)) {
204
+ throw new Error('Worker process not available. Call initializeInProcess() or hotReload() first.');
205
+ }
163
206
  const session = this.currentSessionName;
164
207
  // Extract conversationId from [CHAT:xxx] or [GCHAT:xxx ...] prefix if present
165
208
  let conversationId;
@@ -174,17 +217,25 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
174
217
  });
175
218
  }
176
219
  const queueLen = this.rateLimiter.getQueueLength();
177
- this.logBuffer.append(session, 'info', `← Message received (${cleanMessage.length} chars${conversationId ? `, conv:${conversationId}` : ''}${queueLen > 0 ? `, queue:${queueLen}` : ''})`);
178
- this.logger.debug('Handling message via rate limiter', {
179
- sessionName: session,
180
- messageLength: cleanMessage.length,
181
- historyLength: this.agentRunner.getHistoryLength(),
182
- conversationId,
183
- queueLength: queueLen,
184
- requestsInWindow: this.rateLimiter.getRequestCountInWindow(),
185
- });
220
+ const msgPreview = cleanMessage.length <= 120
221
+ ? `"${cleanMessage}"`
222
+ : `"${cleanMessage.substring(0, 50)}...${cleanMessage.substring(cleanMessage.length - 50)}"`;
223
+ this.logBuffer.append(session, 'info', `← Message received (${cleanMessage.length} chars${conversationId ? `, conv:${conversationId}` : ''}${queueLen > 0 ? `, queue:${queueLen}` : ''}): ${msgPreview}`);
224
+ if (!this.useWorkerProcess) {
225
+ this.logger.debug('Handling message via rate limiter', {
226
+ sessionName: session,
227
+ messageLength: cleanMessage.length,
228
+ historyLength: this.agentRunner.getHistoryLength(),
229
+ conversationId,
230
+ queueLength: queueLen,
231
+ requestsInWindow: this.rateLimiter.getRequestCountInWindow(),
232
+ });
233
+ }
186
234
  // Route through rate limiter for throttling, coalescing, and 429 retry
187
235
  const result = await this.rateLimiter.enqueue(cleanMessage, metadata, async (msg, meta) => {
236
+ if (this.useWorkerProcess) {
237
+ return this.executeMessageViaWorker(session, msg, conversationId, meta);
238
+ }
188
239
  return this.executeMessage(session, msg, conversationId, meta);
189
240
  });
190
241
  return result;
@@ -202,26 +253,137 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
202
253
  * @returns Agent run result
203
254
  */
204
255
  async executeMessage(session, cleanMessage, conversationId, metadata) {
205
- // Soft warning timer — logs if processing exceeds 5 min but does NOT kill it.
206
- // Individual tool calls (bash_exec) have their own per-step timeouts (30s default).
207
- const SOFT_WARNING_MS = 5 * 60 * 1000;
256
+ const SOFT_WARNING_MS = CREWLY_AGENT_DEFAULTS.MESSAGE_SOFT_WARNING_MS;
257
+ // Execution tracking for diagnostics
258
+ const executionTracker = {
259
+ phase: 'queued',
260
+ currentTool: null,
261
+ toolCallsCompleted: [],
262
+ startedAt: new Date(),
263
+ lastActivityAt: new Date(),
264
+ messagePreview: cleanMessage.length <= 100
265
+ ? cleanMessage
266
+ : `${cleanMessage.substring(0, 50)}...${cleanMessage.substring(cleanMessage.length - 50)}`,
267
+ };
268
+ // Soft warning timer — logs if processing exceeds threshold but does NOT kill it.
208
269
  const warningTimer = setTimeout(() => {
209
- this.logger.warn('Message processing exceeding 5 minutes (still running)', {
270
+ executionTracker.lastActivityAt = new Date();
271
+ this.logger.warn(`Message processing exceeding ${SOFT_WARNING_MS / 1000}s (still running)`, {
210
272
  sessionName: session,
273
+ phase: executionTracker.phase,
274
+ toolCallsCompleted: executionTracker.toolCallsCompleted.length,
211
275
  messagePreview: cleanMessage.substring(0, 100),
212
276
  });
213
277
  }, SOFT_WARNING_MS);
278
+ // AbortController for external abort (abortCurrentRun) and repetition detection
279
+ const abortController = new AbortController();
280
+ this.messageAbortController = abortController;
281
+ // Text chunk buffer — collects streaming text and flushes on step boundaries
282
+ let textChunkBuffer = '';
283
+ // Repetition/hallucination detection — tracks recent chunks to detect loops
284
+ const recentChunks = [];
285
+ const REPETITION_WINDOW = 20; // number of recent chunks to track
286
+ const REPETITION_THRESHOLD = 5; // consecutive repeated patterns to trigger abort
287
+ let repetitionDetected = false;
288
+ // Build streaming callbacks that write to InProcessLogBuffer in real-time
289
+ const streamingCallbacks = {
290
+ onTextChunk: (chunk) => {
291
+ if (chunk.length > 0) {
292
+ executionTracker.lastActivityAt = new Date();
293
+ executionTracker.phase = 'model-thinking';
294
+ textChunkBuffer += chunk;
295
+ // Repetition detection: track recent chunks and check for loops
296
+ const trimmed = chunk.trim();
297
+ if (trimmed.length > 0) {
298
+ recentChunks.push(trimmed);
299
+ if (recentChunks.length > REPETITION_WINDOW) {
300
+ recentChunks.shift();
301
+ }
302
+ // Check if the last REPETITION_THRESHOLD chunks are identical
303
+ if (recentChunks.length >= REPETITION_THRESHOLD) {
304
+ const tail = recentChunks.slice(-REPETITION_THRESHOLD);
305
+ const allSame = tail.every(c => c === tail[0]);
306
+ if (allSame && tail[0].length >= 3) {
307
+ repetitionDetected = true;
308
+ this.logBuffer.append(session, 'warn', `⚠️ Repetition loop detected: "${tail[0].substring(0, 80)}" repeated ${REPETITION_THRESHOLD}x — aborting generation`);
309
+ this.logger.warn('Repetition/hallucination loop detected, aborting', {
310
+ sessionName: session,
311
+ repeatedChunk: tail[0].substring(0, 100),
312
+ count: REPETITION_THRESHOLD,
313
+ });
314
+ abortController.abort();
315
+ }
316
+ }
317
+ }
318
+ }
319
+ },
320
+ onToolCallStart: (toolName, _args) => {
321
+ executionTracker.phase = 'tool-calling';
322
+ executionTracker.currentTool = toolName;
323
+ executionTracker.lastActivityAt = new Date();
324
+ },
325
+ onToolCallFinish: (toolName, args, result, _durationMs) => {
326
+ executionTracker.toolCallsCompleted.push(toolName);
327
+ executionTracker.currentTool = null;
328
+ executionTracker.lastActivityAt = new Date();
329
+ const argsPreview = JSON.stringify(args).substring(0, 120);
330
+ this.logBuffer.append(session, 'info', `🔧 ${toolName}(${argsPreview})`);
331
+ // For bash_exec, show the command as an extra log line for readability
332
+ if (toolName === 'bash_exec' && args.command) {
333
+ const cmdPreview = String(args.command).substring(0, 200);
334
+ this.logBuffer.append(session, 'info', ` $ ${cmdPreview}`);
335
+ }
336
+ const resultPreview = result ? JSON.stringify(result).substring(0, 200) : 'void';
337
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
338
+ },
339
+ onStepFinish: (stepIndex, hasToolCalls) => {
340
+ executionTracker.lastActivityAt = new Date();
341
+ // Flush buffered text at each step boundary
342
+ if (textChunkBuffer.trim().length > 0) {
343
+ // Truncate very long text to keep logs readable
344
+ const text = textChunkBuffer.trim();
345
+ const preview = text.length > 500 ? text.substring(0, 500) + '...' : text;
346
+ this.logBuffer.append(session, 'info', `💬 ${preview}`);
347
+ textChunkBuffer = '';
348
+ }
349
+ if (!hasToolCalls) {
350
+ executionTracker.phase = 'model-thinking';
351
+ }
352
+ },
353
+ };
214
354
  try {
215
- const result = await this.agentRunner.run(cleanMessage, conversationId, metadata);
355
+ executionTracker.phase = 'model-thinking';
356
+ const result = await this.agentRunner.run(cleanMessage, conversationId, metadata, {
357
+ abortSignal: abortController.signal,
358
+ streaming: streamingCallbacks,
359
+ });
216
360
  clearTimeout(warningTimer);
217
- // Log tool calls to buffer for frontend visibility
218
- for (const tc of result.toolCalls) {
219
- const argsPreview = JSON.stringify(tc.args).substring(0, 120);
220
- this.logBuffer.append(session, 'info', `🔧 ${tc.toolName}(${argsPreview})`);
221
- const resultPreview = tc.result ? JSON.stringify(tc.result).substring(0, 200) : 'void';
222
- this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
361
+ this.messageAbortController = null;
362
+ // If we got here after a repetition-triggered abort, treat as error
363
+ if (repetitionDetected) {
364
+ throw new Error('Generation aborted: repetition/hallucination loop detected. '
365
+ + `Repeated pattern: "${recentChunks[recentChunks.length - 1]?.substring(0, 80)}"`);
366
+ }
367
+ // Flush any remaining buffered text after the run completes
368
+ if (textChunkBuffer.trim().length > 0) {
369
+ const text = textChunkBuffer.trim();
370
+ const preview = text.length > 500 ? text.substring(0, 500) + '...' : text;
371
+ this.logBuffer.append(session, 'info', `💬 ${preview}`);
372
+ textChunkBuffer = '';
373
+ }
374
+ // Tool calls already logged via streaming callbacks (onToolCallStart/Finish).
375
+ // Only log tool calls retroactively if generateText path was used (test mock).
376
+ if (this.agentRunner._generateTextFn) {
377
+ for (const tc of result.toolCalls) {
378
+ executionTracker.toolCallsCompleted.push(tc.toolName);
379
+ const argsPreview = JSON.stringify(tc.args).substring(0, 120);
380
+ this.logBuffer.append(session, 'info', `🔧 ${tc.toolName}(${argsPreview})`);
381
+ const resultPreview = tc.result ? JSON.stringify(tc.result).substring(0, 200) : 'void';
382
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
383
+ }
223
384
  }
224
385
  // Log response summary
386
+ executionTracker.phase = 'complete';
225
387
  const textPreview = result.text ? result.text.substring(0, 150) : '(no text)';
226
388
  this.logBuffer.append(session, 'info', `→ Response (${result.steps} steps, ${result.toolCalls.length} tools): ${textPreview}`);
227
389
  this.logBuffer.append(session, 'debug', ` Tokens: ${result.usage.input}in/${result.usage.output}out`);
@@ -240,6 +402,14 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
240
402
  }
241
403
  catch (error) {
242
404
  clearTimeout(warningTimer);
405
+ this.messageAbortController = null;
406
+ // If this was a repetition-triggered abort, wrap with a clear error
407
+ if (repetitionDetected) {
408
+ const repErr = new Error('Generation aborted: repetition/hallucination loop detected. '
409
+ + `Repeated pattern: "${recentChunks[recentChunks.length - 1]?.substring(0, 80)}"`);
410
+ this.logBuffer.append(session, 'error', `Agent error: ${repErr.message}`);
411
+ throw repErr;
412
+ }
243
413
  const errMsg = error instanceof Error ? error.message : String(error);
244
414
  this.logBuffer.append(session, 'error', `Agent error: ${errMsg}`);
245
415
  throw error;
@@ -263,8 +433,54 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
263
433
  * @returns True if initializeInProcess() has been called successfully
264
434
  */
265
435
  isReady() {
436
+ if (this.useWorkerProcess) {
437
+ return this.initialized && this.workerProcess !== null && this.workerProcess.connected;
438
+ }
266
439
  return this.initialized && this.agentRunner !== null && this.agentRunner.isInitialized();
267
440
  }
441
+ /**
442
+ * Abort the currently executing message processing.
443
+ *
444
+ * Cancels the active model call, terminates running tool processes,
445
+ * and returns partial results where possible. Safe to call at any time —
446
+ * returns false if no run is in progress.
447
+ *
448
+ * @returns True if an active run was aborted, false if nothing was running
449
+ */
450
+ abortCurrentRun() {
451
+ const session = this.currentSessionName;
452
+ if (this.useWorkerProcess) {
453
+ if (!this.workerProcessing || !this.workerProcess) {
454
+ return false;
455
+ }
456
+ this.sendToWorker({ type: 'abort' });
457
+ if (this.workerRunReject) {
458
+ this.workerRunReject(new Error('Run aborted by user'));
459
+ this.workerRunResolve = null;
460
+ this.workerRunReject = null;
461
+ }
462
+ this.workerProcessing = false;
463
+ if (session) {
464
+ this.logBuffer.append(session, 'warn', '⚠️ Run aborted by user');
465
+ }
466
+ this.logger.info('Agent run aborted (worker)', { sessionName: session });
467
+ return true;
468
+ }
469
+ if (!this.messageAbortController) {
470
+ return false;
471
+ }
472
+ this.messageAbortController.abort();
473
+ this.messageAbortController = null;
474
+ // Also tell the runner to abort (for cases where the runner has its own abort)
475
+ if (this.agentRunner) {
476
+ this.agentRunner.abortCurrentRun();
477
+ }
478
+ if (session) {
479
+ this.logBuffer.append(session, 'warn', '⚠️ Run aborted by user');
480
+ }
481
+ this.logger.info('Agent run aborted', { sessionName: session });
482
+ return true;
483
+ }
268
484
  /**
269
485
  * Get the current agent runner instance (for inspection/testing).
270
486
  *
@@ -288,11 +504,16 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
288
504
  shutdown() {
289
505
  this.logger.info('Shutting down Crewly Agent runtime', {
290
506
  sessionName: this.currentSessionName,
507
+ mode: this.useWorkerProcess ? 'worker' : 'in-process',
291
508
  });
292
509
  // Mark as not initialized first to reject new messages immediately
293
510
  this.initialized = false;
294
511
  // Stop heartbeat timer
295
512
  this.stopHeartbeat();
513
+ // Shut down worker process if running
514
+ if (this.workerProcess) {
515
+ this.terminateWorker();
516
+ }
296
517
  if (this.currentSessionName) {
297
518
  this.logBuffer.append(this.currentSessionName, 'info', 'Crewly Agent shutting down');
298
519
  this.logBuffer.removeSession(this.currentSessionName);
@@ -300,6 +521,321 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
300
521
  this.rateLimiter.reset();
301
522
  this.agentRunner = null;
302
523
  this.currentSessionName = null;
524
+ this.storedConfig = null;
525
+ }
526
+ // ===== Worker process methods =====
527
+ /**
528
+ * Hot-reload the worker process.
529
+ *
530
+ * Kills the existing worker and spawns a fresh one with the stored config.
531
+ * This allows updating agent code without restarting the main backend.
532
+ * Conversation state is reset — use this when deploying new agent logic.
533
+ *
534
+ * @throws Error if not in worker mode or config is missing
535
+ */
536
+ async hotReload() {
537
+ if (!this.useWorkerProcess) {
538
+ throw new Error('hotReload() is only available in worker process mode');
539
+ }
540
+ if (!this.storedConfig) {
541
+ throw new Error('No stored config for hot-reload. Was initializeInProcess() called?');
542
+ }
543
+ const session = this.currentSessionName;
544
+ this.logBuffer.append(session, 'info', 'Hot-reloading worker process...');
545
+ this.logger.info('Hot-reloading worker process', { sessionName: session });
546
+ // Terminate old worker
547
+ this.terminateWorker();
548
+ // Spawn new worker with same config
549
+ await this.initializeWorker(this.storedConfig);
550
+ this.logBuffer.append(session, 'info', 'Worker hot-reload complete');
551
+ this.logger.info('Worker hot-reload complete', { sessionName: session });
552
+ }
553
+ /**
554
+ * Check if the runtime is using a worker process.
555
+ *
556
+ * @returns True if running in worker process mode
557
+ */
558
+ isWorkerMode() {
559
+ return this.useWorkerProcess;
560
+ }
561
+ /**
562
+ * Get the worker process PID (for monitoring/debugging).
563
+ *
564
+ * @returns Worker PID, or null if not in worker mode or worker is not running
565
+ */
566
+ getWorkerPid() {
567
+ return this.workerProcess?.pid ?? null;
568
+ }
569
+ /**
570
+ * Initialize a worker child process via fork().
571
+ *
572
+ * Forks the agent-worker.ts entry point and sends the init message
573
+ * with the agent config. Waits for the 'ready' response before resolving.
574
+ *
575
+ * @param config - Full agent config to send to the worker
576
+ * @throws Error if worker fails to initialize within timeout
577
+ */
578
+ async initializeWorker(config) {
579
+ return new Promise((resolve, reject) => {
580
+ const workerPath = this.getWorkerEntryPath();
581
+ this.workerProcess = fork(workerPath, [], {
582
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
583
+ env: { ...process.env },
584
+ });
585
+ const initTimeout = setTimeout(() => {
586
+ this.terminateWorker();
587
+ reject(new Error('Worker initialization timed out (30s)'));
588
+ }, 30_000);
589
+ let initResolved = false;
590
+ this.workerProcess.on('message', (msg) => {
591
+ // Handle init response
592
+ if (!initResolved && msg.type === 'ready') {
593
+ initResolved = true;
594
+ clearTimeout(initTimeout);
595
+ resolve();
596
+ return;
597
+ }
598
+ if (!initResolved && msg.type === 'error' && msg.code === 'INIT_FAILED') {
599
+ initResolved = true;
600
+ clearTimeout(initTimeout);
601
+ reject(new Error(msg.error));
602
+ return;
603
+ }
604
+ // Handle runtime messages
605
+ this.handleWorkerMessage(msg);
606
+ });
607
+ this.workerProcess.on('exit', (code, signal) => {
608
+ this.logger.warn('Worker process exited', {
609
+ sessionName: this.currentSessionName,
610
+ code,
611
+ signal,
612
+ });
613
+ if (!initResolved) {
614
+ initResolved = true;
615
+ clearTimeout(initTimeout);
616
+ reject(new Error(`Worker exited during init (code=${code}, signal=${signal})`));
617
+ }
618
+ // Reject any pending run
619
+ if (this.workerRunReject) {
620
+ this.workerRunReject(new Error(`Worker process exited unexpectedly (code=${code}, signal=${signal})`));
621
+ this.workerRunResolve = null;
622
+ this.workerRunReject = null;
623
+ this.workerProcessing = false;
624
+ }
625
+ this.workerProcess = null;
626
+ if (this.currentSessionName) {
627
+ this.logBuffer.append(this.currentSessionName, 'warn', `Worker process exited (code=${code}, signal=${signal})`);
628
+ }
629
+ });
630
+ this.workerProcess.on('error', (err) => {
631
+ this.logger.error('Worker process error', {
632
+ sessionName: this.currentSessionName,
633
+ error: err.message,
634
+ });
635
+ if (!initResolved) {
636
+ initResolved = true;
637
+ clearTimeout(initTimeout);
638
+ reject(err);
639
+ }
640
+ });
641
+ // Capture worker stdout/stderr for debugging
642
+ this.workerProcess.stdout?.on('data', (data) => {
643
+ const text = data.toString().trim();
644
+ if (text && this.currentSessionName) {
645
+ this.logBuffer.append(this.currentSessionName, 'debug', `[worker stdout] ${text}`);
646
+ }
647
+ });
648
+ this.workerProcess.stderr?.on('data', (data) => {
649
+ const text = data.toString().trim();
650
+ if (text && this.currentSessionName) {
651
+ this.logBuffer.append(this.currentSessionName, 'warn', `[worker stderr] ${text}`);
652
+ }
653
+ });
654
+ // Send init message with config
655
+ this.sendToWorker({ type: 'init', config });
656
+ });
657
+ }
658
+ /**
659
+ * Execute a message via the worker process using IPC.
660
+ *
661
+ * Sends a 'run' message to the worker and waits for the 'result' or 'error'
662
+ * response. Streaming events are forwarded to the InProcessLogBuffer in real-time.
663
+ *
664
+ * @param session - Session name
665
+ * @param cleanMessage - Message content (prefix already stripped)
666
+ * @param conversationId - Optional conversation ID
667
+ * @param _metadata - Optional metadata (passed to worker)
668
+ * @returns Agent run result from the worker
669
+ */
670
+ executeMessageViaWorker(session, cleanMessage, conversationId, _metadata) {
671
+ return new Promise((resolve, reject) => {
672
+ if (!this.workerProcess || !this.workerProcess.connected) {
673
+ reject(new Error('Worker process not available'));
674
+ return;
675
+ }
676
+ this.workerProcessing = true;
677
+ this.workerRunResolve = (result) => {
678
+ this.workerProcessing = false;
679
+ this.workerRunResolve = null;
680
+ this.workerRunReject = null;
681
+ // Log response summary
682
+ const textPreview = result.text ? result.text.substring(0, 150) : '(no text)';
683
+ this.logBuffer.append(session, 'info', `→ Response (${result.steps} steps, ${result.toolCalls.length} tools): ${textPreview}`);
684
+ this.logBuffer.append(session, 'debug', ` Tokens: ${result.usage.input}in/${result.usage.output}out`);
685
+ this.logger.info('Message processed (worker)', {
686
+ sessionName: session,
687
+ steps: result.steps,
688
+ toolCalls: result.toolCalls.length,
689
+ usage: result.usage,
690
+ finishReason: result.finishReason,
691
+ });
692
+ // Record token usage
693
+ this.recordTokenUsageIfEnabled(session, result).catch(() => { });
694
+ resolve(result);
695
+ };
696
+ this.workerRunReject = (error) => {
697
+ this.workerProcessing = false;
698
+ this.workerRunResolve = null;
699
+ this.workerRunReject = null;
700
+ this.logBuffer.append(session, 'error', `Agent error (worker): ${error.message}`);
701
+ reject(error);
702
+ };
703
+ this.sendToWorker({
704
+ type: 'run',
705
+ message: cleanMessage,
706
+ conversationId,
707
+ metadata: _metadata,
708
+ });
709
+ });
710
+ }
711
+ /**
712
+ * Handle messages received from the worker process.
713
+ *
714
+ * Routes streaming events to the InProcessLogBuffer and resolves/rejects
715
+ * pending run promises on result/error messages.
716
+ *
717
+ * @param msg - Worker message received via IPC
718
+ */
719
+ handleWorkerMessage(msg) {
720
+ const session = this.currentSessionName;
721
+ switch (msg.type) {
722
+ case 'result':
723
+ if (this.workerRunResolve) {
724
+ this.workerRunResolve(msg.data);
725
+ }
726
+ break;
727
+ case 'error':
728
+ if (this.workerRunReject) {
729
+ this.workerRunReject(new Error(msg.error));
730
+ }
731
+ else if (session) {
732
+ // Error outside of a run (e.g. crash notification)
733
+ this.logBuffer.append(session, 'error', `Worker error: ${msg.error}`);
734
+ }
735
+ break;
736
+ case 'log':
737
+ if (session) {
738
+ this.logBuffer.append(session, msg.level, `[worker] ${msg.message}`);
739
+ }
740
+ break;
741
+ case 'stream':
742
+ if (!session)
743
+ break;
744
+ switch (msg.event) {
745
+ case 'text':
746
+ // Text chunks are buffered and flushed at step boundaries
747
+ break;
748
+ case 'toolStart':
749
+ // No-op: logged on finish
750
+ break;
751
+ case 'toolFinish': {
752
+ const { toolName, args, result } = msg.data;
753
+ const argsPreview = JSON.stringify(args).substring(0, 120);
754
+ this.logBuffer.append(session, 'info', `🔧 ${toolName}(${argsPreview})`);
755
+ if (toolName === 'bash_exec' && args.command) {
756
+ const cmdPreview = String(args.command).substring(0, 200);
757
+ this.logBuffer.append(session, 'info', ` $ ${cmdPreview}`);
758
+ }
759
+ const resultPreview = result ? JSON.stringify(result).substring(0, 200) : 'void';
760
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
761
+ break;
762
+ }
763
+ case 'stepFinish':
764
+ // Step boundaries are tracked by the worker
765
+ break;
766
+ }
767
+ break;
768
+ case 'state':
769
+ // State query responses (used internally)
770
+ break;
771
+ default:
772
+ break;
773
+ }
774
+ }
775
+ /**
776
+ * Send a typed message to the worker process.
777
+ *
778
+ * @param msg - Parent message to send
779
+ */
780
+ sendToWorker(msg) {
781
+ if (this.workerProcess && this.workerProcess.connected) {
782
+ this.workerProcess.send(msg);
783
+ }
784
+ }
785
+ /**
786
+ * Terminate the worker process gracefully, with a forced kill fallback.
787
+ */
788
+ terminateWorker() {
789
+ if (!this.workerProcess)
790
+ return;
791
+ // Try graceful shutdown first
792
+ try {
793
+ if (this.workerProcess.connected) {
794
+ this.sendToWorker({ type: 'shutdown' });
795
+ }
796
+ }
797
+ catch {
798
+ // Ignore send errors during shutdown
799
+ }
800
+ // Force kill after 5s if still alive
801
+ const pid = this.workerProcess.pid;
802
+ const killTimer = setTimeout(() => {
803
+ try {
804
+ if (pid)
805
+ process.kill(pid, 'SIGKILL');
806
+ }
807
+ catch {
808
+ // Already dead
809
+ }
810
+ }, 5_000);
811
+ killTimer.unref();
812
+ this.workerProcess.removeAllListeners();
813
+ this.workerProcess = null;
814
+ this.workerProcessing = false;
815
+ // Reject any pending run
816
+ if (this.workerRunReject) {
817
+ this.workerRunReject(new Error('Worker terminated'));
818
+ this.workerRunResolve = null;
819
+ this.workerRunReject = null;
820
+ }
821
+ }
822
+ /**
823
+ * Get the path to the compiled worker entry file.
824
+ *
825
+ * @returns Absolute path to the agent-worker.js compiled file
826
+ */
827
+ /**
828
+ * @internal Visible for testing — override to provide a custom worker path.
829
+ */
830
+ _workerEntryPath = null;
831
+ getWorkerEntryPath() {
832
+ if (this._workerEntryPath)
833
+ return this._workerEntryPath;
834
+ // Resolve path relative to the compiled dist/ directory.
835
+ // The worker file is always alongside this file after tsc compilation.
836
+ // Use __dirname (available in both CJS and compiled ESM with tsconfig module: NodeNext).
837
+ const thisDir = path.dirname(__filename);
838
+ return path.join(thisDir, 'agent-worker.js');
303
839
  }
304
840
  // ===== Private helpers =====
305
841
  /**
@@ -320,7 +856,8 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
320
856
  return;
321
857
  }
322
858
  // Update heartbeat in teamAgentStatus.json (fire-and-forget)
323
- updateAgentHeartbeat(sessionName).catch((err) => {
859
+ // Pass memberId so the entry is keyed by member ID (not session name)
860
+ updateAgentHeartbeat(sessionName, this.currentMemberId).catch((err) => {
324
861
  this.logger.debug('Heartbeat update failed (non-critical)', {
325
862
  sessionName,
326
863
  error: err instanceof Error ? err.message : String(err),