crewly 1.4.31 → 1.4.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/config/templates/content-generation-team/norms/brand-guidelines.md +11 -0
  2. package/config/templates/content-generation-team/norms/content-review.md +8 -0
  3. package/config/templates/content-generation-team/norms/publish-checklist.md +6 -0
  4. package/config/templates/dev-fullstack/norms/code-commit-sop.md +98 -24
  5. package/config/templates/dev-fullstack/norms/quality-gates.md +44 -17
  6. package/config/templates/research-analysis/norms/research-methodology.md +62 -21
  7. package/config/templates/research-analysis/norms/source-citation.md +69 -17
  8. package/dist/backend/backend/src/constants.d.ts +3 -3
  9. package/dist/backend/backend/src/constants.d.ts.map +1 -1
  10. package/dist/backend/backend/src/constants.js +4 -4
  11. package/dist/backend/backend/src/constants.js.map +1 -1
  12. package/dist/backend/backend/src/controllers/cloud/cloud-google-auth.controller.js +6 -6
  13. package/dist/backend/backend/src/controllers/cloud/cloud-google-auth.controller.js.map +1 -1
  14. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.d.ts +23 -0
  15. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.d.ts.map +1 -0
  16. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.js +48 -0
  17. package/dist/backend/backend/src/controllers/monitoring/extension-logs.controller.js.map +1 -0
  18. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.d.ts.map +1 -1
  19. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.js +6 -0
  20. package/dist/backend/backend/src/controllers/monitoring/monitoring.routes.js.map +1 -1
  21. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.d.ts +45 -0
  22. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.d.ts.map +1 -0
  23. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.js +72 -0
  24. package/dist/backend/backend/src/controllers/monitoring/pty-status.controller.js.map +1 -0
  25. package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
  26. package/dist/backend/backend/src/services/agent/agent-registration.service.js +23 -1
  27. package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
  28. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.d.ts +129 -3
  29. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.d.ts.map +1 -1
  30. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.js +456 -14
  31. package/dist/backend/backend/src/services/agent/crewly-agent/agent-runner.service.js.map +1 -1
  32. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.d.ts +86 -0
  33. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.d.ts.map +1 -0
  34. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.js +147 -0
  35. package/dist/backend/backend/src/services/agent/crewly-agent/agent-worker.js.map +1 -0
  36. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.d.ts +122 -17
  37. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.d.ts.map +1 -1
  38. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.js +580 -46
  39. package/dist/backend/backend/src/services/agent/crewly-agent/crewly-agent-runtime.service.js.map +1 -1
  40. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.d.ts +41 -3
  41. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.d.ts.map +1 -1
  42. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.js +116 -3
  43. package/dist/backend/backend/src/services/agent/crewly-agent/in-process-log-buffer.js.map +1 -1
  44. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.d.ts +17 -0
  45. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.d.ts.map +1 -1
  46. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.js +44 -3
  47. package/dist/backend/backend/src/services/agent/crewly-agent/rate-limiter.js.map +1 -1
  48. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.d.ts.map +1 -1
  49. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.js +134 -39
  50. package/dist/backend/backend/src/services/agent/crewly-agent/tool-registry.js.map +1 -1
  51. package/dist/backend/backend/src/services/agent/crewly-agent/types.d.ts +30 -2
  52. package/dist/backend/backend/src/services/agent/crewly-agent/types.d.ts.map +1 -1
  53. package/dist/backend/backend/src/services/agent/crewly-agent/types.js +16 -2
  54. package/dist/backend/backend/src/services/agent/crewly-agent/types.js.map +1 -1
  55. package/dist/backend/backend/src/services/cloud/cloud-client.service.d.ts +2 -2
  56. package/dist/backend/backend/src/services/cloud/cloud-client.service.d.ts.map +1 -1
  57. package/dist/backend/backend/src/services/cloud/cloud-client.service.js +6 -3
  58. package/dist/backend/backend/src/services/cloud/cloud-client.service.js.map +1 -1
  59. package/dist/backend/backend/src/services/core/env.config.js +1 -1
  60. package/dist/backend/backend/src/services/session/session-backend.interface.d.ts +9 -0
  61. package/dist/backend/backend/src/services/session/session-backend.interface.d.ts.map +1 -1
  62. package/dist/backend/backend/src/services/session/session-backend.interface.js.map +1 -1
  63. package/dist/backend/backend/src/websocket/terminal.gateway.d.ts.map +1 -1
  64. package/dist/backend/backend/src/websocket/terminal.gateway.js +17 -5
  65. package/dist/backend/backend/src/websocket/terminal.gateway.js.map +1 -1
  66. package/dist/cli/backend/src/constants.d.ts +3 -3
  67. package/dist/cli/backend/src/constants.d.ts.map +1 -1
  68. package/dist/cli/backend/src/constants.js +4 -4
  69. package/dist/cli/backend/src/constants.js.map +1 -1
  70. package/dist/cli/cli/src/commands/service.d.ts +4 -0
  71. package/dist/cli/cli/src/commands/service.d.ts.map +1 -1
  72. package/dist/cli/cli/src/commands/service.js +248 -2
  73. package/dist/cli/cli/src/commands/service.js.map +1 -1
  74. package/dist/cli/cli/src/index.js +5 -1
  75. package/dist/cli/cli/src/index.js.map +1 -1
  76. package/frontend/dist/assets/{index-c10b16b7.js → index-64b0eb22.js} +338 -337
  77. package/frontend/dist/assets/{index-2b76b01d.css → index-8772d402.css} +1 -1
  78. package/frontend/dist/index.html +2 -2
  79. package/package.json +1 -1
@@ -1,17 +1,18 @@
1
1
  /**
2
2
  * Crewly Agent Runtime Service
3
3
  *
4
- * Concrete RuntimeAgentService subclass for the in-process Crewly Agent.
5
- * Unlike PTY-based runtimes (Claude Code, Gemini CLI), this runtime runs
6
- * entirely inside the Node.js process using the Vercel AI SDK.
7
- *
8
- * No tmux session, no shell commands — messages are routed directly to
9
- * the AgentRunnerService.handleMessage() method.
4
+ * Concrete RuntimeAgentService subclass for the Crewly Agent.
5
+ * Supports two execution modes:
6
+ * - **In-process** (default): AgentRunner runs in the main Node.js process
7
+ * - **Worker process**: AgentRunner runs in a child process via fork(),
8
+ * enabling hot-reload and crash isolation
10
9
  *
11
10
  * @module services/agent/crewly-agent/crewly-agent-runtime.service
12
11
  */
13
12
  import { promises as fs } from 'fs';
14
13
  import * as path from 'path';
14
+ import { fork } from 'child_process';
15
+ // fileURLToPath used for ESM __dirname equivalent — lazy-loaded to avoid CJS issues
15
16
  import { RuntimeAgentService } from '../runtime-agent.service.abstract.js';
16
17
  import { AgentRunnerService } from './agent-runner.service.js';
17
18
  import { RUNTIME_TYPES, CREWLY_CONSTANTS, ADDON_CONSTANTS } from '../../../constants.js';
@@ -24,19 +25,25 @@ import { PtyActivityTrackerService } from '../pty-activity-tracker.service.js';
24
25
  import { TokenUsageService } from '../../monitoring/token-usage.service.js';
25
26
  import { getSettingsService } from '../../settings/settings.service.js';
26
27
  /**
27
- * In-process Crewly Agent runtime powered by AI SDK generateText.
28
+ * Crewly Agent runtime with optional worker process isolation.
28
29
  *
29
- * Key differences from PTY-based runtimes:
30
- * - No tmux session needed runs in-process
31
- * - Messages routed via handleMessage() instead of PTY write
32
- * - System prompt loaded from config/roles/orchestrator/prompt.md
33
- * - Ready immediately after initialization (no CLI startup wait)
30
+ * Supports two modes:
31
+ * - **In-process** (default): AgentRunner runs directly in the main process
32
+ * - **Worker process** (`useWorkerProcess: true`): AgentRunner runs in a
33
+ * forked child process, enabling hot-reload and crash isolation
34
34
  *
35
35
  * @example
36
36
  * ```typescript
37
+ * // In-process mode (default)
37
38
  * const runtime = new CrewlyAgentRuntimeService(sessionHelper, projectRoot);
38
39
  * await runtime.initializeInProcess('crewly-orc');
39
- * const result = await runtime.handleMessage('Check all team statuses');
40
+ *
41
+ * // Worker process mode
42
+ * const runtime = new CrewlyAgentRuntimeService(sessionHelper, projectRoot);
43
+ * await runtime.initializeInProcess('crewly-orc', { useWorkerProcess: true });
44
+ *
45
+ * // Hot-reload: restart worker with fresh code, preserving session
46
+ * await runtime.hotReload();
40
47
  * ```
41
48
  */
42
49
  export class CrewlyAgentRuntimeService extends RuntimeAgentService {
@@ -48,6 +55,21 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
48
55
  logBuffer;
49
56
  rateLimiter;
50
57
  heartbeatTimer = null;
58
+ /** AbortController for the currently executing message — enables external abort */
59
+ messageAbortController = null;
60
+ // ===== Worker process fields =====
61
+ /** Whether this instance uses a worker process instead of in-process execution */
62
+ useWorkerProcess = false;
63
+ /** The forked worker child process */
64
+ workerProcess = null;
65
+ /** Stored config for hot-reload — needed to re-init the worker */
66
+ storedConfig = null;
67
+ /** Pending promise resolver for the current worker run */
68
+ workerRunResolve = null;
69
+ /** Pending promise rejector for the current worker run */
70
+ workerRunReject = null;
71
+ /** Whether the worker is currently processing a message */
72
+ workerProcessing = false;
51
73
  constructor(sessionHelper, projectRoot) {
52
74
  super(sessionHelper, projectRoot);
53
75
  this.logBuffer = InProcessLogBuffer.getInstance();
@@ -70,6 +92,9 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
70
92
  * @returns True if the agent runner is initialized
71
93
  */
72
94
  async detectRuntimeSpecific(_sessionName) {
95
+ if (this.useWorkerProcess) {
96
+ return this.initialized && this.workerProcess !== null && this.workerProcess.connected;
97
+ }
73
98
  return this.initialized && this.agentRunner !== null && this.agentRunner.isInitialized();
74
99
  }
75
100
  /**
@@ -99,18 +124,19 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
99
124
  }
100
125
  // ===== In-process lifecycle methods =====
101
126
  /**
102
- * Initialize the in-process agent runtime.
127
+ * Initialize the agent runtime.
103
128
  *
104
129
  * Loads the system prompt from config/roles/orchestrator/prompt.md,
105
- * creates the AgentRunnerService, and initializes the model.
130
+ * creates the AgentRunnerService (in-process or worker), and initializes the model.
106
131
  *
107
132
  * @param sessionName - Session name for this agent instance
108
- * @param config - Optional partial config overrides
133
+ * @param config - Optional partial config overrides. Set `useWorkerProcess: true` to run in a child process.
109
134
  * @param roleName - Role name for system prompt lookup (default: 'orchestrator')
110
135
  */
111
136
  async initializeInProcess(sessionName, config, roleName) {
112
137
  this.currentSessionName = sessionName;
113
138
  this.currentMemberId = config?.memberId;
139
+ this.useWorkerProcess = config?.useWorkerProcess ?? false;
114
140
  // Build enhanced system prompt with skills and addon awareness
115
141
  const systemPrompt = await this.buildEnhancedSystemPrompt(roleName || 'orchestrator');
116
142
  // Build full config with defaults
@@ -124,15 +150,22 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
124
150
  compactionThreshold: config?.compactionThreshold || CREWLY_AGENT_DEFAULTS.COMPACTION_THRESHOLD,
125
151
  projectPath: config?.projectPath,
126
152
  };
127
- this.agentRunner = new AgentRunnerService(fullConfig);
128
- try {
129
- await this.agentRunner.initialize();
153
+ // Store config for hot-reload
154
+ this.storedConfig = fullConfig;
155
+ if (this.useWorkerProcess) {
156
+ await this.initializeWorker(fullConfig);
130
157
  }
131
- catch (error) {
132
- // Clean up on initialization failure to prevent partial state
133
- this.agentRunner = null;
134
- this.currentSessionName = null;
135
- throw error;
158
+ else {
159
+ this.agentRunner = new AgentRunnerService(fullConfig);
160
+ try {
161
+ await this.agentRunner.initialize();
162
+ }
163
+ catch (error) {
164
+ // Clean up on initialization failure to prevent partial state
165
+ this.agentRunner = null;
166
+ this.currentSessionName = null;
167
+ throw error;
168
+ }
136
169
  }
137
170
  this.initialized = true;
138
171
  this.currentModelString = `${fullConfig.model.provider}/${fullConfig.model.modelId}`;
@@ -140,9 +173,11 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
140
173
  this.startHeartbeat(sessionName);
141
174
  // Register in-process session for frontend terminal visibility
142
175
  this.logBuffer.registerSession(sessionName);
143
- this.logBuffer.append(sessionName, 'info', `Crewly Agent initialized (${this.currentModelString})`);
176
+ const mode = this.useWorkerProcess ? 'worker' : 'in-process';
177
+ this.logBuffer.append(sessionName, 'info', `Crewly Agent initialized [${mode}] (${this.currentModelString})`);
144
178
  this.logger.info('Crewly Agent runtime initialized', {
145
179
  sessionName,
180
+ mode,
146
181
  model: `${fullConfig.model.provider}/${fullConfig.model.modelId}`,
147
182
  maxSteps: fullConfig.maxSteps,
148
183
  });
@@ -159,9 +194,15 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
159
194
  * @throws Error if the runtime is not initialized
160
195
  */
161
196
  async handleMessage(message, metadata) {
162
- if (!this.agentRunner || !this.initialized) {
197
+ if (!this.initialized) {
163
198
  throw new Error('Crewly Agent runtime not initialized. Call initializeInProcess() first.');
164
199
  }
200
+ if (!this.useWorkerProcess && !this.agentRunner) {
201
+ throw new Error('Crewly Agent runtime not initialized. Call initializeInProcess() first.');
202
+ }
203
+ if (this.useWorkerProcess && (!this.workerProcess || !this.workerProcess.connected)) {
204
+ throw new Error('Worker process not available. Call initializeInProcess() or hotReload() first.');
205
+ }
165
206
  const session = this.currentSessionName;
166
207
  // Extract conversationId from [CHAT:xxx] or [GCHAT:xxx ...] prefix if present
167
208
  let conversationId;
@@ -176,17 +217,25 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
176
217
  });
177
218
  }
178
219
  const queueLen = this.rateLimiter.getQueueLength();
179
- this.logBuffer.append(session, 'info', `← Message received (${cleanMessage.length} chars${conversationId ? `, conv:${conversationId}` : ''}${queueLen > 0 ? `, queue:${queueLen}` : ''})`);
180
- this.logger.debug('Handling message via rate limiter', {
181
- sessionName: session,
182
- messageLength: cleanMessage.length,
183
- historyLength: this.agentRunner.getHistoryLength(),
184
- conversationId,
185
- queueLength: queueLen,
186
- requestsInWindow: this.rateLimiter.getRequestCountInWindow(),
187
- });
220
+ const msgPreview = cleanMessage.length <= 120
221
+ ? `"${cleanMessage}"`
222
+ : `"${cleanMessage.substring(0, 50)}...${cleanMessage.substring(cleanMessage.length - 50)}"`;
223
+ this.logBuffer.append(session, 'info', `← Message received (${cleanMessage.length} chars${conversationId ? `, conv:${conversationId}` : ''}${queueLen > 0 ? `, queue:${queueLen}` : ''}): ${msgPreview}`);
224
+ if (!this.useWorkerProcess) {
225
+ this.logger.debug('Handling message via rate limiter', {
226
+ sessionName: session,
227
+ messageLength: cleanMessage.length,
228
+ historyLength: this.agentRunner.getHistoryLength(),
229
+ conversationId,
230
+ queueLength: queueLen,
231
+ requestsInWindow: this.rateLimiter.getRequestCountInWindow(),
232
+ });
233
+ }
188
234
  // Route through rate limiter for throttling, coalescing, and 429 retry
189
235
  const result = await this.rateLimiter.enqueue(cleanMessage, metadata, async (msg, meta) => {
236
+ if (this.useWorkerProcess) {
237
+ return this.executeMessageViaWorker(session, msg, conversationId, meta);
238
+ }
190
239
  return this.executeMessage(session, msg, conversationId, meta);
191
240
  });
192
241
  return result;
@@ -204,26 +253,137 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
204
253
  * @returns Agent run result
205
254
  */
206
255
  async executeMessage(session, cleanMessage, conversationId, metadata) {
207
- // Soft warning timer — logs if processing exceeds 5 min but does NOT kill it.
208
- // Individual tool calls (bash_exec) have their own per-step timeouts (30s default).
209
- const SOFT_WARNING_MS = 5 * 60 * 1000;
256
+ const SOFT_WARNING_MS = CREWLY_AGENT_DEFAULTS.MESSAGE_SOFT_WARNING_MS;
257
+ // Execution tracking for diagnostics
258
+ const executionTracker = {
259
+ phase: 'queued',
260
+ currentTool: null,
261
+ toolCallsCompleted: [],
262
+ startedAt: new Date(),
263
+ lastActivityAt: new Date(),
264
+ messagePreview: cleanMessage.length <= 100
265
+ ? cleanMessage
266
+ : `${cleanMessage.substring(0, 50)}...${cleanMessage.substring(cleanMessage.length - 50)}`,
267
+ };
268
+ // Soft warning timer — logs if processing exceeds threshold but does NOT kill it.
210
269
  const warningTimer = setTimeout(() => {
211
- this.logger.warn('Message processing exceeding 5 minutes (still running)', {
270
+ executionTracker.lastActivityAt = new Date();
271
+ this.logger.warn(`Message processing exceeding ${SOFT_WARNING_MS / 1000}s (still running)`, {
212
272
  sessionName: session,
273
+ phase: executionTracker.phase,
274
+ toolCallsCompleted: executionTracker.toolCallsCompleted.length,
213
275
  messagePreview: cleanMessage.substring(0, 100),
214
276
  });
215
277
  }, SOFT_WARNING_MS);
278
+ // AbortController for external abort (abortCurrentRun) and repetition detection
279
+ const abortController = new AbortController();
280
+ this.messageAbortController = abortController;
281
+ // Text chunk buffer — collects streaming text and flushes on step boundaries
282
+ let textChunkBuffer = '';
283
+ // Repetition/hallucination detection — tracks recent chunks to detect loops
284
+ const recentChunks = [];
285
+ const REPETITION_WINDOW = 20; // number of recent chunks to track
286
+ const REPETITION_THRESHOLD = 5; // consecutive repeated patterns to trigger abort
287
+ let repetitionDetected = false;
288
+ // Build streaming callbacks that write to InProcessLogBuffer in real-time
289
+ const streamingCallbacks = {
290
+ onTextChunk: (chunk) => {
291
+ if (chunk.length > 0) {
292
+ executionTracker.lastActivityAt = new Date();
293
+ executionTracker.phase = 'model-thinking';
294
+ textChunkBuffer += chunk;
295
+ // Repetition detection: track recent chunks and check for loops
296
+ const trimmed = chunk.trim();
297
+ if (trimmed.length > 0) {
298
+ recentChunks.push(trimmed);
299
+ if (recentChunks.length > REPETITION_WINDOW) {
300
+ recentChunks.shift();
301
+ }
302
+ // Check if the last REPETITION_THRESHOLD chunks are identical
303
+ if (recentChunks.length >= REPETITION_THRESHOLD) {
304
+ const tail = recentChunks.slice(-REPETITION_THRESHOLD);
305
+ const allSame = tail.every(c => c === tail[0]);
306
+ if (allSame && tail[0].length >= 3) {
307
+ repetitionDetected = true;
308
+ this.logBuffer.append(session, 'warn', `⚠️ Repetition loop detected: "${tail[0].substring(0, 80)}" repeated ${REPETITION_THRESHOLD}x — aborting generation`);
309
+ this.logger.warn('Repetition/hallucination loop detected, aborting', {
310
+ sessionName: session,
311
+ repeatedChunk: tail[0].substring(0, 100),
312
+ count: REPETITION_THRESHOLD,
313
+ });
314
+ abortController.abort();
315
+ }
316
+ }
317
+ }
318
+ }
319
+ },
320
+ onToolCallStart: (toolName, _args) => {
321
+ executionTracker.phase = 'tool-calling';
322
+ executionTracker.currentTool = toolName;
323
+ executionTracker.lastActivityAt = new Date();
324
+ },
325
+ onToolCallFinish: (toolName, args, result, _durationMs) => {
326
+ executionTracker.toolCallsCompleted.push(toolName);
327
+ executionTracker.currentTool = null;
328
+ executionTracker.lastActivityAt = new Date();
329
+ const argsPreview = JSON.stringify(args).substring(0, 120);
330
+ this.logBuffer.append(session, 'info', `🔧 ${toolName}(${argsPreview})`);
331
+ // For bash_exec, show the command as an extra log line for readability
332
+ if (toolName === 'bash_exec' && args.command) {
333
+ const cmdPreview = String(args.command).substring(0, 200);
334
+ this.logBuffer.append(session, 'info', ` $ ${cmdPreview}`);
335
+ }
336
+ const resultPreview = result ? JSON.stringify(result).substring(0, 200) : 'void';
337
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
338
+ },
339
+ onStepFinish: (stepIndex, hasToolCalls) => {
340
+ executionTracker.lastActivityAt = new Date();
341
+ // Flush buffered text at each step boundary
342
+ if (textChunkBuffer.trim().length > 0) {
343
+ // Truncate very long text to keep logs readable
344
+ const text = textChunkBuffer.trim();
345
+ const preview = text.length > 500 ? text.substring(0, 500) + '...' : text;
346
+ this.logBuffer.append(session, 'info', `💬 ${preview}`);
347
+ textChunkBuffer = '';
348
+ }
349
+ if (!hasToolCalls) {
350
+ executionTracker.phase = 'model-thinking';
351
+ }
352
+ },
353
+ };
216
354
  try {
217
- const result = await this.agentRunner.run(cleanMessage, conversationId, metadata);
355
+ executionTracker.phase = 'model-thinking';
356
+ const result = await this.agentRunner.run(cleanMessage, conversationId, metadata, {
357
+ abortSignal: abortController.signal,
358
+ streaming: streamingCallbacks,
359
+ });
218
360
  clearTimeout(warningTimer);
219
- // Log tool calls to buffer for frontend visibility
220
- for (const tc of result.toolCalls) {
221
- const argsPreview = JSON.stringify(tc.args).substring(0, 120);
222
- this.logBuffer.append(session, 'info', `🔧 ${tc.toolName}(${argsPreview})`);
223
- const resultPreview = tc.result ? JSON.stringify(tc.result).substring(0, 200) : 'void';
224
- this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
361
+ this.messageAbortController = null;
362
+ // If we got here after a repetition-triggered abort, treat as error
363
+ if (repetitionDetected) {
364
+ throw new Error('Generation aborted: repetition/hallucination loop detected. '
365
+ + `Repeated pattern: "${recentChunks[recentChunks.length - 1]?.substring(0, 80)}"`);
366
+ }
367
+ // Flush any remaining buffered text after the run completes
368
+ if (textChunkBuffer.trim().length > 0) {
369
+ const text = textChunkBuffer.trim();
370
+ const preview = text.length > 500 ? text.substring(0, 500) + '...' : text;
371
+ this.logBuffer.append(session, 'info', `💬 ${preview}`);
372
+ textChunkBuffer = '';
373
+ }
374
+ // Tool calls already logged via streaming callbacks (onToolCallStart/Finish).
375
+ // Only log tool calls retroactively if generateText path was used (test mock).
376
+ if (this.agentRunner._generateTextFn) {
377
+ for (const tc of result.toolCalls) {
378
+ executionTracker.toolCallsCompleted.push(tc.toolName);
379
+ const argsPreview = JSON.stringify(tc.args).substring(0, 120);
380
+ this.logBuffer.append(session, 'info', `🔧 ${tc.toolName}(${argsPreview})`);
381
+ const resultPreview = tc.result ? JSON.stringify(tc.result).substring(0, 200) : 'void';
382
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
383
+ }
225
384
  }
226
385
  // Log response summary
386
+ executionTracker.phase = 'complete';
227
387
  const textPreview = result.text ? result.text.substring(0, 150) : '(no text)';
228
388
  this.logBuffer.append(session, 'info', `→ Response (${result.steps} steps, ${result.toolCalls.length} tools): ${textPreview}`);
229
389
  this.logBuffer.append(session, 'debug', ` Tokens: ${result.usage.input}in/${result.usage.output}out`);
@@ -242,6 +402,14 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
242
402
  }
243
403
  catch (error) {
244
404
  clearTimeout(warningTimer);
405
+ this.messageAbortController = null;
406
+ // If this was a repetition-triggered abort, wrap with a clear error
407
+ if (repetitionDetected) {
408
+ const repErr = new Error('Generation aborted: repetition/hallucination loop detected. '
409
+ + `Repeated pattern: "${recentChunks[recentChunks.length - 1]?.substring(0, 80)}"`);
410
+ this.logBuffer.append(session, 'error', `Agent error: ${repErr.message}`);
411
+ throw repErr;
412
+ }
245
413
  const errMsg = error instanceof Error ? error.message : String(error);
246
414
  this.logBuffer.append(session, 'error', `Agent error: ${errMsg}`);
247
415
  throw error;
@@ -265,8 +433,54 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
265
433
  * @returns True if initializeInProcess() has been called successfully
266
434
  */
267
435
  isReady() {
436
+ if (this.useWorkerProcess) {
437
+ return this.initialized && this.workerProcess !== null && this.workerProcess.connected;
438
+ }
268
439
  return this.initialized && this.agentRunner !== null && this.agentRunner.isInitialized();
269
440
  }
441
+ /**
442
+ * Abort the currently executing message processing.
443
+ *
444
+ * Cancels the active model call, terminates running tool processes,
445
+ * and returns partial results where possible. Safe to call at any time —
446
+ * returns false if no run is in progress.
447
+ *
448
+ * @returns True if an active run was aborted, false if nothing was running
449
+ */
450
+ abortCurrentRun() {
451
+ const session = this.currentSessionName;
452
+ if (this.useWorkerProcess) {
453
+ if (!this.workerProcessing || !this.workerProcess) {
454
+ return false;
455
+ }
456
+ this.sendToWorker({ type: 'abort' });
457
+ if (this.workerRunReject) {
458
+ this.workerRunReject(new Error('Run aborted by user'));
459
+ this.workerRunResolve = null;
460
+ this.workerRunReject = null;
461
+ }
462
+ this.workerProcessing = false;
463
+ if (session) {
464
+ this.logBuffer.append(session, 'warn', '⚠️ Run aborted by user');
465
+ }
466
+ this.logger.info('Agent run aborted (worker)', { sessionName: session });
467
+ return true;
468
+ }
469
+ if (!this.messageAbortController) {
470
+ return false;
471
+ }
472
+ this.messageAbortController.abort();
473
+ this.messageAbortController = null;
474
+ // Also tell the runner to abort (for cases where the runner has its own abort)
475
+ if (this.agentRunner) {
476
+ this.agentRunner.abortCurrentRun();
477
+ }
478
+ if (session) {
479
+ this.logBuffer.append(session, 'warn', '⚠️ Run aborted by user');
480
+ }
481
+ this.logger.info('Agent run aborted', { sessionName: session });
482
+ return true;
483
+ }
270
484
  /**
271
485
  * Get the current agent runner instance (for inspection/testing).
272
486
  *
@@ -290,11 +504,16 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
290
504
  shutdown() {
291
505
  this.logger.info('Shutting down Crewly Agent runtime', {
292
506
  sessionName: this.currentSessionName,
507
+ mode: this.useWorkerProcess ? 'worker' : 'in-process',
293
508
  });
294
509
  // Mark as not initialized first to reject new messages immediately
295
510
  this.initialized = false;
296
511
  // Stop heartbeat timer
297
512
  this.stopHeartbeat();
513
+ // Shut down worker process if running
514
+ if (this.workerProcess) {
515
+ this.terminateWorker();
516
+ }
298
517
  if (this.currentSessionName) {
299
518
  this.logBuffer.append(this.currentSessionName, 'info', 'Crewly Agent shutting down');
300
519
  this.logBuffer.removeSession(this.currentSessionName);
@@ -302,6 +521,321 @@ export class CrewlyAgentRuntimeService extends RuntimeAgentService {
302
521
  this.rateLimiter.reset();
303
522
  this.agentRunner = null;
304
523
  this.currentSessionName = null;
524
+ this.storedConfig = null;
525
+ }
526
+ // ===== Worker process methods =====
527
+ /**
528
+ * Hot-reload the worker process.
529
+ *
530
+ * Kills the existing worker and spawns a fresh one with the stored config.
531
+ * This allows updating agent code without restarting the main backend.
532
+ * Conversation state is reset — use this when deploying new agent logic.
533
+ *
534
+ * @throws Error if not in worker mode or config is missing
535
+ */
536
+ async hotReload() {
537
+ if (!this.useWorkerProcess) {
538
+ throw new Error('hotReload() is only available in worker process mode');
539
+ }
540
+ if (!this.storedConfig) {
541
+ throw new Error('No stored config for hot-reload. Was initializeInProcess() called?');
542
+ }
543
+ const session = this.currentSessionName;
544
+ this.logBuffer.append(session, 'info', 'Hot-reloading worker process...');
545
+ this.logger.info('Hot-reloading worker process', { sessionName: session });
546
+ // Terminate old worker
547
+ this.terminateWorker();
548
+ // Spawn new worker with same config
549
+ await this.initializeWorker(this.storedConfig);
550
+ this.logBuffer.append(session, 'info', 'Worker hot-reload complete');
551
+ this.logger.info('Worker hot-reload complete', { sessionName: session });
552
+ }
553
+ /**
554
+ * Check if the runtime is using a worker process.
555
+ *
556
+ * @returns True if running in worker process mode
557
+ */
558
+ isWorkerMode() {
559
+ return this.useWorkerProcess;
560
+ }
561
+ /**
562
+ * Get the worker process PID (for monitoring/debugging).
563
+ *
564
+ * @returns Worker PID, or null if not in worker mode or worker is not running
565
+ */
566
+ getWorkerPid() {
567
+ return this.workerProcess?.pid ?? null;
568
+ }
569
+ /**
570
+ * Initialize a worker child process via fork().
571
+ *
572
+ * Forks the agent-worker.ts entry point and sends the init message
573
+ * with the agent config. Waits for the 'ready' response before resolving.
574
+ *
575
+ * @param config - Full agent config to send to the worker
576
+ * @throws Error if worker fails to initialize within timeout
577
+ */
578
+ async initializeWorker(config) {
579
+ return new Promise((resolve, reject) => {
580
+ const workerPath = this.getWorkerEntryPath();
581
+ this.workerProcess = fork(workerPath, [], {
582
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
583
+ env: { ...process.env },
584
+ });
585
+ const initTimeout = setTimeout(() => {
586
+ this.terminateWorker();
587
+ reject(new Error('Worker initialization timed out (30s)'));
588
+ }, 30_000);
589
+ let initResolved = false;
590
+ this.workerProcess.on('message', (msg) => {
591
+ // Handle init response
592
+ if (!initResolved && msg.type === 'ready') {
593
+ initResolved = true;
594
+ clearTimeout(initTimeout);
595
+ resolve();
596
+ return;
597
+ }
598
+ if (!initResolved && msg.type === 'error' && msg.code === 'INIT_FAILED') {
599
+ initResolved = true;
600
+ clearTimeout(initTimeout);
601
+ reject(new Error(msg.error));
602
+ return;
603
+ }
604
+ // Handle runtime messages
605
+ this.handleWorkerMessage(msg);
606
+ });
607
+ this.workerProcess.on('exit', (code, signal) => {
608
+ this.logger.warn('Worker process exited', {
609
+ sessionName: this.currentSessionName,
610
+ code,
611
+ signal,
612
+ });
613
+ if (!initResolved) {
614
+ initResolved = true;
615
+ clearTimeout(initTimeout);
616
+ reject(new Error(`Worker exited during init (code=${code}, signal=${signal})`));
617
+ }
618
+ // Reject any pending run
619
+ if (this.workerRunReject) {
620
+ this.workerRunReject(new Error(`Worker process exited unexpectedly (code=${code}, signal=${signal})`));
621
+ this.workerRunResolve = null;
622
+ this.workerRunReject = null;
623
+ this.workerProcessing = false;
624
+ }
625
+ this.workerProcess = null;
626
+ if (this.currentSessionName) {
627
+ this.logBuffer.append(this.currentSessionName, 'warn', `Worker process exited (code=${code}, signal=${signal})`);
628
+ }
629
+ });
630
+ this.workerProcess.on('error', (err) => {
631
+ this.logger.error('Worker process error', {
632
+ sessionName: this.currentSessionName,
633
+ error: err.message,
634
+ });
635
+ if (!initResolved) {
636
+ initResolved = true;
637
+ clearTimeout(initTimeout);
638
+ reject(err);
639
+ }
640
+ });
641
+ // Capture worker stdout/stderr for debugging
642
+ this.workerProcess.stdout?.on('data', (data) => {
643
+ const text = data.toString().trim();
644
+ if (text && this.currentSessionName) {
645
+ this.logBuffer.append(this.currentSessionName, 'debug', `[worker stdout] ${text}`);
646
+ }
647
+ });
648
+ this.workerProcess.stderr?.on('data', (data) => {
649
+ const text = data.toString().trim();
650
+ if (text && this.currentSessionName) {
651
+ this.logBuffer.append(this.currentSessionName, 'warn', `[worker stderr] ${text}`);
652
+ }
653
+ });
654
+ // Send init message with config
655
+ this.sendToWorker({ type: 'init', config });
656
+ });
657
+ }
658
+ /**
659
+ * Execute a message via the worker process using IPC.
660
+ *
661
+ * Sends a 'run' message to the worker and waits for the 'result' or 'error'
662
+ * response. Streaming events are forwarded to the InProcessLogBuffer in real-time.
663
+ *
664
+ * @param session - Session name
665
+ * @param cleanMessage - Message content (prefix already stripped)
666
+ * @param conversationId - Optional conversation ID
667
+ * @param _metadata - Optional metadata (passed to worker)
668
+ * @returns Agent run result from the worker
669
+ */
670
+ executeMessageViaWorker(session, cleanMessage, conversationId, _metadata) {
671
+ return new Promise((resolve, reject) => {
672
+ if (!this.workerProcess || !this.workerProcess.connected) {
673
+ reject(new Error('Worker process not available'));
674
+ return;
675
+ }
676
+ this.workerProcessing = true;
677
+ this.workerRunResolve = (result) => {
678
+ this.workerProcessing = false;
679
+ this.workerRunResolve = null;
680
+ this.workerRunReject = null;
681
+ // Log response summary
682
+ const textPreview = result.text ? result.text.substring(0, 150) : '(no text)';
683
+ this.logBuffer.append(session, 'info', `→ Response (${result.steps} steps, ${result.toolCalls.length} tools): ${textPreview}`);
684
+ this.logBuffer.append(session, 'debug', ` Tokens: ${result.usage.input}in/${result.usage.output}out`);
685
+ this.logger.info('Message processed (worker)', {
686
+ sessionName: session,
687
+ steps: result.steps,
688
+ toolCalls: result.toolCalls.length,
689
+ usage: result.usage,
690
+ finishReason: result.finishReason,
691
+ });
692
+ // Record token usage
693
+ this.recordTokenUsageIfEnabled(session, result).catch(() => { });
694
+ resolve(result);
695
+ };
696
+ this.workerRunReject = (error) => {
697
+ this.workerProcessing = false;
698
+ this.workerRunResolve = null;
699
+ this.workerRunReject = null;
700
+ this.logBuffer.append(session, 'error', `Agent error (worker): ${error.message}`);
701
+ reject(error);
702
+ };
703
+ this.sendToWorker({
704
+ type: 'run',
705
+ message: cleanMessage,
706
+ conversationId,
707
+ metadata: _metadata,
708
+ });
709
+ });
710
+ }
711
+ /**
712
+ * Handle messages received from the worker process.
713
+ *
714
+ * Routes streaming events to the InProcessLogBuffer and resolves/rejects
715
+ * pending run promises on result/error messages.
716
+ *
717
+ * @param msg - Worker message received via IPC
718
+ */
719
+ handleWorkerMessage(msg) {
720
+ const session = this.currentSessionName;
721
+ switch (msg.type) {
722
+ case 'result':
723
+ if (this.workerRunResolve) {
724
+ this.workerRunResolve(msg.data);
725
+ }
726
+ break;
727
+ case 'error':
728
+ if (this.workerRunReject) {
729
+ this.workerRunReject(new Error(msg.error));
730
+ }
731
+ else if (session) {
732
+ // Error outside of a run (e.g. crash notification)
733
+ this.logBuffer.append(session, 'error', `Worker error: ${msg.error}`);
734
+ }
735
+ break;
736
+ case 'log':
737
+ if (session) {
738
+ this.logBuffer.append(session, msg.level, `[worker] ${msg.message}`);
739
+ }
740
+ break;
741
+ case 'stream':
742
+ if (!session)
743
+ break;
744
+ switch (msg.event) {
745
+ case 'text':
746
+ // Text chunks are buffered and flushed at step boundaries
747
+ break;
748
+ case 'toolStart':
749
+ // No-op: logged on finish
750
+ break;
751
+ case 'toolFinish': {
752
+ const { toolName, args, result } = msg.data;
753
+ const argsPreview = JSON.stringify(args).substring(0, 120);
754
+ this.logBuffer.append(session, 'info', `🔧 ${toolName}(${argsPreview})`);
755
+ if (toolName === 'bash_exec' && args.command) {
756
+ const cmdPreview = String(args.command).substring(0, 200);
757
+ this.logBuffer.append(session, 'info', ` $ ${cmdPreview}`);
758
+ }
759
+ const resultPreview = result ? JSON.stringify(result).substring(0, 200) : 'void';
760
+ this.logBuffer.append(session, 'debug', ` → ${resultPreview}`);
761
+ break;
762
+ }
763
+ case 'stepFinish':
764
+ // Step boundaries are tracked by the worker
765
+ break;
766
+ }
767
+ break;
768
+ case 'state':
769
+ // State query responses (used internally)
770
+ break;
771
+ default:
772
+ break;
773
+ }
774
+ }
775
+ /**
776
+ * Send a typed message to the worker process.
777
+ *
778
+ * @param msg - Parent message to send
779
+ */
780
+ sendToWorker(msg) {
781
+ if (this.workerProcess && this.workerProcess.connected) {
782
+ this.workerProcess.send(msg);
783
+ }
784
+ }
785
+ /**
786
+ * Terminate the worker process gracefully, with a forced kill fallback.
787
+ */
788
+ terminateWorker() {
789
+ if (!this.workerProcess)
790
+ return;
791
+ // Try graceful shutdown first
792
+ try {
793
+ if (this.workerProcess.connected) {
794
+ this.sendToWorker({ type: 'shutdown' });
795
+ }
796
+ }
797
+ catch {
798
+ // Ignore send errors during shutdown
799
+ }
800
+ // Force kill after 5s if still alive
801
+ const pid = this.workerProcess.pid;
802
+ const killTimer = setTimeout(() => {
803
+ try {
804
+ if (pid)
805
+ process.kill(pid, 'SIGKILL');
806
+ }
807
+ catch {
808
+ // Already dead
809
+ }
810
+ }, 5_000);
811
+ killTimer.unref();
812
+ this.workerProcess.removeAllListeners();
813
+ this.workerProcess = null;
814
+ this.workerProcessing = false;
815
+ // Reject any pending run
816
+ if (this.workerRunReject) {
817
+ this.workerRunReject(new Error('Worker terminated'));
818
+ this.workerRunResolve = null;
819
+ this.workerRunReject = null;
820
+ }
821
+ }
822
+ /**
823
+ * Get the path to the compiled worker entry file.
824
+ *
825
+ * @returns Absolute path to the agent-worker.js compiled file
826
+ */
827
+ /**
828
+ * @internal Visible for testing — override to provide a custom worker path.
829
+ */
830
+ _workerEntryPath = null;
831
+ getWorkerEntryPath() {
832
+ if (this._workerEntryPath)
833
+ return this._workerEntryPath;
834
+ // Resolve path relative to the compiled dist/ directory.
835
+ // The worker file is always alongside this file after tsc compilation.
836
+ // Use __dirname (available in both CJS and compiled ESM with tsconfig module: NodeNext).
837
+ const thisDir = path.dirname(__filename);
838
+ return path.join(thisDir, 'agent-worker.js');
305
839
  }
306
840
  // ===== Private helpers =====
307
841
  /**