crewly 1.14.1 → 1.15.0

package/dist/cli/backend/src/services/agent/agent-registration.service.js

@@ -7,7 +7,7 @@ import { createSessionCommandHelper, getSessionBackendSync, createSessionBackend
 import { RuntimeServiceFactory } from './runtime-service.factory.js';
 import { registerInProcessRuntime, unregisterInProcessRuntime, } from './crewly-agent/in-process-runtime-registry.js';
 import { getCrewlyHomePath } from '../core/crewly-home.utils.js';
-import { CREWLY_CONSTANTS, ENV_CONSTANTS, AGENT_TIMEOUTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, RUNTIME_TYPES, SESSION_COMMAND_DELAYS, EVENT_DELIVERY_CONSTANTS, TERMINAL_PATTERNS, GEMINI_SHELL_MODE_CONSTANTS, GEMINI_STUCK_CONNECTIVITY_PATTERN, GEMINI_ERROR_STATE_CONSTANTS, } from '../../constants.js';
+import { CREWLY_CONSTANTS, ENV_CONSTANTS, AGENT_TIMEOUTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, RUNTIME_TYPES, SESSION_COMMAND_DELAYS, EVENT_DELIVERY_CONSTANTS, TERMINAL_PATTERNS, GEMINI_SHELL_MODE_CONSTANTS, GEMINI_STUCK_CONNECTIVITY_PATTERN, GEMINI_ERROR_STATE_CONSTANTS, REGISTRATION_DELIVERY_CONSTANTS, } from '../../constants.js';
 import { WEB_CONSTANTS } from '../../../../config/constants.js';
 import { delay } from '../../utils/async.utils.js';
 import { getSettingsService } from '../settings/settings.service.js';
@@ -20,6 +20,7 @@ import { SubAgentMessageQueue } from '../messaging/sub-agent-message-queue.servi
 import { AgentSuspendService } from './agent-suspend.service.js';
 import { PromptBuilderService, buildModuleConfigFromTeamMember, } from '../ai/prompt-builder.service.js';
 import { PromptAssemblyService } from '../ai/prompt-modules/prompt-assembly.service.js';
+import { resolveOrcPromptProfile, ORC_PROMPT_PROFILE_ENV, } from '../ai/prompt-modules/prompt-module.interface.js';
 import { stripAnsiCodes } from '../../utils/terminal-output.utils.js';
 import { isPromptLine, containsSpinnerOrWorkingIndicator, containsProcessingIndicator, containsBusyStatusBar, containsRewindMode, extractChatPrefix, stripTuiLineBorders, matchTuiPromptLine, } from '../../utils/terminal-string-ops.js';
 import { PtyActivityTrackerService } from './pty-activity-tracker.service.js';
@@ -620,6 +621,16 @@ export class AgentRegistrationService {
     getCheckInterval() {
         return process.env.NODE_ENV === 'test' ? 1000 : 2000;
     }
+    /**
+     * Use the short test-environment cadence under Jest, otherwise the given
+     * production interval.
+     *
+     * @param productionMs - Interval to use outside the test environment
+     * @returns Interval in milliseconds
+     */
+    testAwareInterval(productionMs) {
+        return process.env.NODE_ENV === 'test' ? this.getCheckInterval() : productionMs;
+    }
     /**
      * Update agent status with safe error handling (non-blocking).
      * Returns true if successful, false if failed.
@@ -658,6 +669,49 @@ export class AgentRegistrationService {
         ['tl', 'team-leader'],
         ['team-lead', 'team-leader'],
     ]);
+    /**
+     * Assemble the orchestrator prompt at the *other* profile and log the
+     * module count + token estimate of both tiers side by side, so the
+     * operator can judge what `CREWLY_ORC_PROMPT_PROFILE=lite` would save (or
+     * did save) without diffing prompts by hand. Read-only and best-effort:
+     * a failure here never affects the prompt that was already assembled.
+     *
+     * @param assembler - The assembler used for the live prompt
+     * @param moduleConfig - Config the live prompt was assembled from
+     * @param liveReport - Report of the live (selected-profile) assembly
+     */
+    async logPromptProfileComparison(assembler, moduleConfig, liveReport) {
+        const otherProfile = liveReport.profile === 'lite' ? 'full' : 'lite';
+        try {
+            const { report: other } = await assembler.assemble({
+                ...moduleConfig,
+                promptProfile: otherProfile,
+            });
+            const byProfile = {
+                [liveReport.profile]: {
+                    moduleCount: liveReport.moduleBreakdown.length,
+                    estimatedTokens: liveReport.totalTokens,
+                },
+                [otherProfile]: {
+                    moduleCount: other.moduleBreakdown.length,
+                    estimatedTokens: other.totalTokens,
+                },
+            };
+            this.logger.info('Orchestrator prompt profile comparison', {
+                sessionName: moduleConfig.sessionName,
+                selected: liveReport.profile,
+                env: ORC_PROMPT_PROFILE_ENV,
+                ...byProfile,
+                liteSavesTokens: (byProfile.full?.estimatedTokens ?? 0) - (byProfile.lite?.estimatedTokens ?? 0),
+            });
+        }
+        catch (error) {
+            this.logger.debug('Prompt profile comparison skipped', {
+                sessionName: moduleConfig.sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
     /**
      * Get prompt file path for a specific role
      * Uses the unified config/roles/{role}/prompt.md structure
@@ -1052,6 +1106,14 @@ export class AgentRegistrationService {
             this.logger.info('Registration prompt loaded, sending to agent', {
                 sessionName, role, runtimeType, promptLength: prompt.length,
             });
+            // Gate on the runtime actually accepting input. `waitForRuntimeReady`
+            // (already passed by the caller) matches banner text that is on screen
+            // while the TUI is still booting — with Codex the instruction landed
+            // ~2 s after spawn, during `model: loading`, and was swallowed while
+            // every log line still said "sent successfully" (server-install finding 3).
+            if (controller.signal.aborted)
+                return;
+            await this.waitForRuntimeInputReady(sessionName, runtimeType, controller.signal);
             if (controller.signal.aborted)
                 return;
             const sent = await this.sendPromptRobustly(sessionName, prompt, runtimeType, controller.signal);
@@ -1061,6 +1123,10 @@ export class AgentRegistrationService {
             else {
                 this.logger.warn('Registration prompt delivery returned false', { sessionName, role, runtimeType });
             }
+            // Make a silent non-registration loud, and re-deliver once. Previously a
+            // swallowed instruction left the agent at `started` / readyAt null forever
+            // with no log line saying so.
+            await this.redeliverIfUnregistered(sessionName, role, prompt, runtimeType, controller.signal);
         }
         catch (error) {
             if (controller.signal.aborted) {
@@ -1077,6 +1143,134 @@ export class AgentRegistrationService {
             this.registrationAbortControllers.delete(sessionName);
         }
     }
+    /**
+     * Poll the PTY screen until the runtime is idle at its input prompt
+     * (`RuntimeAgentService.isReadyForInput`) or the timeout elapses.
+     *
+     * On timeout we log a warning and return `false` but the caller still
+     * sends the instruction — a late instruction is recoverable (re-delivery,
+     * `POST /api/terminal/:s/deliver`), a never-sent one is not.
+     *
+     * @param sessionName - PTY session name
+     * @param runtimeType - Runtime whose prompt heuristics to use
+     * @param abortSignal - Aborts the wait when the runtime exits
+     * @param timeoutMs - Maximum time to wait (defaults to RUNTIME_INPUT_READY_TIMEOUT_MS)
+     * @returns true when the idle prompt was observed, false on timeout/abort
+     */
+    async waitForRuntimeInputReady(sessionName, runtimeType, abortSignal, timeoutMs = REGISTRATION_DELIVERY_CONSTANTS.RUNTIME_INPUT_READY_TIMEOUT_MS) {
+        const runtimeService = this.createRuntimeService(runtimeType);
+        // Mocked / non-TUI runtimes without the predicate: nothing to wait for.
+        if (typeof runtimeService.isReadyForInput !== 'function') {
+            return true;
+        }
+        const sessionHelper = await this.getSessionHelper();
+        const pollMs = this.testAwareInterval(REGISTRATION_DELIVERY_CONSTANTS.RUNTIME_INPUT_READY_POLL_MS);
+        const startedAt = Date.now();
+        let polls = 0;
+        while (!abortSignal?.aborted) {
+            let screen = '';
+            try {
+                screen = sessionHelper.capturePane(sessionName);
+            }
+            catch (captureError) {
+                this.logger.debug('Runtime input-ready capture failed (will retry)', {
+                    sessionName,
+                    error: captureError instanceof Error ? captureError.message : String(captureError),
+                });
+            }
+            polls++;
+            if (runtimeService.isReadyForInput(screen)) {
+                this.logger.info('Runtime idle at input prompt — delivering registration instruction', {
+                    sessionName,
+                    runtimeType,
+                    waitedMs: Date.now() - startedAt,
+                    polls,
+                });
+                return true;
+            }
+            if (Date.now() - startedAt >= timeoutMs) {
+                const lastLines = screen.split('\n').filter((l) => l.trim()).slice(-5);
+                this.logger.warn('Timed out waiting for runtime input prompt — sending registration instruction anyway', {
+                    sessionName,
+                    runtimeType,
+                    timeoutMs,
+                    polls,
+                    lastTerminalLines: lastLines,
+                });
+                return false;
+            }
+            await delay(pollMs);
+        }
+        this.logger.info('Runtime input-ready wait aborted (runtime exited)', { sessionName });
+        return false;
+    }
+    /**
+     * Resolve how long an agent gets to register after its prompt is delivered.
+     * Mirrors the timeout selection used when the session is created.
+     *
+     * @param role - Agent role
+     * @param runtimeType - Agent runtime
+     * @returns Registration timeout in milliseconds
+     */
+    resolveRegistrationTimeout(role, runtimeType) {
+        if (role === ORCHESTRATOR_ROLE)
+            return AGENT_TIMEOUTS.ORCHESTRATOR_INITIALIZATION;
+        if (runtimeType === RUNTIME_TYPES.CLAUDE_CODE)
+            return AGENT_TIMEOUTS.CLAUDE_CODE_INITIALIZATION;
+        return AGENT_TIMEOUTS.REGULAR_AGENT_INITIALIZATION;
+    }
+    /**
+     * After the registration instruction was delivered, wait the registration
+     * timeout for the agent to flip to `active`. If it does not, log an error
+     * (this used to be completely silent) and re-deliver the instruction once,
+     * gated again on the runtime being idle at its prompt.
+     *
+     * @param sessionName - PTY session name
+     * @param role - Agent role
+     * @param prompt - The full registration prompt (re-written to the prompt file)
+     * @param runtimeType - Agent runtime
+     * @param abortSignal - Aborts when the runtime exits
+     */
+    async redeliverIfUnregistered(sessionName, role, prompt, runtimeType, abortSignal) {
+        const timeout = this.resolveRegistrationTimeout(role, runtimeType);
+        const checkInterval = this.testAwareInterval(REGISTRATION_DELIVERY_CONSTANTS.REGISTRATION_CHECK_INTERVAL_MS);
+        for (let redelivery = 0; redelivery <= REGISTRATION_DELIVERY_CONSTANTS.MAX_REDELIVERIES; redelivery++) {
+            const startedAt = Date.now();
+            while (Date.now() - startedAt < timeout) {
+                if (abortSignal.aborted)
+                    return;
+                if (await this.checkAgentRegistration(sessionName, role)) {
+                    this.logger.info('Agent registration confirmed after prompt delivery', {
+                        sessionName, role, redeliveries: redelivery, waitedMs: Date.now() - startedAt,
+                    });
+                    return;
+                }
+                await delay(checkInterval);
+            }
+            if (abortSignal.aborted)
+                return;
+            if (redelivery >= REGISTRATION_DELIVERY_CONSTANTS.MAX_REDELIVERIES) {
+                this.logger.error('Agent never registered after registration prompt delivery', {
+                    sessionName, role, runtimeType, timeoutMs: timeout, redeliveries: redelivery,
+                    hint: `POST /api/terminal/${sessionName}/deliver {force:true} with the prompt-file instruction`,
+                });
+                return;
+            }
+            const sessionHelper = await this.getSessionHelper();
+            if (!sessionHelper.sessionExists(sessionName)) {
+                this.logger.warn('Registration re-delivery skipped — session no longer exists', { sessionName, role });
+                return;
+            }
+            this.logger.warn('No registration within timeout — re-delivering registration instruction', {
+                sessionName, role, runtimeType, timeoutMs: timeout,
+            });
+            await this.waitForRuntimeInputReady(sessionName, runtimeType, abortSignal);
+            if (abortSignal.aborted)
+                return;
+            const resent = await this.sendPromptRobustly(sessionName, prompt, runtimeType, abortSignal);
+            this.logger.info('registration re-delivered', { sessionName, role, runtimeType, resent });
+        }
+    }
     /**
      * Step 3: Kill session and recreate completely
      * Most aggressive approach - completely recreates the session from scratch
@@ -1135,13 +1329,17 @@ export class AgentRegistrationService {
         }
         // Recreate session based on role
         if (role === ORCHESTRATOR_ROLE) {
+            // Same deterministic cwd for the PTY and the runtime `cd` — this site
+            // used to `cd` into process.cwd() (the package install dir under
+            // systemd) while the PTY was created elsewhere.
+            const orchestratorCwd = projectPath || (await this.resolveOrchestratorCwd());
             await this.createOrchestratorSession({
                 sessionName,
-                projectPath: projectPath || process.cwd(),
+                projectPath: orchestratorCwd,
             });
             // Initialize runtime for orchestrator using script (always fresh start)
             const runtimeService = this.createRuntimeService(runtimeType);
-            await runtimeService.executeRuntimeInitScript(sessionName, process.cwd(), effectiveFlags, promptFilePath, agentName);
+            await runtimeService.executeRuntimeInitScript(sessionName, orchestratorCwd, effectiveFlags, promptFilePath, agentName);
             // Wait for runtime to be ready
             const checkInterval = this.getCheckInterval();
             // runtimeService3: Separate instance for orchestrator ready-state detection
@@ -1707,16 +1905,29 @@ export class AgentRegistrationService {
                         teamNormsPath,
                     };
                 }
+                // Prompt tiering: the orchestrator is re-registered on every
+                // cold start and re-reads the whole system prompt at full price.
+                // CREWLY_ORC_PROMPT_PROFILE=lite drops the planning/onboarding
+                // modules; the default stays `full`. Both tiers are assembled
+                // for the orchestrator so the operator can compare their size
+                // in the log before flipping the env.
+                const requestedProfile = role === ORCHESTRATOR_ROLE ? resolveOrcPromptProfile() : undefined;
+                if (requestedProfile)
+                    moduleConfig.promptProfile = requestedProfile;
                 const assembler = new PromptAssemblyService();
                 const { prompt: modularPrompt, report } = await assembler.assemble(moduleConfig);
                 this.logger.info('Modular prompt assembled for registration', {
                     sessionName,
                     role,
+                    profile: report.profile,
                     totalTokens: report.totalTokens,
                     moduleCount: report.moduleBreakdown.length,
                     truncatedCount: report.truncated.length,
                     modules: report.moduleBreakdown.map(m => m.name),
                 });
+                if (requestedProfile) {
+                    await this.logPromptProfileComparison(assembler, moduleConfig, report);
+                }
                 return modularPrompt;
             }
             return prompt;
@@ -2029,6 +2240,44 @@ Loop until done, blocked, or explicitly reassigned:
         this.logger.warn('Fast registration timeout', { sessionName, role, timeout });
         return false;
     }
+    /**
+     * Resolve the orchestrator's working directory deterministically:
+     * `CREWLY_ORC_CWD` env → first stored project whose path exists → CREWLY_HOME.
+     *
+     * Never the package install dir or the server's `process.cwd()`: under
+     * systemd that is `/usr/lib/node_modules/crewly`, and the orchestrator
+     * would see a different filesystem depending on which code path started it.
+     *
+     * @returns Absolute directory to spawn the orchestrator PTY in
+     */
+    async resolveOrchestratorCwd() {
+        const envCwd = process.env[ENV_CONSTANTS.CREWLY_ORC_CWD]?.trim();
+        if (envCwd) {
+            const resolved = path.resolve(envCwd);
+            this.logger.info('Orchestrator cwd resolved from CREWLY_ORC_CWD', { cwd: resolved });
+            return resolved;
+        }
+        try {
+            const projects = await this.storageService.getProjects();
+            const firstProject = projects.find((project) => project.path && existsSync(project.path));
+            if (firstProject) {
+                this.logger.info('Orchestrator cwd resolved from first assigned project', {
+                    cwd: firstProject.path,
+                    projectId: firstProject.id,
+                    projectName: firstProject.name,
+                });
+                return firstProject.path;
+            }
+        }
+        catch (error) {
+            this.logger.debug('Could not read projects for orchestrator cwd (falling back to CREWLY_HOME)', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        const home = getCrewlyHomePath();
+        this.logger.info('Orchestrator cwd resolved to CREWLY_HOME (no env override, no project)', { cwd: home });
+        return home;
+    }
     /**
      * Create orchestrator session - extracted from the original tmux service
      */
@@ -2113,7 +2362,14 @@ Loop until done, blocked, or explicitly reassigned:
      * @returns Promise with success/error information
      */
     async _createAgentSessionImpl(config) {
-        const { sessionName, role, projectPath = process.cwd(), windowName, memberId } = config;
+        const { sessionName, role, windowName, memberId } = config;
+        // The orchestrator's cwd is resolved deterministically (env > first
+        // project > CREWLY_HOME) regardless of which boot path called us — the
+        // boot path, /orchestrator/setup and the restart service used to pass
+        // three different directories (server-install finding 4).
+        const projectPath = role === ORCHESTRATOR_ROLE
+            ? await this.resolveOrchestratorCwd()
+            : (config.projectPath ?? process.cwd());
         const forceRecreate = config.forceRecreate ?? false;
         // Get runtime type from config or default to claude-code
         let runtimeType = config.runtimeType || RUNTIME_TYPES.CLAUDE_CODE;
@@ -2480,7 +2736,11 @@ Loop until done, blocked, or explicitly reassigned:
             await sessionHelper.setEnvironmentVariable(sessionName, ENV_CONSTANTS.CREWLY_PROJECT_PATH, cwdToUse);
             // #222: Set install dir so agents can resolve skill paths regardless of CWD
             await sessionHelper.setEnvironmentVariable(sessionName, ENV_CONSTANTS.CREWLY_INSTALL_DIR, this.projectRoot);
-            // Inject API keys from settings (with override chain) for all runtimes
+            // Inject API keys from settings (with override chain) for the PTY
+            // runtimes (claude-code, gemini-cli, codex-cli). crewly-agent never
+            // reaches this block — it returns from the in-process branch above —
+            // and gets its keys via CrewlyAgentExternalRuntimeService.buildChildEnv
+            // on the child's spawn environment instead.
             const settingsService = getSettingsService();
             const runtimeContext = { runtime: runtimeType };
             // Gemini key — needed by gemini-cli and crewly-agent
@@ -2512,11 +2772,7 @@ Loop until done, blocked, or explicitly reassigned:
             });
             // Use the existing unified registration system
             // #227: Claude Code gets extended timeout for PI protection evaluation
-            const timeout = role === ORCHESTRATOR_ROLE
-                ? AGENT_TIMEOUTS.ORCHESTRATOR_INITIALIZATION
-                : runtimeType === RUNTIME_TYPES.CLAUDE_CODE
-                    ? AGENT_TIMEOUTS.CLAUDE_CODE_INITIALIZATION
-                    : AGENT_TIMEOUTS.REGULAR_AGENT_INITIALIZATION;
+            const timeout = this.resolveRegistrationTimeout(role, runtimeType);
             const initResult = await this.initializeAgentWithRegistration(sessionName, role, projectPath, timeout, memberId, runtimeType, runtimeFlags, config.additionalAllowlistPaths);
             if (!initResult.success) {
                 return {
@@ -2718,9 +2974,23 @@ Loop until done, blocked, or explicitly reassigned:
                 // Process message asynchronously — don't block the caller
                 crewlyRuntime.handleMessage(message, slackMetadata)
                     .then((result) => {
+                    // Usage goes in the main log, not only the per-session buffer: this
+                    // is the line an operator greps to see what a run cost (2026-09-18).
+                    // cachedInput is the cached portion of input, so the rate is cached / input.
+                    const cachedInput = result.usage?.cachedInput ?? 0;
+                    const promptTokens = Math.max(result.usage?.input ?? 0, cachedInput);
                     this.logger.info('Crewly Agent finished processing message', {
                         sessionName, messageLength: message.length,
                         responseLength: result.text?.length ?? 0,
+                        steps: result.steps,
+                        toolCalls: result.toolCalls?.length ?? 0,
+                        finishReason: result.finishReason,
+                        usage: {
+                            input: result.usage?.input ?? 0,
+                            output: result.usage?.output ?? 0,
+                            cachedInput,
+                            cacheHitPercent: promptTokens > 0 ? Math.round((cachedInput / promptTokens) * 100) : 0,
+                        },
                     });
                     // Route response text back to the originating chat conversation.
                     // Without this, crewly-agent responses are discarded — the agent