crewly 1.11.2 → 1.11.4

package/dist/backend/backend/src/index.js

@@ -15,6 +15,7 @@ import os from 'os';
 import { fileURLToPath } from 'url';
 import { StorageService, TmuxService, SchedulerService, MessageSchedulerService, ActivityMonitorService, TeamActivityWebSocketService, TeamsJsonWatcherService, } from './services/index.js';
 import { getSessionBackend, getSessionBackendSync, getSessionStatePersistence, destroySessionBackend, PtySessionBackend, } from './services/session/index.js';
+import { RuntimePidRegistry } from './services/session/runtime-pid-registry.service.js';
 import { ApiController } from './controllers/api.controller.js';
 import { createApiRoutes } from './routes/api.routes.js';
 import { TerminalGateway, setTerminalGateway } from './websocket/terminal.gateway.js';
@@ -36,12 +37,13 @@ import { EventBusService } from './services/event-bus/index.js';
 import { EventToWorkItemBridge } from './services/event-bus/event-to-workitem-bridge.service.js';
 import { AutoLearningSubscriber } from './services/memory/auto-learning.subscriber.js';
 import { MilestoneNotificationSubscriber } from './services/notification/milestone-notification.subscriber.js';
-import { RequestSlaSubscriber, setRequestSlaSubscriber, } from './services/v3/request-sla.subscriber.js';
+import { RequestSlaSubscriber, setRequestSlaSubscriber, getRequestSlaSubscriber, } from './services/v3/request-sla.subscriber.js';
 import { RequestDecomposeSubscriber, setRequestDecomposeSubscriber, } from './services/v3/request-decompose.subscriber.js';
 import { RequestStatusUpdateSubscriber } from './services/v3/request-status-update.subscriber.js';
 import { RequestCascadeSubscriber } from './services/v3/request-cascade.subscriber.js';
 import { setRequestServiceEventBus, RequestService } from './services/v3/request.service.js';
 import { getSlackService } from './services/slack/slack.service.js';
+import { sendBootAnnouncement, isFirstBoot, markBooted } from './services/boot/boot-announce.service.js';
 import { SlackThreadStoreService, setSlackThreadStore, getSlackThreadStore } from './services/slack/slack-thread-store.service.js';
 import { GoogleChatThreadStoreService, setGchatThreadStore } from './services/messaging/gchat-thread-store.service.js';
 import { SlackImageService, setSlackImageService } from './services/slack/slack-image.service.js';
@@ -125,6 +127,8 @@ export class CrewlyServer {
     io;
     config;
     logger = LoggerService.getInstance().createComponentLogger('CrewlyServer');
+    /** Offline-replay summary from this boot, surfaced in the boot announcement. */
+    lastOfflineReplay;
     storageService;
     tmuxService;
     schedulerService;
@@ -903,6 +907,41 @@ export class CrewlyServer {
                     shadowMode: watchdog.getLastSweep()?.shadowMode ?? null,
                 }
                 : { status: 'warming', last_sweep_age_ms: -1, shadowMode: null };
+            // Orchestrator-liveness signal (issue #686). The silent 假死 symptom is
+            // "inbound user messages queue but nobody answers": no active agent AND
+            // outstanding `respond_to_user` SLA trackers. Surface it as a body block
+            // — we deliberately keep top-level status:"healthy" / HTTP 200 so a load
+            // balancer doesn't drop the node on this (the LB keys on the status code;
+            // this signal is for dashboards/monitoring to read from the body).
+            const slaSub = getRequestSlaSubscriber();
+            const pendingUserRequests = slaSub ? slaSub.getPendingUserRequestCount() : 0;
+            // "Down" — no active agent while user work is queued (issue #686).
+            const orchestratorDown = agentCount === 0 && pendingUserRequests > 0;
+            // "Up but hung" — the orchestrator process is alive yet its session keeps
+            // claiming work and never heartbeats (claims get grace-revoked in a loop).
+            // agentCount>0 hides this from the "down" check, so detect it explicitly
+            // via the claim-service hung signal (the Irissair 假死).
+            let orchestratorHung = false;
+            try {
+                orchestratorHung = TaskPoolService.getInstance()
+                    .getHungAgents()
+                    .includes(ORCHESTRATOR_SESSION_NAME);
+            }
+            catch {
+                // Task pool not ready yet — treat as not-hung.
+            }
+            const orchestratorStalled = orchestratorDown || orchestratorHung;
+            const orchestratorBlock = {
+                status: orchestratorStalled ? 'degraded' : 'ok',
+                activeAgents: agentCount,
+                pendingUserRequests,
+                hung: orchestratorHung,
+                reason: orchestratorHung
+                    ? 'orchestrator session is hung — claiming work but not heartbeating (repeated grace-revokes)'
+                    : orchestratorDown
+                        ? `no active agent with ${pendingUserRequests} pending user request(s) — orchestrator may be down`
+                        : null,
+            };
             res.json({
                 status: 'healthy',
                 timestamp: new Date().toISOString(),
@@ -916,6 +955,7 @@ export class CrewlyServer {
                     total: agentCount,
                 },
                 team_health: teamHealthBlock,
+                orchestrator: orchestratorBlock,
             });
         });
         // H5 quick entry static page (served regardless of headless mode)
@@ -1515,6 +1555,11 @@ export class CrewlyServer {
                 const { MessageReplayService } = await import('./services/messaging/message-replay.service.js');
                 const replayService = new MessageReplayService(this.messageQueueService, this.config.crewlyHome);
                 const replayResult = await replayService.replayPendingMessages();
+                // Stash for the boot announcement (surfaced after the orchestrator starts).
+                this.lastOfflineReplay = {
+                    offlineDurationMs: replayResult.offlineDurationMs,
+                    replayedCount: replayResult.replayedCount,
+                };
                 if (replayResult.replayedCount > 0) {
                     this.logger.info('Replayed pending messages from offline period (#247)', {
                         replayed: replayResult.replayedCount,
@@ -1799,6 +1844,23 @@ export class CrewlyServer {
                     error: dispatchErr instanceof Error ? dispatchErr.message : String(dispatchErr),
                 });
             }
+            // Bootstrap SOPService at boot (F8 — fix/f8-get-sops-graceful-fallback).
+            // This materialises `~/.crewly/sops/{system,custom}/` and seeds the
+            // `index.json` so the get-sops skill — which sits on every agent's
+            // session-startup hot path — never hits a missing-file 500.
+            // SOPService.initialize is internally idempotent and the service has
+            // graceful in-memory fallbacks, so this is non-critical: failures here
+            // will be tolerated by the API layer at request time.
+            try {
+                const { SOPService } = await import('./services/sop/sop.service.js');
+                await SOPService.getInstance().initialize();
+                this.logger.info('SOPService bootstrapped — get-sops endpoint ready');
+            }
+            catch (sopErr) {
+                this.logger.warn('SOPService bootstrap failed (non-critical, runtime fallback active)', {
+                    error: sopErr instanceof Error ? sopErr.message : String(sopErr),
+                });
+            }
             // Start AgentAutoClaimService — auto-assign work to idle agents
             try {
                 const { AgentAutoClaimService } = await import('./services/v3/agent-auto-claim.service.js');
@@ -1957,6 +2019,23 @@ export class CrewlyServer {
             this.registerSignalHandlers();
             // Start health monitoring
             this.startHealthMonitoring();
+            // Reap orphaned runtime processes left by a previous, non-graceful
+            // backend death (crash / OOM / force-exit) BEFORE we spawn any new
+            // sessions below — otherwise stray `gemini --yolo` / `claude` runtimes
+            // accumulate across restarts and inflate the process table (#715).
+            // Identity-verified: only kills recorded PIDs whose live cmdline still
+            // matches, so it never touches an unrelated reused PID.
+            try {
+                const reaped = RuntimePidRegistry.getInstance().reapOrphans();
+                if (reaped > 0) {
+                    this.logger.warn('Reaped orphaned runtime processes at startup', { reaped });
+                }
+            }
+            catch (err) {
+                this.logger.warn('Orphan runtime reap failed (non-fatal)', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
             // Auto-start orchestrator if enabled in settings
             await this.autoStartOrchestratorIfEnabled();
             // Auto-restore agent sessions that were running before the last shutdown
@@ -2302,6 +2381,50 @@ export class CrewlyServer {
                 this.terminalGateway.startOrchestratorChatMonitoring(ORCHESTRATOR_SESSION_NAME);
             }
             this.logger.info('Orchestrator auto-started successfully');
+            // Announce "back online" to the owner's channel (best-effort, never
+            // blocks boot). Deterministic system message — reports startup + the
+            // running version, enriched with offline duration + replayed count.
+            try {
+                const settings = await getSettingsService().getSettings();
+                if (settings.general.announceOnBoot) {
+                    let version = 'unknown';
+                    try {
+                        version = VersionCheckService.getInstance().getLocalVersion();
+                    }
+                    catch {
+                        version = process.env.npm_package_version || 'unknown';
+                    }
+                    // First-ever boot → welcome; subsequent boots → "back online".
+                    const bootMarker = path.join(this.config.crewlyHome, '.boot-announced');
+                    const firstBoot = isFirstBoot(bootMarker);
+                    if (firstBoot)
+                        markBooted(bootMarker);
+                    await sendBootAnnouncement({
+                        version,
+                        firstBoot,
+                        offlineDurationMs: this.lastOfflineReplay?.offlineDurationMs,
+                        replayedCount: this.lastOfflineReplay?.replayedCount,
+                    }, {
+                        isSlackConnected: () => getSlackService().isConnected(),
+                        sendSlack: (msg) => getSlackService().sendNotification({
+                            type: 'project_update',
+                            title: msg.title,
+                            message: msg.message,
+                            urgency: 'normal',
+                            timestamp: new Date().toISOString(),
+                        }),
+                        logger: {
+                            info: (m, meta) => this.logger.info(m, meta),
+                            warn: (m, meta) => this.logger.warn(m, meta),
+                        },
+                    });
+                }
+            }
+            catch (announceErr) {
+                this.logger.warn('Boot announce skipped (non-critical)', {
+                    error: announceErr instanceof Error ? announceErr.message : String(announceErr),
+                });
+            }
         }
         catch (error) {
             this.logger.error('Failed to auto-start orchestrator', {