crewly 1.14.1 → 1.16.0

package/dist/backend/backend/src/index.js

@@ -22,7 +22,7 @@ import { TerminalGateway, setTerminalGateway } from './websocket/terminal.gatewa
 import { initializeChatGateway } from './websocket/chat.gateway.js';
 import { LoggerService } from './services/core/logger.service.js';
 import { retryWithBackoff } from './services/core/retry.util.js';
-import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, } from './constants.js';
+import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, API_SECURITY_CONSTANTS, } from './constants.js';
 import { getSettingsService } from './services/settings/index.js';
 import { MemoryService } from './services/memory/memory.service.js';
 import { getImprovementStartupService } from './services/orchestrator/improvement-startup.service.js';
@@ -35,6 +35,13 @@ import { MessageQueueService, QueueProcessorService, ResponseRouterService } fro
 import { ThreadStatusQueueService } from './services/messaging/thread-status-queue.service.js';
 import { EventBusService } from './services/event-bus/index.js';
 import { EventToWorkItemBridge } from './services/event-bus/event-to-workitem-bridge.service.js';
+import { KRCompletionSubscriber } from './services/v3/kr-completion.subscriber.js';
+import { FallbackTriggerCleanupSubscriber } from './services/v3/fallback-trigger-cleanup.subscriber.js';
+import { MissionReminderService } from './services/v3/mission-reminder.service.js';
+import { OKRReviewService } from './services/v3/okr-review.service.js';
+import { bootEscalationService } from './services/v3/escalation-boot.js';
+import { TeamBudgetGateService } from './services/budget/team-budget-gate.service.js';
+import { HierarchyEscalationMonitor } from './services/hierarchy/hierarchy-escalation-monitor.service.js';
 import { AutoLearningSubscriber } from './services/memory/auto-learning.subscriber.js';
 import { MilestoneNotificationSubscriber } from './services/notification/milestone-notification.subscriber.js';
 import { RequestSlaSubscriber, setRequestSlaSubscriber, getRequestSlaSubscriber, } from './services/v3/request-sla.subscriber.js';
@@ -56,6 +63,9 @@ import { setMessageQueueService as setMessagingControllerQueueService } from './
 import { SystemResourceAlertService } from './services/monitoring/system-resource-alert.service.js';
 import { TokenUsageService } from './services/monitoring/token-usage.service.js';
 import { agentHeartbeatMiddleware } from './middleware/agent-heartbeat.middleware.js';
+import { apiTokenMiddleware, socketIoAllowRequest, installWebSocketGate, } from './middleware/api-token.middleware.js';
+import { resolveApiToken } from './services/core/api-token.service.js';
+import { isHeadlessEnvironment, describeNetworkExposure } from './utils/network-exposure.utils.js';
 import { RedisCacheService } from './services/cache/redis-cache.service.js';
 import { OrchestratorRestartService } from './services/orchestrator/orchestrator-restart.service.js';
 import { setOrchestratorSetupDependencies } from './services/orchestrator/orchestrator-setup.service.js';
@@ -66,6 +76,7 @@ import { OrchestratorHeartbeatMonitorService } from './services/orchestrator/orc
 import { RuntimeExitMonitorService } from './services/agent/runtime-exit-monitor.service.js';
 import { ContextWindowMonitorService } from './services/agent/context-window-monitor.service.js';
 import { OAuthReloginMonitorService } from './services/agent/oauth-relogin-monitor.service.js';
+import { getChatV2Service } from './services/chat-v2/chat-v2.singleton.js';
 import { findPackageRoot } from './utils/package-root.js';
 import { assertBuildProvenance } from './utils/build-provenance.js';
 import { isNativeBindingFatalError } from './utils/native-binding.utils.js';
@@ -146,6 +157,10 @@ export class CrewlyServer {
     eventBusService;
     /** BRIDGE-1: subscribes to autonomy events and creates WorkItems. */
     eventToWorkItemBridge = null;
+    krCompletionSubscriber = null;
+    fallbackTriggerCleanup = null;
+    escalationService = null;
+    hierarchyEscalationMonitor = null;
     /** LEARN-1: subscribes to terminal task / mission:replanned events and auto-records learnings. */
     autoLearningSubscriber = null;
     // DF-1 #438 — symmetric to AutoLearningSubscriber; surfaces milestones
@@ -188,6 +203,9 @@ export class CrewlyServer {
                 parseIntWithFallback(process.env.DEFAULT_CHECK_INTERVAL, 30, 'DEFAULT_CHECK_INTERVAL'),
             autoCommitInterval: config?.autoCommitInterval || parseIntWithFallback(process.env.AUTO_COMMIT_INTERVAL, 30, 'AUTO_COMMIT_INTERVAL'),
             headless: config?.headless ?? process.env.CREWLY_HEADLESS === 'true',
+            bindHost: config?.bindHost ||
+                process.env[API_SECURITY_CONSTANTS.ENV.BIND_HOST] ||
+                API_SECURITY_CONSTANTS.DEFAULT_BIND_HOST,
         };
         this.app = express();
         this.httpServer = createServer(this.app);
@@ -214,6 +232,10 @@ export class CrewlyServer {
             // that doesn't match /socket.io/ — killing Crewly in Chrome connections before
             // any data is exchanged (manifests as "Invalid frame header" errors).
             destroyUpgrade: false,
+            // Non-loopback clients must present the API token (query `token`
+            // or the `crewly_token` cookie). Engine.IO's polling handshake
+            // bypasses Express, so the gate has to live here as well.
+            allowRequest: socketIoAllowRequest,
         });
         this.initializeServices();
         this.configureMiddleware();
@@ -275,6 +297,15 @@ export class CrewlyServer {
         // triggers addToPool — the slack listener / TaskPool router below both
         // depend on this for the auto-close path b chain. Idempotent.
         TaskPoolService.getInstance().setEventBusService(this.eventBusService);
+        // Team budget gate (Team.budget was stored + prompt-injected but never
+        // evaluated). Enforced in claimFromPool + WorkItemDispatchSubscriber;
+        // here we give it the bus + queue so cap crossings reach the owner.
+        const teamBudgetGate = TeamBudgetGateService.getInstance();
+        teamBudgetGate.setNotifiers({
+            eventBus: this.eventBusService,
+            messageQueue: this.messageQueueService,
+        });
+        TaskPoolService.getInstance().setTeamBudgetGate(teamBudgetGate);
         // Memory: TaskHistorySubscriber listens on the bus for
         // task:done_by_worker / task:rejected / task:cancelled and writes
         // the resulting TaskHistoryEntry into ProjectMemory. This is the
@@ -444,6 +475,48 @@ export class CrewlyServer {
                     error: reflectErr.message,
                 });
             }
+            // Daily memory consolidation (self-improvement wiring): the
+            // MemoryConsolidationService existed but nothing ever called
+            // consolidate(), so no agent ever had a consolidation.json. Run it
+            // for every active member once per day, with a boot-time catch-up
+            // when the last recorded sweep (CREWLY_HOME/self-improvement-
+            // state.json) is older than the interval. Non-fatal throughout.
+            try {
+                const { ConsolidationSchedulerService } = await import('./services/ai/self-improvement/consolidation-scheduler.service.js');
+                const { createMemoryConsolidationService } = await import('./services/ai/self-improvement/agent-memory-provider.js');
+                const { SELF_IMPROVEMENT_CONSTANTS } = await import('./constants.js');
+                const posIntMs = (raw, fallback) => {
+                    const n = Number(raw);
+                    return Number.isFinite(n) && n > 0 ? n : fallback;
+                };
+                const consolidation = createMemoryConsolidationService();
+                const scheduler = new ConsolidationSchedulerService({
+                    intervalMs: posIntMs(process.env[SELF_IMPROVEMENT_CONSTANTS.CONSOLIDATION_INTERVAL_ENV], SELF_IMPROVEMENT_CONSTANTS.CONSOLIDATION_INTERVAL_MS),
+                    consolidate: (sessionName) => consolidation.consolidate(sessionName),
+                    listActiveSessions: async () => {
+                        const sessions = [];
+                        const orc = await this.storageService.getOrchestratorStatus();
+                        if (orc && orc.agentStatus === CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE)
+                            sessions.push(orc.sessionName);
+                        const teams = await this.storageService.getTeams();
+                        for (const team of teams) {
+                            for (const member of team.members ?? []) {
+                                if (member.agentStatus === CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE && member.sessionName) {
+                                    sessions.push(member.sessionName);
+                                }
+                            }
+                        }
+                        return sessions;
+                    },
+                });
+                ConsolidationSchedulerService.setInstance(scheduler);
+                scheduler.start();
+            }
+            catch (consolidationErr) {
+                this.logger.warn('Memory consolidation scheduler failed to start (non-fatal)', {
+                    error: consolidationErr.message,
+                });
+            }
             // LLM-wiki → WorkItem bridge (2026-05-27): pending wiki queue
             // items + legacy migration candidates become claimable
             // WorkItems in the V3 pool. Replaces the bookkeep/reflect
@@ -479,6 +552,10 @@ export class CrewlyServer {
             // actually delivers OR the budget is exhausted.
             try {
                 const { OrcDeliveryEnforcerService } = await import('./services/orc/orc-delivery-enforcer.service.js');
+                const [{ getChatV2Service }, { synthesizeSlackConversationId }] = await Promise.all([
+                    import('./services/chat-v2/chat-v2.singleton.js'),
+                    import('./services/chat-v2/legacy-dto.utils.js'),
+                ]);
                 const enforcer = new OrcDeliveryEnforcerService({
                     reminderSink: ({ conversationId, text }) => {
                         if (!this.messageQueueService)
@@ -489,6 +566,12 @@ export class CrewlyServer {
                             source: 'system_event',
                         });
                     },
+                    // Issue #731: let the enforcer see the thread's real activity
+                    // (owner silence → don't track; reply via any path → stop).
+                    // Every Slack thread is a chat-v2 channel keyed by its
+                    // synthesized conversation id, and every outbound Slack
+                    // reply is mirrored into it.
+                    threadActivityProvider: ({ channelId, threadTs }) => getChatV2Service().getChannelActivity(synthesizeSlackConversationId(channelId, threadTs)),
                 });
                 OrcDeliveryEnforcerService.setInstance(enforcer);
                 enforcer.start();
@@ -521,6 +604,28 @@ export class CrewlyServer {
         // for idempotency contract + retry cap + cron-recursion guard.
         this.eventToWorkItemBridge = EventToWorkItemBridge.boot(this.eventBusService);
         this.eventToWorkItemBridge.start();
+        // OKR loop closure: auto-measure `task_completion` KRs from task:done /
+        // task:verified and publish `team:all_tasks_done` when a mission has no
+        // active WorkItems left (the bridge turns that into a review WI). See
+        // `kr-completion.subscriber.ts`.
+        this.krCompletionSubscriber = KRCompletionSubscriber.boot(this.eventBusService);
+        this.krCompletionSubscriber.start();
+        // A delegation's fallback timer is cancelled the moment its WorkItem
+        // finishes, so the orchestrator stops being woken to "check" work that
+        // is already verified (26 of 65 WorkItems on steamfun-ops, 2026-09-18).
+        this.fallbackTriggerCleanup = FallbackTriggerCleanupSubscriber.boot(this.eventBusService);
+        this.fallbackTriggerCleanup.start();
+        // OKR loop closure: give the reminder sweep (mission:stale) and the
+        // review service (mission:replanned) a bus to publish on. Both were
+        // declared + bridged events with no publisher before this.
+        MissionReminderService.getInstance().setEventBusService(this.eventBusService);
+        OKRReviewService.getInstance().setEventBusService(this.eventBusService);
+        // Hierarchy escalation: a TL that has not acted on a worker's
+        // verification handoff within 15 min triggers the documented bypass to
+        // the orchestrator (`hierarchy:escalation` + [ESCALATION] queue message).
+        // HierarchyEscalationService had the rules but no runtime consumer.
+        this.hierarchyEscalationMonitor = HierarchyEscalationMonitor.boot(this.eventBusService, this.messageQueueService);
+        this.hierarchyEscalationMonitor.start();
         // LEARN-1: subscribe to terminal task / mission:replanned events and
         // auto-record a learning entry via MemoryService.recordLearning. Closes
         // the prompt-driven "agents-forget-to-record" gap. See
@@ -908,6 +1013,10 @@ export class CrewlyServer {
         // Note: Static files are configured in configureRoutes() after API routes
     }
     configureRoutes() {
+        // API token gate — loopback callers (local skills, local dashboard)
+        // pass; every other address must present the API token. `/health`,
+        // static assets and the SPA shell are outside `/api` and stay open.
+        this.app.use('/api', apiTokenMiddleware);
         // Agent heartbeat middleware - any API call with X-Agent-Session header updates heartbeat
         this.app.use('/api', agentHeartbeatMiddleware);
         // API routes
@@ -1291,12 +1400,44 @@ export class CrewlyServer {
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
-            // Wire OAuthReloginMonitorService EventBus dependency
+            // Wire OAuthReloginMonitorService: event bus, orchestrator queue,
+            // chat + Slack sinks for the owner-facing login notice, and the
+            // boot-level sign-in-screen sweep (server-install finding 7).
             try {
-                OAuthReloginMonitorService.getInstance().setEventBusService(this.eventBusService);
+                const oauthMonitor = OAuthReloginMonitorService.getInstance();
+                oauthMonitor.setEventBusService(this.eventBusService);
+                oauthMonitor.setNoticeQueue(this.messageQueueService);
+                oauthMonitor.setSlackProvider(async () => {
+                    const slack = getSlackService();
+                    return slack.isConnected() ? slack : null;
+                });
+                oauthMonitor.setChatProvider(() => {
+                    const gateway = this.terminalGateway;
+                    if (!gateway)
+                        return null;
+                    return {
+                        getActiveConversationId: () => gateway.getActiveConversationId(),
+                        recordSystemTurn: (conversationId, content) => {
+                            const chatV2 = getChatV2Service();
+                            const channel = chatV2.ensureChannelForLegacyConversation({
+                                conversationId,
+                                agentSession: ORCHESTRATOR_SESSION_NAME,
+                            });
+                            chatV2.recordTurn({
+                                channelId: channel.id,
+                                senderType: 'system',
+                                senderId: 'system',
+                                content,
+                                metadata: { source: 'system' },
+                            });
+                        },
+                        broadcastSystemNotification: (message, type) => gateway.broadcastSystemNotification(message, type),
+                    };
+                });
+                oauthMonitor.start();
             }
             catch (error) {
-                this.logger.warn('Failed to wire OAuthReloginMonitorService EventBus (non-critical)', {
+                this.logger.warn('Failed to wire OAuthReloginMonitorService (non-critical)', {
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
@@ -1832,6 +1973,15 @@ export class CrewlyServer {
                 });
                 await triggerEngine.start();
                 this.logger.info('TriggerEngine started with action handler wired');
+                // MissionPolicy escalation loop (every 5 min: cost/time/failure rules →
+                // notify/pause/block). Sequenced AFTER the action handler above because
+                // EscalationService.start() wraps the installed handler and delegates
+                // non-escalation triggers back to it. Gated by CREWLY_ESCALATION_ENABLED
+                // (default on); a no-op when no mission carries escalation rules.
+                this.escalationService = await bootEscalationService({
+                    messageQueue: this.messageQueueService,
+                    logger: this.logger,
+                });
                 // Wire team-scoped triggers: reconcile declarative Team.triggers spec
                 // against the running engine on every team save, and cancel all of a
                 // team's triggers when it's deleted. Listener is fire-and-forget.
@@ -1894,6 +2044,7 @@ export class CrewlyServer {
             try {
                 const { WorkItemDispatchSubscriber } = await import('./services/v3/workitem-dispatch.subscriber.js');
                 const dispatchSubscriber = WorkItemDispatchSubscriber.getInstance();
+                dispatchSubscriber.setTeamBudgetGate(TeamBudgetGateService.getInstance());
                 dispatchSubscriber.initialize(this.eventBusService);
                 dispatchSubscriber.start();
                 this.logger.info('WorkItemDispatchSubscriber started — workitem:queued events push to target sessions');
@@ -2067,6 +2218,12 @@ export class CrewlyServer {
                 const loadedAddons = await addonLoader.loadAddons(this.app, this.httpServer);
                 if (loadedAddons.length > 0) {
                     this.logger.info('Addons loaded successfully', { addons: loadedAddons });
+                    // An addon that attaches its own WebSocket gateway wraps
+                    // `httpServer.emit` after our gate did; re-install so the
+                    // gate is outermost again (double-wrapping is harmless —
+                    // an allowed upgrade passes both, a refused one is
+                    // answered once by the outer wrapper).
+                    installWebSocketGate(this.httpServer);
                 }
             }
             catch (addonErr) {
@@ -2669,9 +2826,9 @@ export class CrewlyServer {
         const { createServer } = await import('net');
         const testServer = createServer();
         return new Promise((resolve, reject) => {
-            testServer.listen(this.config.webPort, () => {
+            testServer.listen(this.config.webPort, this.config.bindHost, () => {
                 testServer.close(() => {
-                    this.logger.info('Port is available', { port: this.config.webPort });
+                    this.logger.info('Port is available', { port: this.config.webPort, host: this.config.bindHost });
                     resolve();
                 });
             });
@@ -2685,18 +2842,63 @@ export class CrewlyServer {
             });
         });
     }
+    /**
+     * Log where the API token lives and how reachable the API is.
+     *
+     * Resolving the token here also performs the first-boot generation so
+     * the file exists (0600) before the first non-loopback caller arrives.
+     * Emits a WARN for a headless install that binds every interface with
+     * neither `CREWLY_BIND_HOST` nor `CREWLY_API_TOKEN` set.
+     */
+    logNetworkExposure() {
+        try {
+            const token = resolveApiToken();
+            const summary = describeNetworkExposure({
+                bindHost: this.config.bindHost,
+                port: this.config.webPort,
+                bindHostExplicit: Boolean(process.env[API_SECURITY_CONSTANTS.ENV.BIND_HOST]),
+                tokenSource: token.source,
+                tokenFilePath: token.filePath,
+                headless: this.config.headless || isHeadlessEnvironment(),
+            });
+            if (token.source === 'generated') {
+                this.logger.info('Generated API token for non-loopback callers', {
+                    file: token.filePath,
+                    usage: 'crewly token | curl -H "X-Crewly-Token: $(crewly token)" http://<host>:<port>/api/teams',
+                });
+            }
+            if (summary.level === 'warn') {
+                this.logger.warn(summary.message, summary.details);
+            }
+            else {
+                this.logger.info(summary.message, summary.details);
+            }
+        }
+        catch (error) {
+            this.logger.error('Failed to resolve API token', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
     async startHttpServer() {
         return new Promise((resolve, reject) => {
             const startTime = Date.now();
-            this.httpServer.listen(this.config.webPort, () => {
+            // Outermost upgrade wrapper: every gateway (Socket.IO, browser
+            // bridge, chat-v2, terminal) has attached by now, so this gate
+            // runs before any of them sees a non-loopback upgrade.
+            installWebSocketGate(this.httpServer);
+            this.httpServer.listen(this.config.webPort, this.config.bindHost, () => {
                 const duration = Date.now() - startTime;
                 this.logger.info('Crewly server started', {
+                    host: this.config.bindHost,
                     port: this.config.webPort,
+                    listening: `${this.config.bindHost}:${this.config.webPort}`,
                     durationMs: duration,
                     dashboardUrl: `http://localhost:${this.config.webPort}`,
                     websocketUrl: `ws://localhost:${this.config.webPort}`,
                     home: this.config.crewlyHome
                 });
+                this.logNetworkExposure();
                 // B0 (interim) per `.crewly/specs/2026-05-05-trigger-persistence-bug.md`:
                 // Broadcast `system:backend_restarted` exactly once per boot. The
                 // trigger engine (`backend/src/services/v3/trigger-engine.service.ts`)
@@ -2831,7 +3033,6 @@ export class CrewlyServer {
         // Scans active missions and sends Slack alerts for off-track KRs
         setInterval(async () => {
             try {
-                const { MissionReminderService } = await import('./services/v3/mission-reminder.service.js');
                 await MissionReminderService.getInstance().runSweep();
             }
             catch (err) {
@@ -2844,7 +3045,6 @@ export class CrewlyServer {
         setTimeout(() => this.purgeCompletedData(), 30 * 1000);
         setTimeout(async () => {
             try {
-                const { MissionReminderService } = await import('./services/v3/mission-reminder.service.js');
                 await MissionReminderService.getInstance().runSweep();
             }
             catch (err) {
@@ -3187,6 +3387,25 @@ export class CrewlyServer {
                 this.eventToWorkItemBridge.stop();
                 this.eventToWorkItemBridge = null;
             }
+            if (this.krCompletionSubscriber) {
+                this.krCompletionSubscriber.stop();
+                this.krCompletionSubscriber = null;
+            }
+            if (this.fallbackTriggerCleanup) {
+                this.fallbackTriggerCleanup.stop();
+                this.fallbackTriggerCleanup = null;
+            }
+            if (this.escalationService) {
+                try {
+                    await this.escalationService.stop();
+                }
+                catch { /* best-effort */ }
+                this.escalationService = null;
+            }
+            if (this.hierarchyEscalationMonitor) {
+                this.hierarchyEscalationMonitor.stop();
+                this.hierarchyEscalationMonitor = null;
+            }
             // LEARN-1: stop the AutoLearningSubscriber on the same window as the
             // bridge so its in-flight recordLearning calls drain before the bus
             // is cleaned.
@@ -3255,6 +3474,18 @@ export class CrewlyServer {
             LogRotationService.getInstance().stop();
             // Stop auditor scheduler
             AuditorSchedulerService.getInstance().stop();
+            // Persist the token ledger: it flushes every 5 minutes, so a restart
+            // used to drop everything recorded since the last tick (2026-09-18).
+            try {
+                const ledger = TokenUsageService.getInstance();
+                ledger.stopPeriodicFlush();
+                await ledger.flushToDisk();
+            }
+            catch (flushErr) {
+                this.logger.warn('Token ledger flush on shutdown failed (non-fatal)', {
+                    error: flushErr instanceof Error ? flushErr.message : String(flushErr),
+                });
+            }
             // Flush and shutdown OpenTelemetry tracing
             try {
                 const { TracingService: TracingSvc } = await import('./services/core/tracing.service.js');