crewly 1.20.302 → 1.20.304

package/dist/backend/backend/src/index.js

@@ -77,6 +77,7 @@ import { parseSlackThreadKey } from './services/slack/slack-thread-key.js';
 import { LIVENESS_MONITOR_CONSTANTS, INPUT_CIRCUIT_CONSTANTS, INPUT_BLOCKED_RETRY_CONSTANTS } from './constants.js';
 import { InputBlockedRetryService } from './services/messaging/input-blocked-retry.service.js';
 import { driveCapabilities } from './services/drive/drive-agent.service.js';
+import { teamMoveCapabilities } from './services/team-move/team-move-driver.service.js';
 import { SUB_AGENT_QUEUE_CONSTANTS, CHAT_CONTEXT_CONSTANTS, SAFE_RESTART, AUTO_UPDATE_CONSTANTS, PROCESS_EXIT_CODES, CLAUDE_STARTUP_CONSTANTS, WEB_CONSTANTS, TICKET_CONSTANTS, UNASSIGNED_ROUTE_CONSTANTS, CLOUD_TALK_CONSTANTS, STANDING_ANSWERS_CONSTANTS, TICKET_AUTOPILOT_CONSTANTS, TICKET_HYGIENE_CONSTANTS, EXPERIMENT_CONSTANTS, WORK_ITEM_DESTINATION_CONSTANTS, CODEX_USAGE_SYNC_CONSTANTS, ANTIGRAVITY_USAGE_SYNC_CONSTANTS, OWNER_AUTH_CONSTANTS, CREWLY_APPS_CONSTANTS, SLACK_AGENT_DM_CONSTANTS, BRIEFING_CONSTANTS, DRIVE_CONSTANTS, POOL_PRUNE_CONSTANTS } from './constants.js';
 import { randomUUID } from 'crypto';
 import { PtyActivityTrackerService } from './services/agent/pty-activity-tracker.service.js';
@@ -142,7 +143,10 @@ import { getRuntimeFallbackService } from './services/runtime-fallback/runtime-f
 import { getSlackAgentIdentityService } from './services/slack/slack-agent-identity.service.js';
 import { getChatV2Service } from './services/chat-v2/chat-v2.singleton.js';
 import { isOwnerStopped } from './services/agent/owner-stopped.registry.js';
-import { createFollowThrough, settleFollowThrough, idlePendingWorkCheck, releaseWorkForStop, owedWorkReminder } from './services/agent/follow-through.wiring.js';
+import { pausedTeamOfSession } from './services/team/team-pause.registry.js';
+import { getFollowThrough } from './services/agent/follow-through.service.js';
+import { BackgroundWorkWatchdog } from './services/monitoring/background-work-watchdog.js';
+import { createFollowThrough, settleFollowThrough, idlePendingWorkCheck, releaseWorkForStop, restoreStoppedOwingWork, owedWorkReminder, OWED_REMINDER_MARKER } from './services/agent/follow-through.wiring.js';
 import { isSessionPaused } from './services/team/team-pause.registry.js';
 import { findPackageRoot } from './utils/package-root.js';
 import { getLocalApiBaseUrl, setLocalApiPort } from './utils/local-api-url.utils.js';
@@ -184,6 +188,7 @@ import { sessionsToRestore } from './services/agent/restore-filter.js';
 import { ProjectMemoryService } from './services/memory/project-memory.service.js';
 import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
 import { TeamHealthWatchdogService, LiveTeamHealthDataProvider, loadTeamHealthConfig, setTeamHealthWatchdogSingleton, getTeamHealthWatchdogSingleton, } from './services/team-health/index.js';
+import { getUsageLedger } from './services/usage-ledger/usage-ledger.runtime.js';
 import { traceHarness, traceTurnActivity } from './services/trace/trace-recorder.js';
 import { inputCircuitStats, setInputCircuitOpenListener } from './services/session/input-circuit-breaker.js';
 import { looksLikeShellPrompt } from './utils/screen-excerpt.js';
@@ -350,6 +355,7 @@ export class CrewlyServer {
     /** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
     openCommitmentsAtBoot = [];
     healthMonitoringInterval = null;
+    backgroundWatchdog = null;
     constructor(config) {
         // Resolve ~ to actual home directory
         const resolveHomePath = (inputPath) => {
@@ -1297,7 +1303,14 @@ export class CrewlyServer {
         setSlackImageService(slackImageService);
         // Wire agent:idle events to thread status queue for delegation completion
         this.eventBusService.on('eventPublished', (event) => {
+            if (event.type === 'agent:busy' && event.sessionName)
+                getSlackTypingPlaceholderService()?.releaseHold(event.sessionName);
+            if ((event.type === 'agent:idle' || event.type === 'agent:busy') && event.sessionName) {
+                ResourceModeService.getInstance().noteWorkingStatus(event.sessionName, event.type === 'agent:idle');
+            }
             if (event.type === 'agent:inactive' && event.sessionName) {
+                // A runtime that exited while the agent owed work: same hook as the other stop paths.
+                void this.restoreStoppedOwing(event.sessionName, 'runtime exited');
                 this.wakeIfMessagesQueued(event.sessionName);
             }
             // A turn started: the harness's "working on it" watch may be waiting
@@ -1385,6 +1398,7 @@ export class CrewlyServer {
         // 2026-05-16: queued WIs for inactive Atlas could not get woken
         // because the floor was held by idle product/marketing agents).
         liveDataProvider.setAgentRegistrationService(this.apiController.agentRegistrationService);
+        liveDataProvider.setAgentStoppedHook((name, why) => { void this.restoreStoppedOwing(name, why); });
         // Initialize Reconciler Service (V2 — system truth recomputation)
         {
             const reconcilerLogger = LoggerService.getInstance().createComponentLogger('ReconcilerInit');
@@ -1960,6 +1974,14 @@ export class CrewlyServer {
                 crewlyHome: this.config.crewlyHome,
                 sendToAgent: (session, text) => this.apiController.agentRegistrationService.sendMessageToAgent(session, text),
             });
+            // Background subagents / forks that stopped writing are not busy work:
+            // the agent is told and no longer counted busy (2026-10-12, Pia).
+            this.backgroundWatchdog?.stop();
+            this.backgroundWatchdog = new BackgroundWorkWatchdog({
+                turnState: AgentTurnStateService.getInstance(),
+                sendToAgent: (session, text) => this.apiController.agentRegistrationService.sendMessageToAgent(session, text),
+            });
+            this.backgroundWatchdog.start();
             // An agent that owes work — a WorkItem (queued, accepted or running),
             // an assigned ticket, a promise to the owner, an unfulfilled "doing X
             // now" — is never idle-stopped. Under memory pressure it still can
@@ -1970,20 +1992,7 @@ export class CrewlyServer {
             // a slot frees, with a reminder of what it promised (the work itself is
             // already queued for it by the releaser).
             idleDetection.setOwedStopHook((sessionName) => {
-                void (async () => {
-                    const reminder = owedWorkReminder(await TaskPoolService.getInstance().getAllItems(), sessionName);
-                    if (!reminder)
-                        return;
-                    SubAgentMessageQueue.getInstance().enqueue(sessionName, reminder);
-                    this.logger.info('Stopped agent owes work — it will be restarted when a slot frees', { sessionName });
-                    const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
-                    const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
-                    await scheduleDeferredStartRetry(sessionName, {
-                        activate: (name) => activateAgentBySession(this.apiController, name),
-                        hasPending: (name) => SubAgentMessageQueue.getInstance().hasPending(name),
-                        firstPauseMs: 60_000,
-                    });
-                })().catch(() => undefined);
+                void this.restoreStoppedOwing(sessionName, 'idle stop under memory pressure');
             });
             idleDetection.start();
             // Adaptive agent limits (pressure mode cap + start queue).
@@ -2031,12 +2040,21 @@ export class CrewlyServer {
                 // An agent mid-conversation with the owner is the last one stopped
                 // for a slot, and its owner thread hears when it is
                 // (specs/2026-10-08-owner-thread-sentinel.md).
-                owesOwnerThread: (name) => getOwnerThreadSentinel()?.owesRecently(name) === true,
-                onStoppedForSlot: (name) => reportOwnerThreadBlocking(name, { kind: 'stopped', why: 'slot' }),
+                owesOwnerThread: (name) => getOwnerThreadSentinel()?.owesOwner(name) === true || getFollowThrough()?.holdsIntent(name) === true,
+                onStoppedForSlot: (name) => {
+                    reportOwnerThreadBlocking(name, { kind: 'stopped', why: 'slot' });
+                    // Same hook as every other stop path: what it owes goes back to the queue and it is brought back.
+                    void this.restoreStoppedOwing(name, 'stopped to free a running-agent slot');
+                },
                 onStartDeferred: (name) => reportOwnerThreadBlocking(name, { kind: 'start_deferred' }),
                 stopAgent: async (name, role) => {
                     // Its work survives the stop: back in the queue, redelivered on the next start.
                     await releaseWorkForStop(name, 'stopped to free a running-agent slot');
+                    // An agent that still owes the answer keeps its "will pick this up" placeholder:
+                    // the turn-end settle the stop triggers must not call the thread answered.
+                    const owed = TaskPoolService.getInstance().getAllItems().then((items) => owedWorkReminder(items, name) !== null);
+                    if (await owed.catch(() => false))
+                        await getSlackTypingPlaceholderService()?.holdForRestart(name);
                     await agentRegistration.terminateAgentSession(name, role);
                     await StorageService.getInstance().updateAgentStatus(name, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE, 'idle_exit');
                 },
@@ -2658,6 +2676,24 @@ export class CrewlyServer {
                                 error: talkErr instanceof Error ? talkErr.message : String(talkErr),
                             });
                         }
+                        // Team move (crewly#1189): Cloud tells this machine what to do in a
+                        // move of a team to or from another machine. Starting the driver is
+                        // what advertises `teamMove:1` to Cloud, and it resumes pending moves.
+                        try {
+                            const [{ createTeamMoveDriver }, { setTeamMoveDriver }] = await Promise.all([
+                                import('./services/team-move/team-move-runtime.js'),
+                                import('./services/team-move/team-move-driver.service.js'),
+                            ]);
+                            const moveDriver = createTeamMoveDriver({ storage: this.storageService, relay: sync });
+                            setTeamMoveDriver(moveDriver);
+                            moveDriver.start();
+                            void moveDriver.housekeeping().catch(() => undefined);
+                        }
+                        catch (moveErr) {
+                            this.logger.warn('Team move handler wiring skipped', {
+                                error: moveErr instanceof Error ? moveErr.message : String(moveErr),
+                            });
+                        }
                         // "Let me check with the teams" (Drive kickoff): set below, once the
                         // status snapshot sync exists, and read by the Drive handler at run time.
                         let driveStatusCheck = null;
@@ -3507,7 +3543,7 @@ export class CrewlyServer {
                         if (event.kind === 'team-saved') {
                             await reconciler.reconcile(event.team);
                         }
-                        else if (event.kind === 'team-deleted') {
+                        else if (event.kind === 'team-deleted' || event.kind === 'team-moved-away') {
                             await reconciler.unregisterAll(event.teamId);
                         }
                     });
@@ -3844,6 +3880,9 @@ export class CrewlyServer {
                 const tokenUsageService = TokenUsageService.getInstance();
                 await tokenUsageService.loadFromDisk();
                 tokenUsageService.startPeriodicFlush();
+                // Per-ticket time / token accounting (specs/2026-10-11-usage-ledger.md):
+                // records facts from here on and rebuilds history once, shortly after boot.
+                await getUsageLedger().start();
                 // Read Claude Code's own session transcripts on an interval so
                 // the Usage dashboard reflects claude-code agents, and so the
                 // context monitor gets a real context size for them (Claude
@@ -5330,7 +5369,7 @@ export class CrewlyServer {
                 // whether this machine takes `talk_message` — also for machines
                 // without Slack, which never send the registry heartbeat.
                 roster: async () => buildAgentRoster(await this.storageService.getTeams()),
-                capabilities: () => [...cloudTalkCapabilities(), ...waitingActionCapabilities(), ...talkTranscribeCapabilities(), ...driveCapabilities()],
+                capabilities: () => [...cloudTalkCapabilities(), ...waitingActionCapabilities(), ...talkTranscribeCapabilities(), ...driveCapabilities(), ...teamMoveCapabilities()],
                 onNewMessage: (listener) => {
                     chat.on('chat_message', listener);
                     return () => chat.off('chat_message', listener);
@@ -5402,7 +5441,7 @@ export class CrewlyServer {
                 agentNames: async () => new Map(buildAgentRoster(await this.storageService.getTeams())
                     .filter((a) => a.displayName)
                     .map((a) => [a.agentSession, a.displayName])),
-                capabilities: () => [...cloudTalkCapabilities(), ...waitingActionCapabilities(), ...talkTranscribeCapabilities(), ...driveCapabilities()],
+                capabilities: () => [...cloudTalkCapabilities(), ...waitingActionCapabilities(), ...talkTranscribeCapabilities(), ...driveCapabilities(), ...teamMoveCapabilities()],
                 onTicketChange: (listener) => RequestService.getInstance().onChange(() => listener()),
             });
             setWaitingItemsSyncService(service);
@@ -6238,6 +6277,34 @@ export class CrewlyServer {
             });
         }
     }
+    /**
+     * The one hook every stop path runs: what the stopped agent owes goes back
+     * in the queue and the agent is brought back with a reminder.
+     *
+     * @param sessionName - The agent that stopped
+     * @param why - Reason text
+     * @returns True when the agent is being brought back
+     */
+    restoreStoppedOwing(sessionName, why) {
+        if (sessionName === ORCHESTRATOR_SESSION_NAME)
+            return Promise.resolve(false);
+        return restoreStoppedOwingWork(sessionName, why, {
+            listWorkItems: () => TaskPoolService.getInstance().getAllItems(),
+            isHeldBack: (name) => isOwnerStopped(name) || !!pausedTeamOfSession(name),
+            hasReminder: (name) => SubAgentMessageQueue.getInstance().peek(name).some((m) => m.data.startsWith(OWED_REMINDER_MARKER)),
+            enqueue: (name, text) => SubAgentMessageQueue.getInstance().enqueue(name, text),
+            holdPlaceholders: (name) => getSlackTypingPlaceholderService()?.holdForRestart(name) ?? Promise.resolve(0),
+            restart: async (name) => {
+                const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
+                const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
+                void scheduleDeferredStartRetry(name, {
+                    activate: (n) => activateAgentBySession(this.apiController, n),
+                    hasPending: (n) => SubAgentMessageQueue.getInstance().hasPending(n),
+                    firstPauseMs: 60_000,
+                });
+            },
+        });
+    }
     /**
      * An agent went down with messages still queued for it: start it again so
      * they are delivered (registration drains the queue). Without this the
@@ -6708,6 +6775,7 @@ export class CrewlyServer {
         }, timeoutMs);
         try {
             // Clear health monitoring interval first
+            this.backgroundWatchdog?.stop();
             if (this.healthMonitoringInterval) {
                 clearInterval(this.healthMonitoringInterval);
                 this.healthMonitoringInterval = null;
@@ -6935,6 +7003,15 @@ export class CrewlyServer {
             WorktreeJanitorService.getInstance().stop();
             // Stop auditor scheduler
             AuditorSchedulerService.getInstance().stop();
+            // Write the usage ledger's buffered buckets
+            try {
+                await getUsageLedger().stop();
+            }
+            catch (ledgerErr) {
+                this.logger.warn('Usage ledger flush on shutdown failed (non-fatal)', {
+                    error: ledgerErr instanceof Error ? ledgerErr.message : String(ledgerErr),
+                });
+            }
             // Persist the token ledger: it flushes every 5 minutes, so a restart
             // used to drop everything recorded since the last tick (2026-09-18).
             try {