crewly 1.20.301 → 1.20.303

package/dist/backend/backend/src/index.js

@@ -142,7 +142,10 @@ import { getRuntimeFallbackService } from './services/runtime-fallback/runtime-f
 import { getSlackAgentIdentityService } from './services/slack/slack-agent-identity.service.js';
 import { getChatV2Service } from './services/chat-v2/chat-v2.singleton.js';
 import { isOwnerStopped } from './services/agent/owner-stopped.registry.js';
-import { createFollowThrough, settleFollowThrough, idlePendingWorkCheck, releaseWorkForStop, owedWorkReminder } from './services/agent/follow-through.wiring.js';
+import { pausedTeamOfSession } from './services/team/team-pause.registry.js';
+import { getFollowThrough } from './services/agent/follow-through.service.js';
+import { BackgroundWorkWatchdog } from './services/monitoring/background-work-watchdog.js';
+import { createFollowThrough, settleFollowThrough, idlePendingWorkCheck, releaseWorkForStop, restoreStoppedOwingWork, owedWorkReminder, OWED_REMINDER_MARKER } from './services/agent/follow-through.wiring.js';
 import { isSessionPaused } from './services/team/team-pause.registry.js';
 import { findPackageRoot } from './utils/package-root.js';
 import { getLocalApiBaseUrl, setLocalApiPort } from './utils/local-api-url.utils.js';
@@ -184,6 +187,7 @@ import { sessionsToRestore } from './services/agent/restore-filter.js';
 import { ProjectMemoryService } from './services/memory/project-memory.service.js';
 import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
 import { TeamHealthWatchdogService, LiveTeamHealthDataProvider, loadTeamHealthConfig, setTeamHealthWatchdogSingleton, getTeamHealthWatchdogSingleton, } from './services/team-health/index.js';
+import { getUsageLedger } from './services/usage-ledger/usage-ledger.runtime.js';
 import { traceHarness, traceTurnActivity } from './services/trace/trace-recorder.js';
 import { inputCircuitStats, setInputCircuitOpenListener } from './services/session/input-circuit-breaker.js';
 import { looksLikeShellPrompt } from './utils/screen-excerpt.js';
@@ -350,6 +354,7 @@ export class CrewlyServer {
     /** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
     openCommitmentsAtBoot = [];
     healthMonitoringInterval = null;
+    backgroundWatchdog = null;
     constructor(config) {
         // Resolve ~ to actual home directory
         const resolveHomePath = (inputPath) => {
@@ -1297,7 +1302,14 @@ export class CrewlyServer {
         setSlackImageService(slackImageService);
         // Wire agent:idle events to thread status queue for delegation completion
         this.eventBusService.on('eventPublished', (event) => {
+            if (event.type === 'agent:busy' && event.sessionName)
+                getSlackTypingPlaceholderService()?.releaseHold(event.sessionName);
+            if ((event.type === 'agent:idle' || event.type === 'agent:busy') && event.sessionName) {
+                ResourceModeService.getInstance().noteWorkingStatus(event.sessionName, event.type === 'agent:idle');
+            }
             if (event.type === 'agent:inactive' && event.sessionName) {
+                // A runtime that exited while the agent owed work: same hook as the other stop paths.
+                void this.restoreStoppedOwing(event.sessionName, 'runtime exited');
                 this.wakeIfMessagesQueued(event.sessionName);
             }
             // A turn started: the harness's "working on it" watch may be waiting
@@ -1385,6 +1397,7 @@ export class CrewlyServer {
         // 2026-05-16: queued WIs for inactive Atlas could not get woken
         // because the floor was held by idle product/marketing agents).
         liveDataProvider.setAgentRegistrationService(this.apiController.agentRegistrationService);
+        liveDataProvider.setAgentStoppedHook((name, why) => { void this.restoreStoppedOwing(name, why); });
         // Initialize Reconciler Service (V2 — system truth recomputation)
         {
             const reconcilerLogger = LoggerService.getInstance().createComponentLogger('ReconcilerInit');
@@ -1960,6 +1973,14 @@ export class CrewlyServer {
                 crewlyHome: this.config.crewlyHome,
                 sendToAgent: (session, text) => this.apiController.agentRegistrationService.sendMessageToAgent(session, text),
             });
+            // Background subagents / forks that stopped writing are not busy work:
+            // the agent is told and no longer counted busy (2026-10-12, Pia).
+            this.backgroundWatchdog?.stop();
+            this.backgroundWatchdog = new BackgroundWorkWatchdog({
+                turnState: AgentTurnStateService.getInstance(),
+                sendToAgent: (session, text) => this.apiController.agentRegistrationService.sendMessageToAgent(session, text),
+            });
+            this.backgroundWatchdog.start();
             // An agent that owes work — a WorkItem (queued, accepted or running),
             // an assigned ticket, a promise to the owner, an unfulfilled "doing X
             // now" — is never idle-stopped. Under memory pressure it still can
@@ -1970,20 +1991,7 @@ export class CrewlyServer {
             // a slot frees, with a reminder of what it promised (the work itself is
             // already queued for it by the releaser).
             idleDetection.setOwedStopHook((sessionName) => {
-                void (async () => {
-                    const reminder = owedWorkReminder(await TaskPoolService.getInstance().getAllItems(), sessionName);
-                    if (!reminder)
-                        return;
-                    SubAgentMessageQueue.getInstance().enqueue(sessionName, reminder);
-                    this.logger.info('Stopped agent owes work — it will be restarted when a slot frees', { sessionName });
-                    const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
-                    const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
-                    await scheduleDeferredStartRetry(sessionName, {
-                        activate: (name) => activateAgentBySession(this.apiController, name),
-                        hasPending: (name) => SubAgentMessageQueue.getInstance().hasPending(name),
-                        firstPauseMs: 60_000,
-                    });
-                })().catch(() => undefined);
+                void this.restoreStoppedOwing(sessionName, 'idle stop under memory pressure');
             });
             idleDetection.start();
             // Adaptive agent limits (pressure mode cap + start queue).
@@ -2031,12 +2039,21 @@ export class CrewlyServer {
                 // An agent mid-conversation with the owner is the last one stopped
                 // for a slot, and its owner thread hears when it is
                 // (specs/2026-10-08-owner-thread-sentinel.md).
-                owesOwnerThread: (name) => getOwnerThreadSentinel()?.owesRecently(name) === true,
-                onStoppedForSlot: (name) => reportOwnerThreadBlocking(name, { kind: 'stopped', why: 'slot' }),
+                owesOwnerThread: (name) => getOwnerThreadSentinel()?.owesOwner(name) === true || getFollowThrough()?.holdsIntent(name) === true,
+                onStoppedForSlot: (name) => {
+                    reportOwnerThreadBlocking(name, { kind: 'stopped', why: 'slot' });
+                    // Same hook as every other stop path: what it owes goes back to the queue and it is brought back.
+                    void this.restoreStoppedOwing(name, 'stopped to free a running-agent slot');
+                },
                 onStartDeferred: (name) => reportOwnerThreadBlocking(name, { kind: 'start_deferred' }),
                 stopAgent: async (name, role) => {
                     // Its work survives the stop: back in the queue, redelivered on the next start.
                     await releaseWorkForStop(name, 'stopped to free a running-agent slot');
+                    // An agent that still owes the answer keeps its "will pick this up" placeholder:
+                    // the turn-end settle the stop triggers must not call the thread answered.
+                    const owed = TaskPoolService.getInstance().getAllItems().then((items) => owedWorkReminder(items, name) !== null);
+                    if (await owed.catch(() => false))
+                        await getSlackTypingPlaceholderService()?.holdForRestart(name);
                     await agentRegistration.terminateAgentSession(name, role);
                     await StorageService.getInstance().updateAgentStatus(name, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE, 'idle_exit');
                 },
@@ -3844,6 +3861,9 @@ export class CrewlyServer {
                 const tokenUsageService = TokenUsageService.getInstance();
                 await tokenUsageService.loadFromDisk();
                 tokenUsageService.startPeriodicFlush();
+                // Per-ticket time / token accounting (specs/2026-10-11-usage-ledger.md):
+                // records facts from here on and rebuilds history once, shortly after boot.
+                await getUsageLedger().start();
                 // Read Claude Code's own session transcripts on an interval so
                 // the Usage dashboard reflects claude-code agents, and so the
                 // context monitor gets a real context size for them (Claude
@@ -6238,6 +6258,34 @@ export class CrewlyServer {
             });
         }
     }
+    /**
+     * The one hook every stop path runs: what the stopped agent owes goes back
+     * in the queue and the agent is brought back with a reminder.
+     *
+     * @param sessionName - The agent that stopped
+     * @param why - Reason text
+     * @returns True when the agent is being brought back
+     */
+    restoreStoppedOwing(sessionName, why) {
+        if (sessionName === ORCHESTRATOR_SESSION_NAME)
+            return Promise.resolve(false);
+        return restoreStoppedOwingWork(sessionName, why, {
+            listWorkItems: () => TaskPoolService.getInstance().getAllItems(),
+            isHeldBack: (name) => isOwnerStopped(name) || !!pausedTeamOfSession(name),
+            hasReminder: (name) => SubAgentMessageQueue.getInstance().peek(name).some((m) => m.data.startsWith(OWED_REMINDER_MARKER)),
+            enqueue: (name, text) => SubAgentMessageQueue.getInstance().enqueue(name, text),
+            holdPlaceholders: (name) => getSlackTypingPlaceholderService()?.holdForRestart(name) ?? Promise.resolve(0),
+            restart: async (name) => {
+                const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
+                const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
+                void scheduleDeferredStartRetry(name, {
+                    activate: (n) => activateAgentBySession(this.apiController, n),
+                    hasPending: (n) => SubAgentMessageQueue.getInstance().hasPending(n),
+                    firstPauseMs: 60_000,
+                });
+            },
+        });
+    }
     /**
      * An agent went down with messages still queued for it: start it again so
      * they are delivered (registration drains the queue). Without this the
@@ -6708,6 +6756,7 @@ export class CrewlyServer {
         }, timeoutMs);
         try {
             // Clear health monitoring interval first
+            this.backgroundWatchdog?.stop();
             if (this.healthMonitoringInterval) {
                 clearInterval(this.healthMonitoringInterval);
                 this.healthMonitoringInterval = null;
@@ -6935,6 +6984,15 @@ export class CrewlyServer {
             WorktreeJanitorService.getInstance().stop();
             // Stop auditor scheduler
             AuditorSchedulerService.getInstance().stop();
+            // Write the usage ledger's buffered buckets
+            try {
+                await getUsageLedger().stop();
+            }
+            catch (ledgerErr) {
+                this.logger.warn('Usage ledger flush on shutdown failed (non-fatal)', {
+                    error: ledgerErr instanceof Error ? ledgerErr.message : String(ledgerErr),
+                });
+            }
             // Persist the token ledger: it flushes every 5 minutes, so a restart
             // used to drop everything recorded since the last tick (2026-09-18).
             try {