@mxf-dev/core 1.0.1 → 2.0.0

package/src/services/TaskEffectivenessService.ts

@@ -55,8 +55,10 @@ export class TaskEffectivenessService {
     private readonly taskDefinitions = new Map<string, TaskDefinition>();
     private readonly taskEvents = new Map<string, TaskExecutionEvent[]>();
     
-    // Historical data (in production, this would be in MongoDB)
-    private readonly completedTasks = new Map<string, TaskEffectivenessMetrics>();
+    // Completed tasks live in MongoDB (TaskEffectivenessModel), not in memory.
+    // They were previously kept in a `completedTasks` Map, so all effectiveness
+    // history vanished on restart and any ranking built from it only ever saw
+    // the tasks this process happened to run.
     
     // Configuration
     private config: EffectivenessConfig = {
@@ -213,7 +215,9 @@ export class TaskEffectivenessService {
         updates: Partial<TaskEffectivenessMetrics['quality']>,
         agentId?: AgentId
     ): void {
-        const metrics = this.activeTasks.get(taskId) || this.completedTasks.get(taskId);
+        // Quality is recorded while a task is running. A completed task's metrics
+        // are already persisted and are not amended here.
+        const metrics = this.activeTasks.get(taskId);
         if (metrics) {
             metrics.quality = { ...metrics.quality, ...updates };
             
@@ -261,12 +265,10 @@ export class TaskEffectivenessService {
             metrics.collaboration.collaborationScore = this.calculateCollaborationScore(metrics);
         }
         
-        // Move to completed
-        this.activeTasks.delete(taskId);
-        this.completedTasks.set(taskId, metrics);
-        
-        // Persist to MongoDB
+        // Persist to MongoDB, then drop from the active set. Persist first: if
+        // it throws, the task stays active rather than disappearing entirely.
         await this.persistTaskToDB(taskId, metrics);
+        this.activeTasks.delete(taskId);
         
         // Emit completion event
         EventBus.server.emit(Events.Analytics.TASK_COMPLETED, createBaseEventPayload(
@@ -285,12 +287,25 @@ export class TaskEffectivenessService {
     }
 
     /**
-     * Get comparison with baseline
+     * Compare a task against its baseline.
+     *
+     * Reads a running task from memory and a finished one from MongoDB, so this
+     * keeps working across a restart.
+     *
+     * @returns null when the task or its definition is unknown
      */
-    public compareWithBaseline(taskId: string): EffectivenessComparison | null {
-        const metrics = this.completedTasks.get(taskId) || this.activeTasks.get(taskId);
-        const definition = this.taskDefinitions.get(taskId);
-        
+    public async compareWithBaseline(taskId: string): Promise<EffectivenessComparison | null> {
+        let metrics = this.activeTasks.get(taskId);
+        let definition = this.taskDefinitions.get(taskId);
+
+        if (!metrics || !definition) {
+            const taskDoc = await TaskEffectivenessModel.findOne({ taskId }).exec();
+            if (taskDoc) {
+                metrics = metrics ?? (taskDoc.metrics as TaskEffectivenessMetrics);
+                definition = definition ?? (taskDoc.definition as TaskDefinition);
+            }
+        }
+
         if (!metrics || !definition) {
             return null;
         }
@@ -341,99 +356,82 @@ export class TaskEffectivenessService {
     }
 
     /**
-     * Get analytics for a time period
+     * Effectiveness analytics for a time period, computed from persisted tasks.
+     *
+     * Reads from MongoDB, so results cover every task ever completed — not just
+     * the ones this process happened to run.
+     *
+     * @param startTime Start of the window (epoch ms)
+     * @param endTime End of the window (epoch ms)
+     * @param channelId Optional channel filter
+     * @param taskType Optional task-type filter
+     * @throws If the query fails. Effectiveness numbers computed from an empty
+     *         result set would look like real "no activity" data.
      */
-    public getAnalytics(
+    public async getAnalytics(
         startTime: number,
         endTime: number,
-        channelId?: ChannelId
-    ): EffectivenessAnalytics {
-        const relevantTasks = Array.from(this.completedTasks.values()).filter(task => {
-            const inTimeRange = task.metadata.startTime >= startTime && 
-                               task.metadata.startTime <= endTime;
-            const inChannel = !channelId || 
-                             this.taskDefinitions.get(task.taskId)?.channelId === channelId;
-            return inTimeRange && inChannel;
-        });
-        
-        // Initialize analytics
-        const analytics: EffectivenessAnalytics = {
-            period: { start: startTime, end: endTime },
-            byTaskType: {},
-            byChannel: {},
-            trends: {
-                effectivenessOverTime: [],
-                improving: [],
-                declining: []
-            },
-            patterns: {
-                highPerformanceTasks: [],
-                lowPerformanceTasks: [],
-                effectiveTeams: []
-            }
+        channelId?: ChannelId,
+        taskType?: string
+    ): Promise<EffectivenessAnalytics> {
+        const tasks = await this.findCompletedTasks({ startTime, endTime, channelId, taskType });
+
+        return this.computeAnalyticsFromMetrics(tasks, startTime, endTime);
+    }
+
+    /**
+     * Query persisted task effectiveness records.
+     *
+     * @throws If the query fails
+     */
+    private async findCompletedTasks(filter: {
+        startTime: number;
+        endTime: number;
+        channelId?: ChannelId;
+        taskType?: string;
+        agentId?: AgentId;
+    }): Promise<TaskEffectivenessMetrics[]> {
+        const query: Record<string, unknown> = {
+            'metrics.metadata.startTime': { $gte: filter.startTime, $lte: filter.endTime }
         };
-        
-        // Aggregate by task type
-        for (const task of relevantTasks) {
-            const type = task.metadata.type;
-            if (!analytics.byTaskType[type]) {
-                analytics.byTaskType[type] = {
-                    count: 0,
-                    avgCompletionTime: 0,
-                    successRate: 0,
-                    avgAutonomyScore: 0,
-                    commonTools: []
-                };
-            }
-            
-            const typeStats = analytics.byTaskType[type];
-            typeStats.count++;
-            typeStats.avgCompletionTime = 
-                (typeStats.avgCompletionTime * (typeStats.count - 1) + 
-                 (task.performance.completionTime || 0)) / typeStats.count;
-            typeStats.avgAutonomyScore = 
-                (typeStats.avgAutonomyScore * (typeStats.count - 1) + 
-                 task.performance.autonomyScore) / typeStats.count;
-            
-            // Track tools
-            for (const tool of task.performance.uniqueTools) {
-                if (!typeStats.commonTools.includes(tool)) {
-                    typeStats.commonTools.push(tool);
-                }
-            }
+
+        if (filter.channelId) {
+            query.channelId = filter.channelId;
         }
-        
-        // Calculate success rates
-        for (const [type, stats] of Object.entries(analytics.byTaskType)) {
-            const typeTasks = relevantTasks.filter(t => t.metadata.type === type);
-            const successful = typeTasks.filter(t => t.quality.goalAchieved).length;
-            stats.successRate = typeTasks.length > 0 ? successful / typeTasks.length : 0;
+
+        if (filter.taskType) {
+            query['metrics.metadata.type'] = filter.taskType;
+        }
+
+        if (filter.agentId) {
+            query.agentIds = filter.agentId;
+        }
+
+        try {
+            const docs = await TaskEffectivenessModel.find(query).exec();
+            return docs.map(doc => doc.metrics as TaskEffectivenessMetrics);
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            this.logger.error(`Failed to query task effectiveness data: ${message}`);
+            throw new Error(`Failed to query task effectiveness data: ${message}`);
         }
-        
-        // Identify patterns
-        analytics.patterns.highPerformanceTasks = Object.entries(analytics.byTaskType)
-            .filter(([_, stats]) => stats.successRate > 0.8 && stats.avgAutonomyScore > 0.7)
-            .map(([type]) => type);
-            
-        analytics.patterns.lowPerformanceTasks = Object.entries(analytics.byTaskType)
-            .filter(([_, stats]) => stats.successRate < 0.5 || stats.avgAutonomyScore < 0.3)
-            .map(([type]) => type);
-        
-        return analytics;
     }
 
     /**
-     * Persist task effectiveness data to MongoDB
+     * Persist task effectiveness data to MongoDB.
+     *
+     * @throws If the task has no definition, or the write fails. A completed task
+     *         that is not written down is history that is simply gone; the caller
+     *         needs to know.
      */
     private async persistTaskToDB(taskId: string, metrics: TaskEffectivenessMetrics): Promise<void> {
-        try {
-            const definition = this.taskDefinitions.get(taskId);
-            if (!definition) {
-                this.logger.warn(`No definition found for task ${taskId}, skipping DB persistence`);
-                return;
-            }
+        const definition = this.taskDefinitions.get(taskId);
+        if (!definition) {
+            throw new Error(`Cannot persist task ${taskId}: no task definition was registered for it`);
+        }
 
-            const taskDoc = await TaskEffectivenessModel.findOneAndUpdate(
+        try {
+            await TaskEffectivenessModel.findOneAndUpdate(
                 { taskId },
                 {
                     $set: {
@@ -445,67 +443,11 @@ export class TaskEffectivenessService {
                     }
                 },
                 { upsert: true, new: true }
-            );
-
+            ).exec();
         } catch (error) {
-            this.logger.error(`Failed to persist task effectiveness data: ${error}`);
-        }
-    }
-
-    /**
-     * Load task effectiveness data from MongoDB
-     */
-    private async loadTaskFromDB(taskId: string): Promise<TaskEffectivenessMetrics | null> {
-        try {
-            const taskDoc = await TaskEffectivenessModel.findOne({ taskId });
-            if (!taskDoc) {
-                return null;
-            }
-
-            // Store in caches
-            this.completedTasks.set(taskId, taskDoc.metrics);
-            this.taskDefinitions.set(taskId, taskDoc.definition);
-
-            return taskDoc.metrics;
-        } catch (error) {
-            this.logger.error(`Failed to load task effectiveness data: ${error}`);
-            return null;
-        }
-    }
-
-    /**
-     * Enhanced analytics with MongoDB queries
-     */
-    public async getEnhancedAnalytics(
-        startTime: number,
-        endTime: number,
-        channelId?: ChannelId,
-        taskType?: string
-    ): Promise<EffectivenessAnalytics> {
-        try {
-            const query: any = {
-                'metrics.metadata.startTime': { $gte: startTime, $lte: endTime }
-            };
-
-            if (channelId) {
-                query.channelId = channelId;
-            }
-
-            if (taskType) {
-                query['metrics.metadata.type'] = taskType;
-            }
-
-            const tasks = await TaskEffectivenessModel.find(query).exec();
-            
-            // Convert to in-memory format and use existing analytics logic
-            const taskMetrics = tasks.map(task => task.metrics);
-            
-            // Use existing analytics logic but with MongoDB data
-            return this.computeAnalyticsFromMetrics(taskMetrics, startTime, endTime);
-        } catch (error) {
-            this.logger.error(`Failed to get enhanced analytics: ${error}`);
-            // Fallback to in-memory analytics
-            return this.getAnalytics(startTime, endTime, channelId);
+            const message = error instanceof Error ? error.message : String(error);
+            this.logger.error(`Failed to persist task effectiveness data for ${taskId}: ${message}`);
+            throw new Error(`Failed to persist task effectiveness data for ${taskId}: ${message}`);
         }
     }
 
@@ -820,29 +762,20 @@ export class TaskEffectivenessService {
      * Get task metrics by ID
      */
     public async getTaskMetrics(taskId: string): Promise<TaskEffectivenessMetrics | null> {
-        // Check active tasks first
+        // A running task lives in memory; a finished one lives in MongoDB.
         const activeTask = this.activeTasks.get(taskId);
         if (activeTask) {
             return activeTask;
         }
-        
-        // Check completed tasks
-        const completedTask = this.completedTasks.get(taskId);
-        if (completedTask) {
-            return completedTask;
-        }
-        
-        // Try to load from MongoDB
+
         try {
-            const taskDoc = await TaskEffectivenessModel.findOne({ taskId });
-            if (taskDoc) {
-                return taskDoc.metrics as TaskEffectivenessMetrics;
-            }
+            const taskDoc = await TaskEffectivenessModel.findOne({ taskId }).exec();
+            return taskDoc ? (taskDoc.metrics as TaskEffectivenessMetrics) : null;
         } catch (error) {
-            this.logger.error(`Error loading task from DB: ${error}`);
+            const message = error instanceof Error ? error.message : String(error);
+            this.logger.error(`Failed to load task ${taskId}: ${message}`);
+            throw new Error(`Failed to load task ${taskId}: ${message}`);
         }
-        
-        return null;
     }
 
     /**
@@ -856,17 +789,18 @@ export class TaskEffectivenessService {
     ): Promise<any> {
         const dataPoints: any[] = [];
         const intervals = Math.floor((endTime - startTime) / intervalMs);
-        
+
+        // One query for the whole window, then bucket in memory.
+        const tasks = await this.findCompletedTasks({ startTime, endTime, channelId });
+
         for (let i = 0; i < intervals; i++) {
             const intervalStart = startTime + (i * intervalMs);
             const intervalEnd = intervalStart + intervalMs;
-            
-            const intervalTasks = Array.from(this.completedTasks.values()).filter(task => {
-                const taskChannel = this.taskDefinitions.get(task.taskId)?.channelId;
-                return task.metadata.startTime >= intervalStart &&
-                       task.metadata.startTime < intervalEnd &&
-                       (!channelId || taskChannel === channelId);
-            });
+
+            const intervalTasks = tasks.filter(task =>
+                task.metadata.startTime >= intervalStart &&
+                task.metadata.startTime < intervalEnd
+            );
             
             if (intervalTasks.length > 0) {
                 const avgScore = intervalTasks.reduce((sum, task) => 
@@ -904,15 +838,8 @@ export class TaskEffectivenessService {
         endTime: number,
         channelId?: ChannelId
     ): Promise<any> {
-        const agentTasks = Array.from(this.completedTasks.values()).filter(task => {
-            const taskChannel = this.taskDefinitions.get(task.taskId)?.channelId;
-            const hasAgent = task.collaboration.participatingAgents.includes(agentId);
-            
-            return hasAgent &&
-                   task.metadata.startTime >= startTime &&
-                   task.metadata.startTime <= endTime &&
-                   (!channelId || taskChannel === channelId);
-        });
+        // The agentIds index on the document does the channel/time/agent filtering.
+        const agentTasks = await this.findCompletedTasks({ startTime, endTime, channelId, agentId });
         
         const totalTasks = agentTasks.length;
         const successfulTasks = agentTasks.filter(t => t.quality.goalAchieved).length;

package/src/services/WebSearchService.ts

@@ -25,7 +25,8 @@
 
 import { BrowserManager, BrowserConfig } from './BrowserManager.js';
 import { ContentProcessor, OptimizationOptions } from './ContentProcessor.js';
-import { Page } from 'puppeteer';
+// Type-only: erased at compile time, so puppeteer stays an optional peer dependency.
+import type { Page } from 'puppeteer';
 
 export interface WebSearchArgs {
     query: string;

package/src/services/dag/TaskDagService.ts

@@ -163,6 +163,8 @@ export class TaskDagService {
         this.cleanupInterval = setInterval(() => {
             this.cleanupExpiredCaches();
         }, cleanupIntervalMs);
+        // Cache expiry sweep — must not keep the event loop alive on its own.
+        this.cleanupInterval.unref();
     }
 
     /**

package/src/services/orpar-memory/OrparMemoryCoordinator.ts

@@ -211,6 +211,8 @@ export class OrparMemoryCoordinator {
         this.cleanupIntervalId = setInterval(() => {
             this.cleanupStaleCycles();
         }, OrparMemoryCoordinator.CLEANUP_INTERVAL_MS);
+        // Background sweep — must not keep the event loop alive on its own.
+        this.cleanupIntervalId.unref();
 
         this.logger.debug('[OrparMemoryCoordinator] Started stale cycle cleanup interval');
     }
@@ -323,6 +325,20 @@ export class OrparMemoryCoordinator {
             this.onOrparToolEvent('reflection', payload);
         });
 
+        // Fill phase memory usage from actual retrievals.
+        //
+        // PhaseStrataRouter already emits every phase retrieval with the memory ids it
+        // returned, but nothing consumed it: the only way to populate phaseUsage was the
+        // public recordMemoryUsage(), which no production code ever called. phaseUsage was
+        // therefore always empty, so PhaseWeightedRewarder returned no rewards and cycle
+        // consolidation iterated zero memories. Subscribing here is what makes
+        // phase-weighted rewards real.
+        EventBus.server.on(OrparMemoryEvents.PHASE_MEMORY_RETRIEVED, (payload: any) => {
+            if (this.enabled) {
+                this.onPhaseMemoryRetrieved(payload);
+            }
+        });
+
         // Listen for task completion to retroactively update rewards when
         // the ORPAR cycle completed before task_complete was called
         EventBus.server.on(Events.Task.COMPLETED, (payload: any) => {
@@ -331,7 +347,7 @@ export class OrparMemoryCoordinator {
             }
         });
 
-        this.logger.debug('[OrparMemoryCoordinator] Event listeners registered for ControlLoop, OrparTools, and TaskEvents');
+        this.logger.debug('[OrparMemoryCoordinator] Event listeners registered for ControlLoop, OrparTools, PhaseMemory, and TaskEvents');
     }
 
     /**
@@ -341,11 +357,23 @@ export class OrparMemoryCoordinator {
     private onPhaseEvent(phase: OrparPhase, payload: any, source: string = 'controlLoop'): void {
         if (!this.enabled) return;
 
-        const { agentId, channelId } = payload.data || payload;
-        if (!agentId) return;
+        // agentId/channelId are top-level fields on BaseEventPayload, and loopId lives in
+        // `data` (ControlLoopSpecificData). This previously read both from the wrong
+        // level — `payload.data || payload` resolved to `data`, which carries no agentId —
+        // so every control-loop phase event hit the guard below and returned, and no
+        // server-side ORPAR cycle ever accumulated a phase.
+        const agentId = payload.agentId;
+        const channelId = payload.channelId;
+        if (!agentId) {
+            this.logger.warn(
+                `[OrparMemoryCoordinator] Ignoring ${phase} event from ${source}: payload has no agentId`
+            );
+            return;
+        }
 
-        // Get or create cycle state
-        const cycleId = payload.loopId || `${agentId}-${Date.now()}`;
+        // Fall back to a stable per-agent key rather than a timestamp: a timestamped key
+        // would start a fresh cycle on every event and no cycle would ever accumulate.
+        const cycleId = payload.data?.loopId || `${agentId}:${channelId ?? 'default'}`;
 
         // Fix #2: Acquire lock before modifying cycle state
         if (!this.acquireCycleLock(cycleId)) {
@@ -510,14 +538,28 @@ export class OrparMemoryCoordinator {
         const cycleState = this.activeCycles.get(cycleKey);
         if (!cycleState || cycleState.isComplete) return;
 
-        // Build outcome from reflection data
+        // Build the outcome from what we actually observed.
+        //
+        // This previously hardcoded qualityScore (0.8/0.4), errorCount (0), toolCallCount
+        // (5) and taskCompleted (true) and fed them into the reward calculation as though
+        // they were measurements. Rewards derived from invented numbers are worse than no
+        // rewards, so every field below is either observed or omitted.
         const reflectionData = reflectionPayload.data || {};
+
+        // Phases actually entered during this cycle — a real count, not an assumption
+        // that every cycle runs all five.
+        const phasesObserved = Object.keys(cycleState.phaseStartTimes).length;
+
         const outcome: CycleOutcome = {
             success: reflectionData.expectationsMet !== false,
-            qualityScore: reflectionData.expectationsMet !== false ? 0.8 : 0.4,
-            errorCount: 0,
-            toolCallCount: 5, // ORPAR tools count as one call per phase
-            taskCompleted: true,
+            // Only report a quality score if reflection actually produced one.
+            ...(typeof reflectionData.qualityScore === 'number'
+                ? { qualityScore: reflectionData.qualityScore }
+                : {}),
+            errorCount: Array.isArray(reflectionData.errors) ? reflectionData.errors.length : 0,
+            toolCallCount: phasesObserved,
+            // Only true when reflection says so — the coordinator cannot see task state.
+            taskCompleted: reflectionData.taskCompleted === true,
             metadata: {
                 source: 'orparTools',
                 learnings: reflectionData.learnings,
@@ -735,6 +777,40 @@ export class OrparMemoryCoordinator {
     /**
      * Record memory usage in a cycle
      */
+    /**
+     * Record a phase retrieval reported by PhaseStrataRouter.
+     *
+     * Resolves the agent's active cycle and records every returned memory against the
+     * phase it was retrieved in, so the cycle carries real usage by the time it completes.
+     */
+    private onPhaseMemoryRetrieved(payload: any): void {
+        const agentId = payload.agentId;
+        const channelId = payload.channelId;
+        const phase: OrparPhase | undefined = payload.data?.phase;
+        const memoryIds: string[] = payload.data?.memoryIds ?? [];
+
+        if (!agentId || !phase || memoryIds.length === 0) {
+            return;
+        }
+
+        // Match the cycle key used by onPhaseEvent so retrievals land on the live cycle.
+        const cycleId = payload.data?.loopId || `${agentId}:${channelId ?? 'default'}`;
+        if (!this.activeCycles.has(cycleId)) {
+            this.logger.debug(
+                `[OrparMemoryCoordinator] Phase retrieval for ${cycleId} has no active cycle; ignoring`
+            );
+            return;
+        }
+
+        for (const memoryId of memoryIds) {
+            this.recordMemoryUsage(cycleId, phase, memoryId, 'context');
+        }
+
+        this.logger.debug(
+            `[OrparMemoryCoordinator] Recorded ${memoryIds.length} memories for cycle ${cycleId} (phase=${phase})`
+        );
+    }
+
     public recordMemoryUsage(
         cycleId: string,
         phase: OrparPhase,

package/src/types/MemoryUtilityTypes.ts

@@ -448,6 +448,8 @@ export interface UtilityRetrievalOptions {
     maxResults?: number;
     /** Similarity threshold */
     similarityThreshold?: number;
+    /** Hybrid search balance passed to Meilisearch (0 = keyword, 1 = semantic) */
+    hybridRatio?: number;
     /** Include score breakdown */
     includeBreakdown?: boolean;
 }