@evalops/maestro 0.10.8 → 0.10.10

package/dist/agent/agent.js

@@ -80,6 +80,9 @@
  * @module agent/agent
  */
 import { validate as uuidValidate } from "uuid";
+import { buildDiagnosticRepairFollowUpMessage, getDiagnosticDeltaFromToolResult, hasDiagnosticRepairOptOut, } from "../lsp/diagnostic-repair.js";
+import { recordMaestroEvalScored, recordMaestroSkillInvoked, recordMaestroSkillOutcome, recordMaestroToolCallAttempt, recordMaestroToolCallCompleted, } from "../telemetry/maestro-event-bus.js";
+import { createQueryProfilerFromEnv } from "../utils/checkpoint-profiler.js";
 import { createLogger } from "../utils/logger.js";
 import { describeToolActivity, describeToolDisplayName, summarizeToolBatch, summarizeToolUse, } from "../utils/tool-use-summary.js";
 import { AgentContextManager, } from "./context-manager.js";
@@ -88,6 +91,63 @@ import { convertAppMessagesToLlm, isCoreMessage } from "./custom-messages.js";
 import { chainPreprocessMessages, defaultPreprocessMessages, } from "./preprocess-messages.js";
 import { isAssistantMessage } from "./type-guards.js";
 const logger = createLogger("agent");
+function getToolEvaluationSummary(details) {
+    if (!details || typeof details !== "object" || Array.isArray(details)) {
+        return undefined;
+    }
+    const normalized = details;
+    if (!normalized.evaluation ||
+        typeof normalized.evaluation !== "object" ||
+        Array.isArray(normalized.evaluation)) {
+        return undefined;
+    }
+    const evaluation = normalized.evaluation;
+    const summary = {};
+    if (typeof evaluation.score === "number" &&
+        Number.isFinite(evaluation.score)) {
+        summary.score = evaluation.score;
+    }
+    if (typeof evaluation.threshold === "number" &&
+        Number.isFinite(evaluation.threshold)) {
+        summary.threshold = evaluation.threshold;
+    }
+    if (typeof evaluation.passed === "boolean") {
+        summary.passed = evaluation.passed;
+    }
+    if (typeof evaluation.rationale === "string" &&
+        evaluation.rationale.trim().length > 0) {
+        summary.rationale = evaluation.rationale.trim();
+    }
+    if (Array.isArray(normalized.assertions) &&
+        normalized.assertions.length > 0) {
+        summary.assertionCount = normalized.assertions.length;
+    }
+    return Object.keys(summary).length > 0 ? summary : undefined;
+}
+function isFailedToolEvaluation(summary) {
+    if (summary.passed === false) {
+        return true;
+    }
+    if (summary.passed === true) {
+        return false;
+    }
+    return (typeof summary.score === "number" &&
+        typeof summary.threshold === "number" &&
+        summary.score < summary.threshold);
+}
+function buildEvaluationFailureMessage(failure) {
+    if (failure.rationale) {
+        return failure.rationale;
+    }
+    if (typeof failure.score === "number" &&
+        typeof failure.threshold === "number") {
+        return `Eval score ${failure.score} below threshold ${failure.threshold}`;
+    }
+    if (typeof failure.score === "number") {
+        return `Eval score ${failure.score} did not pass`;
+    }
+    return "Selected skill failed evaluation";
+}
 /**
  * Default message transformer that converts app messages to LLM-ready format.
  *
@@ -163,6 +223,9 @@ function mapThinkingLevel(level) {
         case "medium":
             return "medium"; // Moderate depth
         case "high":
+            return "high"; // Thorough reasoning
+        case "ultra":
+            return "ultra"; // Provider max where supported
         case "max":
             return "high"; // Maximum reasoning (max maps to high for providers)
         default:
@@ -382,6 +445,8 @@ export class Agent {
         this.completedToolBatch = [];
         this.suppressRecoverableOverflowErrors = false;
         this.withheldRecoverableLengthMessages = [];
+        this.diagnosticRepairAttempts = new Map();
+        this.currentTurnSkillSelections = new Map();
         /** Tracks if a partial message was accepted during abort */
         this._partialAccepted = null;
         this.transport = opts.transport;
@@ -679,10 +744,13 @@ export class Agent {
      * This is useful before starting a fresh run, or after operations like
      * compaction that rewrite history while the agent is otherwise idle.
      */
-    clearTransientRunState() {
+    clearTransientRunState(options) {
         this._state.isStreaming = false;
         this._state.streamMessage = null;
         this._state.pendingToolCalls.clear();
+        if (options?.clearSessionContextCache) {
+            this.contextManager.clearSessionCache();
+        }
         this.activeToolBatchIds = null;
         this.completedToolBatch = [];
         this.promptOnlyQueue = [];
@@ -693,6 +761,8 @@ export class Agent {
             this.withheldRecoverableLengthMessages = [];
         }
         this._partialAccepted = null;
+        this.currentTurnEvaluationFailure = undefined;
+        this.currentTurnSkillSelections.clear();
     }
     setRecoverableOverflowErrorSuppression(enabled) {
         this.suppressRecoverableOverflowErrors = enabled;
@@ -1015,6 +1085,9 @@ export class Agent {
         this.withheldRecoverableLengthMessages = [];
         this.pendingMergedRecoverableTurnEnd = undefined;
         this.currentTaskBudget = undefined;
+        this.diagnosticRepairAttempts.clear();
+        this.currentTurnEvaluationFailure = undefined;
+        this.currentTurnSkillSelections.clear();
         // Note: Do NOT clear listeners - they contain the TUI subscription
         // and must persist across resets for UI updates to work
     }
@@ -1161,6 +1234,8 @@ export class Agent {
         if (this.runningPrompt) {
             throw new Error("A prompt is already in progress. Wait for it to complete before starting another.");
         }
+        const queryProfiler = createQueryProfilerFromEnv();
+        queryProfiler.checkpoint("input:received");
         this.clearTransientRunState();
         // Set up running prompt tracking
         this.runningPrompt = new Promise((resolve) => {
@@ -1186,6 +1261,9 @@ export class Agent {
         try {
             const transformedMessages = await this.messageTransformer(this._state.messages);
             const messagesToSend = this.buildProviderMessages(transformedMessages);
+            queryProfiler.checkpoint("context:loaded", {
+                messages: messagesToSend.length,
+            });
             // Determine reasoning level
             const level = this._state.thinkingLevel;
             const reasoning = this._state.model.reasoning
@@ -1209,6 +1287,9 @@ export class Agent {
                     stack: error instanceof Error ? error.stack : undefined,
                 });
             }
+            queryProfiler.checkpoint("prompt:assembled", {
+                system_bytes: systemPrompt.length,
+            });
             const runConfig = {
                 systemPrompt,
                 tools: this._state.tools,
@@ -1225,7 +1306,11 @@ export class Agent {
                 temperature: this._state.temperature,
                 topP: this._state.topP,
                 taskBudget: this.currentTaskBudget,
+                queryProfiler,
             };
+            queryProfiler.checkpoint("tools:prepared", {
+                tools: this._state.tools.length,
+            });
             for await (const event of this.transport.run(messagesToSend, // Include userMessage in messages array
             userMessage, runConfig, abortController.signal)) {
                 if (event.type === "message_start" ||
@@ -1270,10 +1355,12 @@ export class Agent {
         catch (error) {
             if (error instanceof Error && error.name === "AbortError") {
                 aborted = true;
+                queryProfiler.terminal("turn:aborted");
             }
             else {
                 this._state.error =
                     error instanceof Error ? error.message : String(error);
+                queryProfiler.terminal("turn:error");
                 throw error;
             }
         }
@@ -1294,6 +1381,10 @@ export class Agent {
             }
             const partialAccepted = this._partialAccepted;
             this._partialAccepted = null;
+            this.publishCurrentTurnSkillOutcomes({
+                aborted,
+                stopReason: lastStopReason,
+            });
             this.emit({
                 type: "agent_end",
                 messages: this._state.messages,
@@ -1318,6 +1409,8 @@ export class Agent {
         if (this._state.messages.length === 0) {
             throw new Error("No messages to continue from");
         }
+        const queryProfiler = createQueryProfilerFromEnv();
+        queryProfiler.checkpoint("input:received", { continuation: true });
         this.clearTransientRunState();
         // Set up running prompt tracking
         this.runningPrompt = new Promise((resolve) => {
@@ -1337,6 +1430,9 @@ export class Agent {
             const transformedMessages = await this.messageTransformer(this._state.messages);
             const continuationPrompt = options?.continuationPrompt?.trim();
             const providerMessages = this.buildProviderMessages(transformedMessages, continuationPrompt);
+            queryProfiler.checkpoint("context:loaded", {
+                messages: providerMessages.length,
+            });
             // Determine reasoning level
             const level = this._state.thinkingLevel;
             const reasoning = this._state.model.reasoning
@@ -1359,6 +1455,9 @@ export class Agent {
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
+            queryProfiler.checkpoint("prompt:assembled", {
+                system_bytes: systemPrompt.length,
+            });
             // Create a synthetic continuation trigger message
             // This signals to the transport that we want the model to continue responding
             const continuationMessage = {
@@ -1388,7 +1487,11 @@ export class Agent {
                 topP: this._state.topP,
                 taskBudget: this.currentTaskBudget,
                 emitUserMessageEnd: false,
+                queryProfiler,
             };
+            queryProfiler.checkpoint("tools:prepared", {
+                tools: this._state.tools.length,
+            });
             for await (const event of this.transport.run(providerMessages, continuationMessage, runConfig, abortController.signal)) {
                 if (event.type === "message_start" ||
                     event.type === "message_update" ||
@@ -1432,10 +1535,12 @@ export class Agent {
         catch (error) {
             if (error instanceof Error && error.name === "AbortError") {
                 aborted = true;
+                queryProfiler.terminal("turn:aborted");
             }
             else {
                 this._state.error =
                     error instanceof Error ? error.message : String(error);
+                queryProfiler.terminal("turn:error");
                 throw error;
             }
         }
@@ -1456,6 +1561,10 @@ export class Agent {
             }
             const partialAccepted = this._partialAccepted;
             this._partialAccepted = null;
+            this.publishCurrentTurnSkillOutcomes({
+                aborted,
+                stopReason: lastStopReason,
+            });
             this.emit({
                 type: "agent_end",
                 messages: this._state.messages,
@@ -1476,6 +1585,17 @@ export class Agent {
             displayName,
             summaryLabel,
         });
+        recordMaestroToolCallAttempt({
+            tool_call_id: event.toolCallId,
+            tool_execution_id: event.toolExecutionId,
+            prompt_metadata: this._state.promptMetadata,
+            tool_name: event.toolName,
+            safe_arguments: event.args,
+            correlation: {
+                session_id: this._state.session?.id,
+                agent_run_step_id: event.toolCallId,
+            },
+        });
         this.emitStatus(describeToolActivity(event.toolName, event.args, tool), {
             kind: "tool_execution_summary",
             toolCallId: event.toolCallId,
@@ -1515,6 +1635,50 @@ export class Agent {
             displayName: pending?.displayName,
             summaryLabel: pending?.summaryLabel,
         });
+        recordMaestroToolCallCompleted({
+            tool_call_id: event.toolCallId,
+            tool_execution_id: event.toolExecutionId,
+            prompt_metadata: this._state.promptMetadata,
+            status: event.isError
+                ? "MAESTRO_TOOL_CALL_STATUS_FAILED"
+                : "MAESTRO_TOOL_CALL_STATUS_SUCCEEDED",
+            skill_metadata: event.skillMetadata,
+            approval_request_id: event.approvalRequestId,
+            governed_outcome: event.governedOutcome,
+            error_code: event.errorCode,
+            error_message: event.isError
+                ? event.result.content
+                    .map((part) => (part.type === "text" ? part.text : ""))
+                    .filter(Boolean)
+                    .join("\n")
+                : undefined,
+            correlation: {
+                session_id: this._state.session?.id,
+                agent_run_step_id: event.toolCallId,
+            },
+        });
+        if (event.skillMetadata) {
+            this.recordCurrentTurnSkillSelection({
+                skillMetadata: event.skillMetadata,
+                toolCallId: event.toolCallId,
+                toolExecutionId: event.toolExecutionId,
+            });
+            if (event.toolName === "Skill") {
+                recordMaestroSkillInvoked({
+                    prompt_metadata: this._state.promptMetadata,
+                    skill_metadata: event.skillMetadata,
+                    tool_call_id: event.toolCallId,
+                    tool_execution_id: event.toolExecutionId,
+                    correlation: {
+                        session_id: this._state.session?.id,
+                        agent_run_step_id: event.toolCallId,
+                    },
+                    invoked_at: new Date(event.result.timestamp).toISOString(),
+                });
+            }
+        }
+        this.publishCurrentTurnEvalScores(event);
+        this.handleDiagnosticDeltaToolResult(event);
         if (!pending) {
             return;
         }
@@ -1533,6 +1697,209 @@ export class Agent {
             this.emitToolBatchSummary();
         }
     }
+    buildSkillSelectionKey(skillMetadata) {
+        return [
+            skillMetadata.artifactId,
+            skillMetadata.source,
+            skillMetadata.name,
+            skillMetadata.version,
+            skillMetadata.hash,
+        ]
+            .filter(Boolean)
+            .join(":");
+    }
+    recordCurrentTurnSkillSelection(selection) {
+        const key = this.buildSkillSelectionKey(selection.skillMetadata);
+        if (!key) {
+            return;
+        }
+        const existing = this.currentTurnSkillSelections.get(key);
+        this.currentTurnSkillSelections.set(key, {
+            ...existing,
+            ...selection,
+            evaluationFailure: existing?.evaluationFailure,
+        });
+    }
+    getCurrentTurnSkillSelectionForEvaluation(event) {
+        if (event.skillMetadata) {
+            return this.currentTurnSkillSelections.get(this.buildSkillSelectionKey(event.skillMetadata));
+        }
+        if (this.currentTurnSkillSelections.size === 1) {
+            for (const selection of this.currentTurnSkillSelections.values()) {
+                return selection;
+            }
+        }
+        return undefined;
+    }
+    publishCurrentTurnEvalScores(event) {
+        const evaluation = getToolEvaluationSummary(event.result.details);
+        if (!evaluation || this.currentTurnSkillSelections.size === 0) {
+            return;
+        }
+        const scoredAt = new Date(event.result.timestamp).toISOString();
+        const failedEvaluation = isFailedToolEvaluation(evaluation)
+            ? {
+                toolCallId: event.toolCallId,
+                toolExecutionId: event.toolExecutionId,
+                toolName: event.toolName,
+                score: evaluation.score,
+                threshold: evaluation.threshold,
+                rationale: evaluation.rationale,
+                assertionCount: evaluation.assertionCount,
+            }
+            : undefined;
+        const evaluationTarget = failedEvaluation
+            ? this.getCurrentTurnSkillSelectionForEvaluation(event)
+            : undefined;
+        if (failedEvaluation) {
+            this.currentTurnEvaluationFailure = failedEvaluation;
+        }
+        if (failedEvaluation && evaluationTarget) {
+            evaluationTarget.evaluationFailure = failedEvaluation;
+        }
+        for (const selection of this.currentTurnSkillSelections.values()) {
+            recordMaestroEvalScored({
+                prompt_metadata: this._state.promptMetadata,
+                skill_metadata: selection.skillMetadata,
+                tool_call_id: event.toolCallId,
+                tool_execution_id: event.toolExecutionId,
+                tool_name: event.toolName,
+                score: evaluation.score,
+                threshold: evaluation.threshold,
+                passed: evaluation.passed,
+                rationale: evaluation.rationale,
+                assertion_count: evaluation.assertionCount,
+                correlation: {
+                    session_id: this._state.session?.id,
+                    agent_run_step_id: event.toolCallId,
+                },
+                scored_at: scoredAt,
+            });
+        }
+    }
+    publishCurrentTurnSkillOutcomes(options) {
+        if (this.currentTurnSkillSelections.size === 0) {
+            return;
+        }
+        const baseTurnStatus = options.aborted
+            ? "aborted"
+            : this._state.error !== undefined
+                ? "error"
+                : undefined;
+        for (const selection of this.currentTurnSkillSelections.values()) {
+            const evaluationFailure = selection.evaluationFailure;
+            const turnStatus = baseTurnStatus ??
+                (evaluationFailure || this.currentTurnEvaluationFailure
+                    ? "evaluation_failed"
+                    : "success");
+            const outcomeEvaluationFailure = evaluationFailure ?? this.currentTurnEvaluationFailure;
+            const errorMessage = turnStatus === "aborted"
+                ? "Operation aborted"
+                : turnStatus === "error"
+                    ? this._state.error
+                    : turnStatus === "evaluation_failed" && outcomeEvaluationFailure
+                        ? buildEvaluationFailureMessage(outcomeEvaluationFailure)
+                        : undefined;
+            recordMaestroSkillOutcome({
+                prompt_metadata: this._state.promptMetadata,
+                skill_metadata: selection.skillMetadata,
+                tool_call_id: selection.toolCallId,
+                tool_execution_id: selection.toolExecutionId,
+                turn_status: turnStatus,
+                error_category: turnStatus === "success"
+                    ? undefined
+                    : turnStatus === "evaluation_failed"
+                        ? "evaluation"
+                        : "runtime",
+                error_message: errorMessage,
+                evaluation_tool_name: evaluationFailure?.toolName,
+                evaluation_tool_call_id: evaluationFailure?.toolCallId,
+                evaluation_tool_execution_id: evaluationFailure?.toolExecutionId,
+                evaluation_score: evaluationFailure?.score,
+                evaluation_threshold: evaluationFailure?.threshold,
+                evaluation_assertion_count: evaluationFailure?.assertionCount,
+                evaluation_rationale: evaluationFailure?.rationale,
+                stop_reason: options.stopReason,
+                correlation: {
+                    session_id: this._state.session?.id,
+                },
+            });
+        }
+        this.currentTurnEvaluationFailure = undefined;
+        this.currentTurnSkillSelections.clear();
+    }
+    handleDiagnosticDeltaToolResult(event) {
+        const summary = getDiagnosticDeltaFromToolResult(event.result);
+        if (!summary) {
+            return;
+        }
+        if (summary.introducedCount === 0 && summary.repairedCount === 0) {
+            return;
+        }
+        const repairKey = [event.toolName, summary.file, summary.fingerprint].join(":");
+        const nextAttempt = (this.diagnosticRepairAttempts.get(repairKey) ?? 0) + 1;
+        const hasPendingUserInput = this.steeringQueue.length > 0 || this.followUpQueue.length > 0;
+        const hasOptOut = hasDiagnosticRepairOptOut(this._state.messages);
+        const canAutoFollowUp = !event.isError &&
+            summary.repair.shouldFollowUp &&
+            !hasPendingUserInput &&
+            !hasOptOut &&
+            nextAttempt <= summary.repair.maxAttempts;
+        const reason = !summary.repair.shouldFollowUp
+            ? summary.repair.reason
+            : event.isError
+                ? "Tool call failed; leaving diagnostics in the tool result."
+                : hasPendingUserInput
+                    ? "Queued user input exists; not adding an automatic diagnostic repair turn."
+                    : hasOptOut
+                        ? "User asked Maestro not to continue or repair automatically."
+                        : nextAttempt > summary.repair.maxAttempts
+                            ? "Automatic diagnostic repair attempt limit reached."
+                            : "Queued a bounded diagnostic repair follow-up.";
+        if (summary.repair.shouldFollowUp) {
+            this.diagnosticRepairAttempts.set(repairKey, nextAttempt);
+        }
+        this.emit({
+            type: "diagnostic_delta",
+            toolCallId: event.toolCallId,
+            toolName: event.toolName,
+            file: summary.file,
+            displayPath: summary.displayPath,
+            usedDelta: summary.usedDelta,
+            introducedCount: summary.introducedCount,
+            repairedCount: summary.repairedCount,
+            remainingCount: summary.remainingCount,
+            fingerprint: summary.fingerprint,
+            repairAttempt: summary.repair.shouldFollowUp ? nextAttempt : undefined,
+            maxRepairAttempts: summary.repair.shouldFollowUp
+                ? summary.repair.maxAttempts
+                : undefined,
+            willAutoFollowUp: canAutoFollowUp,
+            reason,
+        });
+        if (!canAutoFollowUp) {
+            return;
+        }
+        this.enqueueFollowUpMessageSync(buildDiagnosticRepairFollowUpMessage({
+            summary,
+            toolName: event.toolName,
+            toolCallId: event.toolCallId,
+            attempt: nextAttempt,
+        }), "front");
+    }
+    enqueueFollowUpMessageSync(message, position = "back") {
+        const queued = {
+            id: this.nextQueuedMessageId++,
+            createdAt: Date.now(),
+            original: message,
+            llm: message,
+        };
+        if (position === "front") {
+            this.followUpQueue.unshift(queued);
+            return;
+        }
+        this.followUpQueue.push(queued);
+    }
     primeToolBatch(message) {
         if (message.role !== "assistant" || message.stopReason !== "toolUse") {
             return;