@axiom-lattice/core 3.0.6 → 3.0.7

package/dist/index.d.mts

@@ -5283,6 +5283,27 @@ interface LatticeAgentStepConfig {
 type OutputType = {
     type: "message_content";
 };
+/**
+ * A known-answer probe used to verify the judge agent can discriminate
+ * correct from incorrect outputs before a batch run (calibrateJudge).
+ */
+interface CalibrationProbe {
+    /** Stable identifier for error messages and reporting. */
+    id: string;
+    /** The expected output description, including the problem itself — the judge must verify by reasoning. */
+    task: string;
+    /** The output under test. */
+    finalOutput: string;
+    /** Optional execution trajectory for process verification. */
+    trajectory?: string;
+    /** Whether the judge must mark this probe as PASS. */
+    expectedPass: boolean;
+    /**
+     * Minimum acceptable final_score. Default rule when absent:
+     * expectedPass=true requires final_score >= 80, expectedPass=false requires < 80.
+     */
+    expectedScoreMin?: number;
+}
 interface LatticeEvalProjectType {
     projectName: string;
     version?: string;
@@ -5291,6 +5312,8 @@ interface LatticeEvalProjectType {
     templates?: LatticeEvalTemplate[];
     judge_agent_config: {
         modelKey: string;
+        /** Optional per-project calibration probes; defaults to DEFAULT_CALIBRATION_PROBES when absent. */
+        calibration_cases?: CalibrationProbe[];
     };
     lattice_server_config: {
         tenant_id?: string;
@@ -5666,6 +5689,12 @@ declare class LatticeEvalSuite {
     runAllCases(concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
 }
 
+/**
+ * Built-in known-answer calibration probes: multi-step reasoning problems,
+ * each paired with a correct (PASS) and a plausible-but-wrong (FAIL) output.
+ * A judge that cannot discriminate these should not be trusted with real cases.
+ */
+declare const DEFAULT_CALIBRATION_PROBES: CalibrationProbe[];
 /**
  * Manages a project with multiple evaluation suites.
  * Registers a per-project judge agent (keyed by project name and tenant)
@@ -5686,8 +5715,10 @@ declare class LatticeEvalProject {
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
     /**
      * Verify the judge agent can produce parseable, correct verdicts
-     * before committing to a full run. Uses two known-answer cases
-     * (one expected PASS, one expected FAIL) to catch broken judges.
+     * before committing to a full run. Uses known-answer probes (default
+     * multi-step reasoning set, overridable per project via
+     * judge_agent_config.calibration_cases) to catch broken judges.
+     * Checks both the pass verdict and the final_score direction.
      */
     calibrateJudge(): Promise<{
         ok: boolean;
@@ -8171,4 +8202,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CalibrationProbe, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, DEFAULT_CALIBRATION_PROBES, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };

package/dist/index.d.ts

@@ -5283,6 +5283,27 @@ interface LatticeAgentStepConfig {
 type OutputType = {
     type: "message_content";
 };
+/**
+ * A known-answer probe used to verify the judge agent can discriminate
+ * correct from incorrect outputs before a batch run (calibrateJudge).
+ */
+interface CalibrationProbe {
+    /** Stable identifier for error messages and reporting. */
+    id: string;
+    /** The expected output description, including the problem itself — the judge must verify by reasoning. */
+    task: string;
+    /** The output under test. */
+    finalOutput: string;
+    /** Optional execution trajectory for process verification. */
+    trajectory?: string;
+    /** Whether the judge must mark this probe as PASS. */
+    expectedPass: boolean;
+    /**
+     * Minimum acceptable final_score. Default rule when absent:
+     * expectedPass=true requires final_score >= 80, expectedPass=false requires < 80.
+     */
+    expectedScoreMin?: number;
+}
 interface LatticeEvalProjectType {
     projectName: string;
     version?: string;
@@ -5291,6 +5312,8 @@ interface LatticeEvalProjectType {
     templates?: LatticeEvalTemplate[];
     judge_agent_config: {
         modelKey: string;
+        /** Optional per-project calibration probes; defaults to DEFAULT_CALIBRATION_PROBES when absent. */
+        calibration_cases?: CalibrationProbe[];
     };
     lattice_server_config: {
         tenant_id?: string;
@@ -5666,6 +5689,12 @@ declare class LatticeEvalSuite {
     runAllCases(concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
 }
 
+/**
+ * Built-in known-answer calibration probes: multi-step reasoning problems,
+ * each paired with a correct (PASS) and a plausible-but-wrong (FAIL) output.
+ * A judge that cannot discriminate these should not be trusted with real cases.
+ */
+declare const DEFAULT_CALIBRATION_PROBES: CalibrationProbe[];
 /**
  * Manages a project with multiple evaluation suites.
  * Registers a per-project judge agent (keyed by project name and tenant)
@@ -5686,8 +5715,10 @@ declare class LatticeEvalProject {
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
     /**
      * Verify the judge agent can produce parseable, correct verdicts
-     * before committing to a full run. Uses two known-answer cases
-     * (one expected PASS, one expected FAIL) to catch broken judges.
+     * before committing to a full run. Uses known-answer probes (default
+     * multi-step reasoning set, overridable per project via
+     * judge_agent_config.calibration_cases) to catch broken judges.
+     * Checks both the pass verdict and the final_score direction.
      */
     calibrateJudge(): Promise<{
         ok: boolean;
@@ -8171,4 +8202,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CalibrationProbe, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, DEFAULT_CALIBRATION_PROBES, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };

package/dist/index.js

@@ -1623,6 +1623,7 @@ __export(index_exports, {
   ConsoleLoggerClient: () => ConsoleLoggerClient,
   CustomMetricsClient: () => CustomMetricsClient,
   CustomMiddlewareRegistry: () => CustomMiddlewareRegistry,
+  DEFAULT_CALIBRATION_PROBES: () => DEFAULT_CALIBRATION_PROBES,
   DaytonaInstance: () => DaytonaInstance,
   DaytonaProvider: () => DaytonaProvider,
   DefaultScheduleClient: () => DefaultScheduleClient,
@@ -27953,7 +27954,7 @@ File content: ${files[key4]}`
       {
         dimension: "correctness",
         weight: 100,
-        description: "\u6574\u4F53\u6B63\u786E\u6027\uFF0C\u662F\u5426\u7B26\u5408\u9884\u671F\u8F93\u51FA\u63CF\u8FF0\u3002"
+        description: "Overall correctness \u2014 whether the result matches the expected output description."
       }
     ];
     const evalRubrics = evalCase.eval.eval_rubrics && evalCase.eval.eval_rubrics.length > 0 ? evalCase.eval.eval_rubrics : defaultRubrics;
@@ -27967,53 +27968,53 @@ File content: ${files[key4]}`
 ${evalRubrics.map(
       (r) => `- **${r.dimension}**\uFF08\u6743\u91CD\uFF1A${r.weight}\uFF09\uFF1A${r.description}`
     ).join("\n")}`;
-    const testPrompt = `# \u89D2\u8272
-\u4F60\u662F\u4E00\u540D\u8D44\u6DF1\u7684 AI Agent \u8BC4\u4F30\u4E13\u5BB6\uFF0C\u8D1F\u8D23\u6839\u636E\u9884\u8BBE\u7684\u6307\u6807\uFF08Rubrics\uFF09\u5BF9 Agent \u7684\u6267\u884C\u8FC7\u7A0B\u4E0E\u7ED3\u679C\u8FDB\u884C"\u9ED1\u76D2\u6D4B\u8BD5"\u5224\u5B9A\u3002
+    const testPrompt = `# Role
+You are a senior AI Agent evaluation expert. Your job is to perform a "black-box test" judgment of the agent's execution process and results against the preset evaluation rubrics.
 
-# \u8F93\u5165\u4FE1\u606F
-\u6D4B\u8BD5\u6846\u67B6\u5C06\u4E3A\u4F60\u63D0\u4F9B\u4EE5\u4E0B\u4E94\u4E2A\u6838\u5FC3\u4E0A\u4E0B\u6587\uFF1A
+# Input Information
+The test framework provides you with the following five core contexts:
 
-1. **\u7528\u6237\u610F\u56FE\uFF08User Intent\uFF09**\uFF1A${evalCase.input.message}
+1. **User Intent**: ${evalCase.input.message}
 
-2. **\u8F93\u5165\u6587\u4EF6\uFF08Input Files\uFF09**\uFF1A${testCaseFilesContent || "\u65E0"}
+2. **Input Files**: ${testCaseFilesContent || "None"}
 
-3. **\u6267\u884C\u8FC7\u7A0B\uFF08Execution Trajectory\uFF0CAgent \u5168\u7A0B\u7684\u6D88\u606F/\u5DE5\u5177\u8C03\u7528\u8BB0\u5F55\uFF09**\uFF1A
+3. **Execution Trajectory** (the agent's full message/tool-call record):
 ${trajectory}
 
-4. **\u6700\u7EC8\u8F93\u51FA\uFF08Final Output\uFF0CAgent \u7684\u6700\u540E\u4E00\u6761\u56DE\u590D\uFF09**\uFF1A
+4. **Final Output** (the agent's last reply):
 ${finalOutput}
 
-5. **\u671F\u671B\u8F93\u51FA\u63CF\u8FF0\uFF08Expected Output Description\uFF09**\uFF1A${evalCase.eval.content_assertion}
+5. **Expected Output Description**: ${evalCase.eval.content_assertion}
 ${rubricsSection}
 
-# \u4EFB\u52A1
-\u4F60\u5FC5\u987B\u4E25\u683C\u5BF9\u7167"\u8BC4\u4F30\u6307\u6807\uFF08Evaluation Rubrics\uFF09"\u4E2D\u7684\u6BCF\u4E00\u9879\u6307\u6807\uFF0C\u7ED3\u5408"\u6267\u884C\u8FC7\u7A0B"\u4E0E"\u6700\u7EC8\u8F93\u51FA"\uFF0C\u5206\u6790 Agent \u662F\u5426\u8FBE\u6807\u3002\u8BC4\u4F30\u65F6\u65E2\u8981\u68C0\u67E5\u6700\u7EC8\u7ED3\u679C\uFF0C\u4E5F\u8981\u68C0\u67E5\u8FC7\u7A0B\u4E2D\u662F\u5426\u6B63\u786E\u5B8C\u6210\u4E86\u5FC5\u8981\u7684\u6B65\u9AA4\uFF08\u5982\u5DE5\u5177\u8C03\u7528\u3001\u4FE1\u606F\u68C0\u7D22\u7B49\uFF09\u3002
+# Task
+You must strictly evaluate the agent against every rubric in the "Evaluation Rubrics" section, using both the "Execution Trajectory" and the "Final Output". Evaluate the final result AND whether the process correctly performed the required intermediate steps (tool calls, information retrieval, etc.).
 
-# \u89C4\u5219
-1. **\u5BA2\u89C2\u6027**\uFF1A\u4EC5\u6839\u636E\u63D0\u4F9B\u7684\u4E0A\u4E0B\u6587\u5224\u5B9A\u3002\u5982\u679C\u6807\u51C6\u8981\u6C42"\u5305\u542B\u6570\u5B57"\uFF0C\u4F46\u8F93\u51FA\u53EA\u6709\u6587\u5B57\uFF0C\u5373\u4F7F\u8BED\u6C14\u518D\u597D\u4E5F\u5FC5\u987B\u6263\u5206\u3002
-2. **\u7ED3\u679C\u6821\u9A8C**\uFF1A\u5982\u679C"\u6700\u7EC8\u8F93\u51FA"\u4E2D\u7F3A\u5931\u9884\u671F\u7684\u5185\u5BB9\uFF0C\u6216\u5185\u5BB9\u4E0D\u7B26\u5408"\u8BC4\u4F30\u6307\u6807"\u4E2D\u7684\u6807\u51C6\uFF0C\u5BF9\u5E94\u7684\u6307\u6807\u5E94\u5224\u5B9A\u4E3A\u5931\u8D25\u3002
-3. **\u8FC7\u7A0B\u6821\u9A8C**\uFF1A\u5982\u679C"\u6267\u884C\u8FC7\u7A0B"\u663E\u793A Agent \u672A\u6267\u884C\u5FC5\u8981\u7684\u4E2D\u95F4\u6B65\u9AA4\uFF08\u5982\u5E94\u8C03\u7528\u5DE5\u5177\u800C\u672A\u8C03\u7528\uFF09\uFF0C\u5373\u4F7F\u6700\u7EC8\u8F93\u51FA\u770B\u4F3C\u5408\u7406\uFF0C\u4E5F\u5E94\u5728\u5BF9\u5E94\u6307\u6807\u4E0A\u6263\u5206\u3002
-4. **\u8BC1\u636E\u5BFC\u5411**\uFF1A\u5728\u7ED9\u51FA\u539F\u56E0\uFF08reason\uFF09\u65F6\uFF0C\u5FC5\u987B\u5F15\u7528\u6267\u884C\u8FC7\u7A0B\u6216\u6700\u7EC8\u8F93\u51FA\u4E2D\u7684\u5177\u4F53\u5185\u5BB9\u3002
-5. **\u52A0\u6743\u8BA1\u7B97**\uFF1A\u6700\u7EC8\u5206\u6570\u4E3A\u5404\u9879\u6307\u6807\u5F97\u5206\u4E0E\u5176\u6743\u91CD\u7684\u4E58\u79EF\u4E4B\u548C\uFF080-100\u5206\u5236\uFF09\u3002
-6. **HITL \u4E2D\u65AD\u5224\u5B9A**\uFF1A\u5982\u679C\u6267\u884C\u8FC7\u7A0B\u4E2D\u51FA\u73B0\u300CHITL \u6682\u505C\uFF1AAgent \u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u300D\u6761\u76EE\uFF0C\u8BF4\u660E Agent \u5728\u7B49\u5F85\u4EBA\u5DE5\u786E\u8BA4\u3002\u8BF7\u628A\u5B83\u5F53\u4F5C\u88AB\u6D4B\u4E1A\u52A1\u884C\u4E3A\u6765\u8BC4\u5224\uFF1A\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u81EA\u4E3B\u5B8C\u6210\uFF08\u5982"\u65E0\u9700\u786E\u8BA4\u81EA\u52A8\u6267\u884C"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u5E94\u5224\u5931\u8D25\uFF1B\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u5148\u83B7\u5F97\u786E\u8BA4\u6216\u6279\u51C6\uFF08\u5982"\u6267\u884C\u524D\u5FC5\u987B\u8BF7\u6C42\u6279\u51C6"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u662F\u6B63\u786E\u884C\u4E3A\uFF0C\u5E94\u7ED3\u5408\u5176\u65F6\u673A\u4E0E\u5185\u5BB9\u5224\u5B9A\u4E3A\u901A\u8FC7\u6216\u6309\u6307\u6807\u7ED9\u5206\u3002
-7. **HITL \u81EA\u52A8\u54CD\u5E94\u5224\u5B9A**\uFF1A\u5982\u679C\u300CHITL \u6682\u505C\u300D\u6761\u76EE\u4E4B\u540E\u51FA\u73B0\u300C\u5DF2\u81EA\u52A8\u54CD\u5E94\uFF08\u6D4B\u8BD5\u7B56\u7565 auto-approve/auto-reject/canned-response\uFF09\u300D\u6761\u76EE\uFF0C\u8BF4\u660E\u6D4B\u8BD5\u6846\u67B6\u6CE8\u5165\u4E86\u4EBA\u5DE5\u56DE\u590D\u3001\u6D41\u7A0B\u5DF2\u7EE7\u7EED\u2014\u2014\u8BF7\u6309**\u5B8C\u6574\u6D41\u7A0B**\u8BC4\u5224\u6682\u505C\u4E4B\u540E\u7684\u884C\u4E3A\uFF08\u5982\u6279\u51C6\u540E\u662F\u5426\u6B63\u786E\u6267\u884C\u4E86\u64CD\u4F5C\uFF09\uFF0C\u5E76\u6838\u5BF9\u81EA\u52A8\u54CD\u5E94\u5185\u5BB9\u662F\u5426\u7B26\u5408\u4EBA\u5DE5\u56DE\u590D\u7684\u5408\u7406\u9884\u671F\u3002
+# Rules
+1. **Objectivity**: Judge solely from the provided context. If the standard requires "contains a number" but the output has only text, points must be deducted even if the tone is good.
+2. **Result verification**: If the "Final Output" is missing expected content, or does not meet the criteria in the "Evaluation Rubrics", the corresponding rubric must be marked as failed.
+3. **Process verification**: If the "Execution Trajectory" shows the agent did not perform a necessary intermediate step (e.g., should have called a tool but did not), deduct points on the corresponding rubric even if the final output looks plausible.
+4. **Evidence-based**: When giving a reason, you must quote specific content from the execution trajectory or final output.
+5. **Weighted scoring**: The final score is the weighted sum of the rubric scores (on a 0-100 scale).
+6. **HITL interrupt judgment**: If the trajectory contains a "HITL pause: agent requested human input" entry, the agent is waiting for human confirmation. Treat this as the business behavior under test: if the expected output requires autonomous completion (e.g., "execute automatically without confirmation"), requesting human input should be judged a failure; if the expected output requires confirmation or approval first (e.g., "must request approval before executing"), requesting human input is correct behavior \u2014 judge its timing and content, passing or scoring according to the rubrics.
+7. **HITL auto-response judgment**: If a "HITL pause" entry is followed by an "auto-responded (test policy auto-approve/auto-reject/canned-response)" entry, the test framework injected a human reply and the flow continued \u2014 evaluate the behavior AFTER the pause as the complete flow (e.g., whether the operation was correctly executed after approval), and check whether the auto-response content matches a reasonable human reply.
 
-# \u8F93\u51FA\u683C\u5F0F\uFF08\u4EC5JSON\uFF09
-\u4F60\u5FC5\u987B\u4EC5\u4EE5 JSON \u683C\u5F0F\u56DE\u590D\uFF0C\u7ED3\u6784\u5982\u4E0B\uFF1A
+# Output Format (JSON only)
+You MUST reply with JSON only, using this structure:
 {
   "pass": true | false,
   "final_score": number,
   "dimension_results": [
     {
-      "name": "\u6307\u6807\u540D\u79F0",
+      "name": "rubric name",
       "score": number,
-      "reason": "\u5177\u4F53\u7684\u6263\u5206\u6216\u7ED9\u5206\u7406\u7531\uFF0C\u9700\u5F15\u7528\u8BC1\u636E"
+      "reason": "specific reason for deduction or credit, citing evidence"
     }
   ],
-  "summary": "\u5BF9 Agent \u8868\u73B0\u7684\u6574\u4F53\u8BC4\u4EF7"
+  "summary": "overall evaluation of the agent's performance"
 }
 
-\u6CE8\u610F\uFF1A\u5982\u679C final_score >= 80 \u4E14\u6CA1\u6709\u81F4\u547D\u6027\u9519\u8BEF\uFF0Cpass \u5E94\u4E3A true\uFF1B\u5426\u5219\u4E3A false\u3002`;
+Note: if final_score >= 80 and there are no fatal errors, pass should be true; otherwise false.`;
     this.lastTestPrompt = testPrompt;
     const judgeThreadId = (0, import_uuid9.v4)();
     this.lastJudgeThreadId = judgeThreadId;
@@ -28396,6 +28397,32 @@ var LatticeEvalSuite = class {
 var import_protocols16 = require("@axiom-lattice/protocols");
 var import_messages7 = require("@langchain/core/messages");
 var import_uuid10 = require("uuid");
+var DEFAULT_CALIBRATION_PROBES = [
+  {
+    id: "catch-up-correct",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "11:30",
+    expectedPass: true
+  },
+  {
+    id: "catch-up-wrong",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "12:30",
+    expectedPass: false
+  },
+  {
+    id: "discount-tax-correct",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "129.6",
+    expectedPass: true
+  },
+  {
+    id: "discount-tax-wrong",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "162",
+    expectedPass: false
+  }
+];
 var LatticeEvalProject = class {
   constructor(project, onCaseComplete) {
     this.suites = /* @__PURE__ */ new Map();
@@ -28503,21 +28530,22 @@ var LatticeEvalProject = class {
   }
   /**
    * Verify the judge agent can produce parseable, correct verdicts
-   * before committing to a full run. Uses two known-answer cases
-   * (one expected PASS, one expected FAIL) to catch broken judges.
+   * before committing to a full run. Uses known-answer probes (default
+   * multi-step reasoning set, overridable per project via
+   * judge_agent_config.calibration_cases) to catch broken judges.
+   * Checks both the pass verdict and the final_score direction.
    */
   async calibrateJudge() {
     const tenantId2 = this.project.lattice_server_config.tenant_id || "default";
     const judgeAgent = await getAgentClient(tenantId2, this.judgeAgentKey);
-    const cases = [
-      { output: "7", expected: "7", expectedPass: true },
-      { output: "7", expected: "999", expectedPass: false }
-    ];
-    for (const c of cases) {
-      const prompt = `\u4F60\u662F\u8BC4\u4F30\u4E13\u5BB6\u3002\u5224\u5B9A\u6700\u7EC8\u8F93\u51FA\u662F\u5426\u7B26\u5408\u671F\u671B\u3002
-\u6700\u7EC8\u8F93\u51FA\uFF1A${c.output}
-\u671F\u671B\u8F93\u51FA\uFF1A${c.expected}
-\u4EC5\u8F93\u51FA JSON\uFF1A{"pass": true|false, "final_score": 0-100, "summary": "\u7406\u7531"}`;
+    const probes = this.project.judge_agent_config.calibration_cases && this.project.judge_agent_config.calibration_cases.length > 0 ? this.project.judge_agent_config.calibration_cases : DEFAULT_CALIBRATION_PROBES;
+    for (const probe of probes) {
+      const trajectorySection = probe.trajectory ? `
+Trajectory: ${probe.trajectory}` : "";
+      const prompt = `You are an evaluation expert. Judge whether the final output meets the expected output description.
+Expected output description: ${probe.task}
+Final output: ${probe.finalOutput}${trajectorySection}
+Respond with JSON only: {"pass": true|false, "final_score": 0-100, "summary": "reason"}`;
       let raw = "";
       let invokeError = null;
       for (let attempt = 0; attempt < 2; attempt++) {
@@ -28542,13 +28570,24 @@ var LatticeEvalProject = class {
         return { ok: false, reason: `Calibration output unparseable: ${parsed.error}`, bypassed: true };
       }
       const actualPass = parsed.pass !== void 0 ? parsed.pass : (parsed.final_score ?? 0) >= 80;
-      if (actualPass !== c.expectedPass) {
+      if (actualPass !== probe.expectedPass) {
         return {
           ok: false,
-          reason: `Calibration mismatch: output="${c.output}" expected="${c.expected}" \u2014 judge said ${actualPass ? "PASS" : "FAIL"}, expected ${c.expectedPass ? "PASS" : "FAIL"}`,
+          reason: `Calibration mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "PASS" : "FAIL"}, judge said ${actualPass ? "PASS" : "FAIL"}`,
           bypassed: true
         };
       }
+      if (parsed.final_score !== void 0) {
+        const minScore = probe.expectedScoreMin ?? 80;
+        const scoreOk = probe.expectedPass ? parsed.final_score >= minScore : parsed.final_score < minScore;
+        if (!scoreOk) {
+          return {
+            ok: false,
+            reason: `Calibration score mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "final_score >= " + minScore : "final_score < " + minScore}, judge gave ${parsed.final_score}`,
+            bypassed: true
+          };
+        }
+      }
     }
     return { ok: true };
   }
@@ -32948,6 +32987,7 @@ registerBuiltinPlugins();
   ConsoleLoggerClient,
   CustomMetricsClient,
   CustomMiddlewareRegistry,
+  DEFAULT_CALIBRATION_PROBES,
   DaytonaInstance,
   DaytonaProvider,
   DefaultScheduleClient,