@axiom-lattice/core 3.0.5 → 3.0.7

package/dist/index.d.mts

@@ -5283,6 +5283,27 @@ interface LatticeAgentStepConfig {
 type OutputType = {
     type: "message_content";
 };
+/**
+ * A known-answer probe used to verify the judge agent can discriminate
+ * correct from incorrect outputs before a batch run (calibrateJudge).
+ */
+interface CalibrationProbe {
+    /** Stable identifier for error messages and reporting. */
+    id: string;
+    /** The expected output description, including the problem itself — the judge must verify by reasoning. */
+    task: string;
+    /** The output under test. */
+    finalOutput: string;
+    /** Optional execution trajectory for process verification. */
+    trajectory?: string;
+    /** Whether the judge must mark this probe as PASS. */
+    expectedPass: boolean;
+    /**
+     * Minimum acceptable final_score. Default rule when absent:
+     * expectedPass=true requires final_score >= 80, expectedPass=false requires < 80.
+     */
+    expectedScoreMin?: number;
+}
 interface LatticeEvalProjectType {
     projectName: string;
     version?: string;
@@ -5291,6 +5312,8 @@ interface LatticeEvalProjectType {
     templates?: LatticeEvalTemplate[];
     judge_agent_config: {
         modelKey: string;
+        /** Optional per-project calibration probes; defaults to DEFAULT_CALIBRATION_PROBES when absent. */
+        calibration_cases?: CalibrationProbe[];
     };
     lattice_server_config: {
         tenant_id?: string;
@@ -5666,6 +5689,12 @@ declare class LatticeEvalSuite {
     runAllCases(concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
 }
 
+/**
+ * Built-in known-answer calibration probes: multi-step reasoning problems,
+ * each paired with a correct (PASS) and a plausible-but-wrong (FAIL) output.
+ * A judge that cannot discriminate these should not be trusted with real cases.
+ */
+declare const DEFAULT_CALIBRATION_PROBES: CalibrationProbe[];
 /**
  * Manages a project with multiple evaluation suites.
  * Registers a per-project judge agent (keyed by project name and tenant)
@@ -5686,8 +5715,10 @@ declare class LatticeEvalProject {
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
     /**
      * Verify the judge agent can produce parseable, correct verdicts
-     * before committing to a full run. Uses two known-answer cases
-     * (one expected PASS, one expected FAIL) to catch broken judges.
+     * before committing to a full run. Uses known-answer probes (default
+     * multi-step reasoning set, overridable per project via
+     * judge_agent_config.calibration_cases) to catch broken judges.
+     * Checks both the pass verdict and the final_score direction.
      */
     calibrateJudge(): Promise<{
         ok: boolean;
@@ -8171,4 +8202,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CalibrationProbe, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, DEFAULT_CALIBRATION_PROBES, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };

package/dist/index.d.ts

@@ -5283,6 +5283,27 @@ interface LatticeAgentStepConfig {
 type OutputType = {
     type: "message_content";
 };
+/**
+ * A known-answer probe used to verify the judge agent can discriminate
+ * correct from incorrect outputs before a batch run (calibrateJudge).
+ */
+interface CalibrationProbe {
+    /** Stable identifier for error messages and reporting. */
+    id: string;
+    /** The expected output description, including the problem itself — the judge must verify by reasoning. */
+    task: string;
+    /** The output under test. */
+    finalOutput: string;
+    /** Optional execution trajectory for process verification. */
+    trajectory?: string;
+    /** Whether the judge must mark this probe as PASS. */
+    expectedPass: boolean;
+    /**
+     * Minimum acceptable final_score. Default rule when absent:
+     * expectedPass=true requires final_score >= 80, expectedPass=false requires < 80.
+     */
+    expectedScoreMin?: number;
+}
 interface LatticeEvalProjectType {
     projectName: string;
     version?: string;
@@ -5291,6 +5312,8 @@ interface LatticeEvalProjectType {
     templates?: LatticeEvalTemplate[];
     judge_agent_config: {
         modelKey: string;
+        /** Optional per-project calibration probes; defaults to DEFAULT_CALIBRATION_PROBES when absent. */
+        calibration_cases?: CalibrationProbe[];
     };
     lattice_server_config: {
         tenant_id?: string;
@@ -5666,6 +5689,12 @@ declare class LatticeEvalSuite {
     runAllCases(concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
 }
 
+/**
+ * Built-in known-answer calibration probes: multi-step reasoning problems,
+ * each paired with a correct (PASS) and a plausible-but-wrong (FAIL) output.
+ * A judge that cannot discriminate these should not be trusted with real cases.
+ */
+declare const DEFAULT_CALIBRATION_PROBES: CalibrationProbe[];
 /**
  * Manages a project with multiple evaluation suites.
  * Registers a per-project judge agent (keyed by project name and tenant)
@@ -5686,8 +5715,10 @@ declare class LatticeEvalProject {
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
     /**
      * Verify the judge agent can produce parseable, correct verdicts
-     * before committing to a full run. Uses two known-answer cases
-     * (one expected PASS, one expected FAIL) to catch broken judges.
+     * before committing to a full run. Uses known-answer probes (default
+     * multi-step reasoning set, overridable per project via
+     * judge_agent_config.calibration_cases) to catch broken judges.
+     * Checks both the pass verdict and the final_score direction.
      */
     calibrateJudge(): Promise<{
         ok: boolean;
@@ -8171,4 +8202,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CalibrationProbe, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, DEFAULT_CALIBRATION_PROBES, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, type DeleteResult, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };