@axiom-lattice/core 3.0.0 → 3.0.1

package/dist/index.d.mts

@@ -5259,9 +5259,12 @@ interface EvalRunService {
      *
      * @param tenantId - Tenant that owns the project
      * @param projectId - Project to evaluate
+     * @param suiteIds - Optional suite filter — only these suites run.
+     *   Omit to run all suites. Used to keep the validation suite
+     *   untouched during the fix loop (hold-out isolation).
      * @returns The newly created run ID
      */
-    startRun(tenantId: string, projectId: string): Promise<string>;
+    startRun(tenantId: string, projectId: string, suiteIds?: string[]): Promise<string>;
     /**
      * Abort a running evaluation.
      *
@@ -5460,6 +5463,27 @@ interface CaseRunResult {
     logs: LatticeEvalLogEvent[];
 }
 
+interface JudgeVerdict {
+    pass?: boolean;
+    final_score?: number;
+    dimension_results?: Array<{
+        name: string;
+        score: number;
+        reason: string;
+    }>;
+    summary?: string;
+    error?: string;
+}
+/**
+ * Parse judge agent's raw output into a structured verdict.
+ * Returns error when no valid JSON is found — callers should treat
+ * unparseable output as FAIL (never guess pass from keywords).
+ *
+ * Type-guards verdict fields: a string `pass: "false"` or `final_score: "90"`
+ * is treated as missing (undefined) rather than truthy — a judge emitting
+ * wrong-typed verdicts must not produce a false PASS.
+ */
+declare function parseJudgeVerdict(raw: string): JudgeVerdict;
 /**
  * Configuration for Lattice evaluation.
  */
@@ -5575,6 +5599,15 @@ declare class LatticeEvalProject {
     runCase(suiteName: string, caseId: string): Promise<CaseRunResult>;
     runSuite(suiteName: string, concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
+    /**
+     * Verify the judge agent can produce parseable, correct verdicts
+     * before committing to a full run. Uses two known-answer cases
+     * (one expected PASS, one expected FAIL) to catch broken judges.
+     */
+    calibrateJudge(): Promise<{
+        ok: boolean;
+        reason?: string;
+    }>;
     /**
      * Run all suites as a batch and build an in-memory report.
      */
@@ -8032,4 +8065,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };

package/dist/index.d.ts

@@ -5259,9 +5259,12 @@ interface EvalRunService {
      *
      * @param tenantId - Tenant that owns the project
      * @param projectId - Project to evaluate
+     * @param suiteIds - Optional suite filter — only these suites run.
+     *   Omit to run all suites. Used to keep the validation suite
+     *   untouched during the fix loop (hold-out isolation).
      * @returns The newly created run ID
      */
-    startRun(tenantId: string, projectId: string): Promise<string>;
+    startRun(tenantId: string, projectId: string, suiteIds?: string[]): Promise<string>;
     /**
      * Abort a running evaluation.
      *
@@ -5460,6 +5463,27 @@ interface CaseRunResult {
     logs: LatticeEvalLogEvent[];
 }
 
+interface JudgeVerdict {
+    pass?: boolean;
+    final_score?: number;
+    dimension_results?: Array<{
+        name: string;
+        score: number;
+        reason: string;
+    }>;
+    summary?: string;
+    error?: string;
+}
+/**
+ * Parse judge agent's raw output into a structured verdict.
+ * Returns error when no valid JSON is found — callers should treat
+ * unparseable output as FAIL (never guess pass from keywords).
+ *
+ * Type-guards verdict fields: a string `pass: "false"` or `final_score: "90"`
+ * is treated as missing (undefined) rather than truthy — a judge emitting
+ * wrong-typed verdicts must not produce a false PASS.
+ */
+declare function parseJudgeVerdict(raw: string): JudgeVerdict;
 /**
  * Configuration for Lattice evaluation.
  */
@@ -5575,6 +5599,15 @@ declare class LatticeEvalProject {
     runCase(suiteName: string, caseId: string): Promise<CaseRunResult>;
     runSuite(suiteName: string, concurrency?: number, signal?: AbortSignal): Promise<CaseRunResult[]>;
     runAllSuites(concurrency?: number, signal?: AbortSignal): Promise<Map<string, CaseRunResult[]>>;
+    /**
+     * Verify the judge agent can produce parseable, correct verdicts
+     * before committing to a full run. Uses two known-answer cases
+     * (one expected PASS, one expected FAIL) to catch broken judges.
+     */
+    calibrateJudge(): Promise<{
+        ok: boolean;
+        reason?: string;
+    }>;
     /**
      * Run all suites as a batch and build an in-memory report.
      */
@@ -8032,4 +8065,4 @@ declare class IdRemapper {
     remapSkillIds(graphDefinition: Record<string, unknown>, skillRemap: Record<string, string>): Record<string, unknown>;
 }
 
-export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };
+export { AGENT_TASK_EVENT, type AddMessageParams, Agent, type AgentClient, type AgentExecutor, AgentInstanceManager, type AgentLattice, AgentLatticeManager, type AgentLifecycleEventName, AgentManager, type AgentStreamExecutor, type AgentThreadInterface, BUILTIN_PLUGINS, BUILTIN_SKILLS, type BackendFactory, type BackendProtocol, type BufferStats, type CacheEntry, type CaseRunResult, type Chunk, ChunkBuffer, ChunkBufferLatticeManager, CollectionLatticeManager, type ColumnInfo, CompositeBackend, type ConflictItem, type ConnectAllChannelsOptions, ConnectionRegistry, ConsoleLoggerClient, type CreateSandboxProviderConfig, type CronFields, CustomMetricsClient, type CustomMiddlewareFactory, CustomMiddlewareRegistry, type DatabaseConfig, type DatabaseType, DaytonaInstance, DaytonaProvider, type DaytonaProviderConfig, DefaultScheduleClient, DependencyResolver, E2BInstance, E2BProvider, type E2BProviderConfig, EMPTY_CONTENT_WARNING, type EditResult, type EmbeddingsInfo, type EmbeddingsLatticeInterface, EmbeddingsLatticeManager, type EnsureMicrosandboxInput, type EvalRunService, type ExportBundle, type ExportJob, type ExportableEntity, type ExportableEntityDefinition, ExportableEntityRegistry, type ExportableTypeInfo, type FileData, type FileInfo, FileSystemSkillStore, type FileSystemSkillStoreOptions, FilesystemBackend, type FsEntry, type GrepMatch, type IMessageQueueStore, type IMetricsServerClient, type ISqlDatabase, IdRemapper, type ImportApplyResult, type ImportEntityResult, type ImportEntityStatus, type ImportPreviewResult, InMemoryA2AApiKeyStore, InMemoryAssistantStore, InMemoryBindingStore, InMemoryChannelInstallationStore, InMemoryChunkBuffer, InMemoryCollectionStore, InMemoryDatabaseConfigStore, InMemoryMailboxStore, InMemoryMenuStore, InMemoryTaskListStore, InMemoryTaskStore, InMemoryTenantStore, InMemoryThreadMessageQueueStore, InMemoryThreadStore, InMemoryUserStore, InMemoryUserTenantLinkStore, type InsertionItem, type JudgeVerdict, LINE_NUMBER_WIDTH, type LangGraphStateChecker, type LatticeAgentStepConfig, LatticeEval, type LatticeEvalBatchReport, type LatticeEvalCase, type LatticeEvalCaseType, type LatticeEvalCaseWithTemplate, type LatticeEvalConfig, type LatticeEvalLogEvent, type LatticeEvalLogLevel, LatticeEvalProject, type LatticeEvalProjectType, type LatticeEvalResult, type LatticeEvalRubric, LatticeEvalSuite, type LatticeEvalSuiteType, type LatticeEvalTemplate, LocalSandboxInstance, LocalSandboxProvider, type LocalSandboxProviderConfig, type LoggerLattice, LoggerLatticeManager, MAX_LINE_LENGTH, type MailboxMessage, type MailboxStore, type McpLatticeInterface, McpLatticeManager, type McpServerInfo, MemoryBackend, MemoryLatticeManager, MemoryQueueClient, MemoryScheduleStorage, type MessageCompletedEvent, type MessageFailedEvent, type MessageStartedEvent, MessageType, MetricsServerManager, MicrosandboxRemoteInstance, MicrosandboxRemoteProvider, type MicrosandboxRemoteProviderClient, type MicrosandboxRemoteProviderConfig, MicrosandboxServiceClient, type MicrosandboxServiceClientConfig, type MicrosandboxShellExecInput, type ModelConfig, type ModelLatticeInterface, ModelLatticeManager, MysqlDatabase, type OutputType, type PendingMessage, PersonalAssistantConfig, PinoLoggerClient, PluginRegistry, PostgresDatabase, type PreviewError, PrometheusClient, type QueryResult, type QueueLattice, QueueLatticeManager, QueueMode, type QueuePendingEvent, RemoteSandboxInstance, RemoteSandboxProvider, type RemoteSandboxProviderConfig, type Resolution, type ResolutionAction, type ResolveAgentFn, type ResolvedConfig, type RunSandboxConfig, type RuntimeModelConfig, type STTModelInfo, STTModelLattice, type STTModelLatticeInterface, STTModelLatticeManager, type SandboxFileInfo, type SandboxFileService, SandboxFilesystem, type SandboxInstance, type SandboxIsolationLevel, SandboxLatticeManager, type SandboxManagerProtocol, type SandboxProvider, type SandboxProviderFactory, type SandboxProviderType, type SandboxShellService, SandboxSkillStore, type SandboxSkillStoreOptions, type SandboxVolumeDefinition, type ScheduleLattice, ScheduleLatticeManager, type SchedulerMiddlewareOptions, SemanticMetricsClient, type SharePayload, SimpleMemoryVectorStore, type SkillLattice, SkillLatticeManager, type SkillMeta, type SkillResource, SqlDatabaseManager, type StateAndStore, StateBackend, StoreBackend, type StoreLattice, StoreLatticeManager, type StoreType, type StoreTypeMap, TOOL_RESULT_TOKEN_LIMIT, TRUNCATION_GUIDANCE, type TableInfo, type TableSchema, type TaskEvent, type TaskListStore, type TaskSpec, TaskStatus, type TaskUpdatable, TeamAgentGraphBuilder, type TeamConfig, type TeamMiddlewareOptions, type TeamTask, type TeammateSpec, type TeammateToolsOptions, type ThreadBuffer, type ThreadBufferConfig, type ThreadBusyEvent, type ThreadIdleEvent, type ThreadInfo, type ThreadQueueConfig, type ThreadState, ThreadStatus, type ThreadStatusChangedEvent, TokenCache, type ToolDefinition, type ToolLattice, ToolLatticeManager, type UnknownToolHandlerConfig, type VectorStoreLatticeInterface, VectorStoreLatticeManager, type VectorStoreProviderLattice, VectorStoreProviderManager, VolumeFilesystem, type VolumeFsClient, type WorkflowValidationError, type WriteResult, abortWorkflowRun, agentInstanceManager, agentLatticeManager, buildGrepResultsDict, buildInput, buildNamedVolumeName, buildSandboxMetadataEnv, buildSkillFile, buildStateAnnotation, buildTableName, checkEmptyContent, clearEncryptionKeyCache, clearEvalRunService, collectionLatticeManager, compileInternal, compileWorkflow, computeSandboxName, configureStores, connectAllChannels, createAgentNode, createAgentTeam, createExecuteSqlQueryTool, createFileData, createInfoSqlTool, createListMetricsDataSourcesTool, createListMetricsServersTool, createListTablesSqlTool, createMapNode, createModelSelectorMiddleware, createNodeHandler, createQueryCheckerSqlTool, createQueryMetricDefinitionTool, createQueryMetricsListTool, createQuerySemanticMetricDataTool, createQuerySqlTool, createQueryTableDefinitionTool, createQueryTablesListTool, createResourceAddress, createSandboxProvider, createSchedulerMiddleware, createSharePayload, createTaskMiddleware, createTeamMiddleware, createTeammateTools, createUnknownToolHandlerMiddleware, createWidgetMiddleware, decrypt, describeCronExpression, documentLearningPlugin, documentParserPlugin, embeddingsLatticeManager, encrypt, ensureBuiltinAgentsForTenant, evaluateLatticeCaseWithLogs, eventBus, eventBus as eventBusDefault, extractFetcherError, extractOutput, fileDataToString, formatContentWithLineNumbers, formatGrepMatches, formatGrepResults, formatReadResponse, generateToken, getAgentClient, getAgentConfig, getAllAgentConfigs, getAllBuiltInSkillMetas, getAllToolDefinitions, getBindingRegistry, getBuiltInSkillContent, getBuiltInSkillMeta, getBuiltInSkillNames, getCheckpointSaver, getChunkBuffer, getCollectionEntryCount, getEmbeddingsClient, getEmbeddingsLattice, getEncryptionKey, getEvalRunService, getLoggerLattice, getMenuRegistry, getModelLattice, getNextCronTime, getOrCreateCollectionVectorStore, getQueueLattice, getSTTClient, getSTTClientWithTenant, getSTTModelLattice, getSandBoxManager, getScheduleLattice, getStoreLattice, getToolClient, getToolDefinition, getToolLattice, getVectorStoreClient, getVectorStoreLattice, getVectorStoreProvider, getWorkflowSignal, globSearchFiles, grepMatchesFromFiles, grepSearchFiles, hasChunkBuffer, invokeWithRetry, isBuiltInSkill, isUsingDefaultKey, isValidCronExpression, isValidSandboxName, isValidSkillName, listCollectionEntries, listSandboxProviderTypes, loggerLatticeManager, mcpManager, metricsServerManager, modelLatticeManager, normalizeSandboxName, parallelLimit, parseCronExpression, parseJudgeVerdict, parseSkillFrontmatter, parseYaml, performStringReplacement, queueLatticeManager, registerAgentLattice, registerAgentLatticeWithTenant, registerAgentLattices, registerBuiltinSkill, registerCheckpointSaver, registerChunkBuffer, registerEmbeddingsLattice, registerExistingTool, registerLoggerLattice, registerModelLattice, registerQueueLattice, registerSTTModelLattice, registerSandboxProviderType, registerScheduleLattice, registerStoreLattice, registerTeammateAgent, registerToolLattice, registerVectorStoreLattice, registerVectorStoreProvider, registerWorkflowRun, removeCollectionVectorStore, renderTemplate, resolvePath, sandboxLatticeManager, sanitizeToolCallId, scheduleLatticeManager, serializePluginMeta, setBindingRegistry, setEvalRunService, setMenuRegistry, skillLatticeManager, sqlDatabaseManager, storeLatticeManager, sttModelLatticeManager, toJsonSchema, toSafeStateExpr, toolLatticeManager, truncateIfTooLong, unregisterTeammateAgent, unregisterWorkflowRun, updateFileData, validateAgentInput, validateDSL, validateEncryptionKey, validatePath, validateSkillName, validateToolInput, vectorStoreLatticeManager, vectorStoreProviderManager };