@gleanwork/mcp-server-tester 2.0.0-beta.2 → 2.0.0-beta.4

package/dist/index.d.cts

@@ -9,6 +9,76 @@ import '@modelcontextprotocol/sdk/client/auth.js';
 import '@modelcontextprotocol/sdk/shared/auth.js';
 import 'oauth4webapi';
 
+interface AuditCoworkNativeRunOptions {
+    rawResultsPath: string;
+    nativeRoot: string;
+    expectedCases: number;
+    expectedModel?: string;
+}
+type CoworkNativeAuditIssue = 'INVALID_OPTIONS' | 'INPUT_UNREADABLE' | 'INPUT_LIMIT' | 'UNSAFE_PATH' | 'INVALID_RESULTS' | 'CASE_COUNT_MISMATCH' | 'DUPLICATE_CASE_ID' | 'INVALID_CASE_ID' | 'INVALID_SESSION_ID' | 'DUPLICATE_SESSION_ID' | 'SESSION_UNREADABLE' | 'SESSION_ID_MISMATCH' | 'TRANSCRIPT_AMBIGUOUS' | 'NATIVE_PARSE_FAILED' | 'AUDIT_NOT_PARSED' | 'TRANSCRIPT_NOT_PARSED' | 'PARSE_WARNINGS' | 'NATIVE_INCOMPLETE' | 'NATIVE_ERROR' | 'USAGE_UNAVAILABLE' | 'COST_UNAVAILABLE' | 'TIMING_UNAVAILABLE' | 'MODEL_UNAVAILABLE' | 'MODEL_MISMATCH' | 'PROMPT_MISMATCH' | 'FINAL_RESPONSE_MISMATCH' | 'EVENTS_MISMATCH' | 'TOOL_CALLS_MISMATCH' | 'USAGE_MISMATCH' | 'TIMING_MISMATCH' | 'TELEMETRY_MISMATCH' | 'EVIDENCE_FLAGS_MISMATCH' | 'ARM_RESULTS_MISMATCH' | 'TOOL_OUTPUT_UNAVAILABLE' | 'ATTACHMENT_MISSING' | 'ATTACHMENT_EMPTY' | 'ATTACHMENT_UNSAFE' | 'ATTACHMENT_LIMIT';
+interface CoworkNativeAuditUsage {
+    inputTokens: number | null;
+    outputTokens: number | null;
+    cacheReadInputTokens: number | null;
+    cacheCreationInputTokens: number | null;
+    totalCostUsd: number | null;
+}
+interface CoworkNativeAuditTiming {
+    durationMs: number | null;
+    durationApiMs: number | null;
+}
+interface CoworkNativeAuditAttachment {
+    /** Index in the native-normalized tool call order. No recorded paths are returned. */
+    toolCallIndex: number;
+    valid: boolean;
+    sizeBytes: number | null;
+    sha256: string | null;
+    issues: CoworkNativeAuditIssue[];
+}
+interface CoworkNativeAuditCase {
+    /** Recognized numeric case IDs are retained; all other IDs are SHA-256 pseudonyms. */
+    id: string;
+    pass: boolean | null;
+    sessionId: string | null;
+    model: string | null;
+    usage: CoworkNativeAuditUsage | null;
+    timing: CoworkNativeAuditTiming | null;
+    toolCounts: {
+        total: number;
+        mcp: number;
+        host: number;
+        errors: number | null;
+    } | null;
+    validity: {
+        auditParsed: boolean | null;
+        transcriptParsed: boolean | null;
+        complete: boolean | null;
+        nonError: boolean | null;
+        hasUsage: boolean | null;
+        hasCost: boolean | null;
+        noWarnings: boolean | null;
+    };
+    attachments: CoworkNativeAuditAttachment[];
+    evidencePassed: boolean;
+    issues: CoworkNativeAuditIssue[];
+}
+interface CoworkNativeAuditReport {
+    schemaVersion: 1;
+    expectedCases: number;
+    observedCases: number;
+    qualityPassed: boolean | null;
+    evidencePassed: boolean;
+    status: 'verified' | 'failed';
+    cases: CoworkNativeAuditCase[];
+    issues: CoworkNativeAuditIssue[];
+    /** Available only for verified evidence. Each unknown field stays null. */
+    totals: (CoworkNativeAuditUsage & CoworkNativeAuditTiming & {
+        costScope: 'native-inference-only';
+    }) | null;
+}
+/** Offline, metadata-only audit of MST schema-v1 raw results and a saved Linux Cowork archive. */
+declare function auditCoworkNativeRun(options: AuditCoworkNativeRunOptions): Promise<CoworkNativeAuditReport>;
+
 /**
  * MCP Host Simulation - Main entry point
  *
@@ -914,4 +984,4 @@ declare function getExternalHostReference(): Record<string, unknown>;
  */
 declare function createJudge(config?: JudgeConfig): Judge;
 
-export { AuthType, BUILT_IN_METRICS, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };
+export { type AuditCoworkNativeRunOptions, AuthType, BUILT_IN_METRICS, type CoworkNativeAuditAttachment, type CoworkNativeAuditCase, type CoworkNativeAuditIssue, type CoworkNativeAuditReport, type CoworkNativeAuditTiming, type CoworkNativeAuditUsage, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, auditCoworkNativeRun, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };

package/dist/index.d.ts

@@ -9,6 +9,76 @@ import '@modelcontextprotocol/sdk/client/auth.js';
 import '@modelcontextprotocol/sdk/shared/auth.js';
 import 'oauth4webapi';
 
+interface AuditCoworkNativeRunOptions {
+    rawResultsPath: string;
+    nativeRoot: string;
+    expectedCases: number;
+    expectedModel?: string;
+}
+type CoworkNativeAuditIssue = 'INVALID_OPTIONS' | 'INPUT_UNREADABLE' | 'INPUT_LIMIT' | 'UNSAFE_PATH' | 'INVALID_RESULTS' | 'CASE_COUNT_MISMATCH' | 'DUPLICATE_CASE_ID' | 'INVALID_CASE_ID' | 'INVALID_SESSION_ID' | 'DUPLICATE_SESSION_ID' | 'SESSION_UNREADABLE' | 'SESSION_ID_MISMATCH' | 'TRANSCRIPT_AMBIGUOUS' | 'NATIVE_PARSE_FAILED' | 'AUDIT_NOT_PARSED' | 'TRANSCRIPT_NOT_PARSED' | 'PARSE_WARNINGS' | 'NATIVE_INCOMPLETE' | 'NATIVE_ERROR' | 'USAGE_UNAVAILABLE' | 'COST_UNAVAILABLE' | 'TIMING_UNAVAILABLE' | 'MODEL_UNAVAILABLE' | 'MODEL_MISMATCH' | 'PROMPT_MISMATCH' | 'FINAL_RESPONSE_MISMATCH' | 'EVENTS_MISMATCH' | 'TOOL_CALLS_MISMATCH' | 'USAGE_MISMATCH' | 'TIMING_MISMATCH' | 'TELEMETRY_MISMATCH' | 'EVIDENCE_FLAGS_MISMATCH' | 'ARM_RESULTS_MISMATCH' | 'TOOL_OUTPUT_UNAVAILABLE' | 'ATTACHMENT_MISSING' | 'ATTACHMENT_EMPTY' | 'ATTACHMENT_UNSAFE' | 'ATTACHMENT_LIMIT';
+interface CoworkNativeAuditUsage {
+    inputTokens: number | null;
+    outputTokens: number | null;
+    cacheReadInputTokens: number | null;
+    cacheCreationInputTokens: number | null;
+    totalCostUsd: number | null;
+}
+interface CoworkNativeAuditTiming {
+    durationMs: number | null;
+    durationApiMs: number | null;
+}
+interface CoworkNativeAuditAttachment {
+    /** Index in the native-normalized tool call order. No recorded paths are returned. */
+    toolCallIndex: number;
+    valid: boolean;
+    sizeBytes: number | null;
+    sha256: string | null;
+    issues: CoworkNativeAuditIssue[];
+}
+interface CoworkNativeAuditCase {
+    /** Recognized numeric case IDs are retained; all other IDs are SHA-256 pseudonyms. */
+    id: string;
+    pass: boolean | null;
+    sessionId: string | null;
+    model: string | null;
+    usage: CoworkNativeAuditUsage | null;
+    timing: CoworkNativeAuditTiming | null;
+    toolCounts: {
+        total: number;
+        mcp: number;
+        host: number;
+        errors: number | null;
+    } | null;
+    validity: {
+        auditParsed: boolean | null;
+        transcriptParsed: boolean | null;
+        complete: boolean | null;
+        nonError: boolean | null;
+        hasUsage: boolean | null;
+        hasCost: boolean | null;
+        noWarnings: boolean | null;
+    };
+    attachments: CoworkNativeAuditAttachment[];
+    evidencePassed: boolean;
+    issues: CoworkNativeAuditIssue[];
+}
+interface CoworkNativeAuditReport {
+    schemaVersion: 1;
+    expectedCases: number;
+    observedCases: number;
+    qualityPassed: boolean | null;
+    evidencePassed: boolean;
+    status: 'verified' | 'failed';
+    cases: CoworkNativeAuditCase[];
+    issues: CoworkNativeAuditIssue[];
+    /** Available only for verified evidence. Each unknown field stays null. */
+    totals: (CoworkNativeAuditUsage & CoworkNativeAuditTiming & {
+        costScope: 'native-inference-only';
+    }) | null;
+}
+/** Offline, metadata-only audit of MST schema-v1 raw results and a saved Linux Cowork archive. */
+declare function auditCoworkNativeRun(options: AuditCoworkNativeRunOptions): Promise<CoworkNativeAuditReport>;
+
 /**
  * MCP Host Simulation - Main entry point
  *
@@ -914,4 +984,4 @@ declare function getExternalHostReference(): Record<string, unknown>;
  */
 declare function createJudge(config?: JudgeConfig): Judge;
 
-export { AuthType, BUILT_IN_METRICS, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };
+export { type AuditCoworkNativeRunOptions, AuthType, BUILT_IN_METRICS, type CoworkNativeAuditAttachment, type CoworkNativeAuditCase, type CoworkNativeAuditIssue, type CoworkNativeAuditReport, type CoworkNativeAuditTiming, type CoworkNativeAuditUsage, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, auditCoworkNativeRun, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };