@tangle-network/agent-eval 0.72.0 → 0.72.3

package/dist/workflow/index.d.ts

@@ -0,0 +1,494 @@
+import { W as WorkflowTopology } from '../harness-optimizer-EnEnQPsr.js';
+import { b as RunSplitTag, a as RunTokenUsage, R as RunRecord } from '../run-record-BgTFzO2r.js';
+import { c as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-CRD68aH7.js';
+import { F as FailureClusterInsight } from '../insight-report-Df3lxYXM.js';
+import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-DlWCXuxL.js';
+import { F as FailureClusterReport } from '../failure-cluster-CL7IVgkJ.js';
+import { R as RedactionRule, a as RedactionReport } from '../redact-B40YG2M_.js';
+import { D as DatasetSplit } from '../dataset-B2kL-fSM.js';
+import { a as FeedbackTrajectory } from '../feedback-trajectory-B3rErRsh.js';
+import { a as PairedBootstrapResult } from '../statistics-B7yCbi9i.js';
+import '../pareto-E-pembql.js';
+import '../run-critic-BAIjX99r.js';
+import '../schema-m0gsnbt3.js';
+import '../store-CKUAgsJz.js';
+import '../errors-Dwqw-T_m.js';
+import '../store-GmBE2pZZ.js';
+import '../types-Croy5h7V.js';
+import '@tangle-network/tcloud';
+import '../llm-client-DbjLfz-K.js';
+import '../raw-provider-sink-C46HDghv.js';
+import '../summary-report-ByiOUrHj.js';
+import '../judge-calibration-DilmB3Ml.js';
+import '../control-runtime-DuFBYg7A.js';
+import '../emitter-DEZwY14K.js';
+
+type WorkflowTraceVersion = 'workflow-trace-v1';
+type WorkflowTraceEventKind = 'workflow.started' | 'workflow.phase' | 'workflow.log' | 'workflow.parallel.started' | 'workflow.parallel.ended' | 'workflow.pipeline.started' | 'workflow.pipeline.ended' | 'workflow.branch.started' | 'workflow.branch.ended' | 'workflow.branch.failed' | 'workflow.agent.started' | 'workflow.agent.ended' | 'workflow.agent.failed' | 'workflow.loop.started' | 'workflow.loop.ended' | 'workflow.loop.failed' | 'workflow.verifier.started' | 'workflow.verifier.ended' | 'workflow.verifier.failed' | 'workflow.analyst.started' | 'workflow.analyst.ended' | 'workflow.analyst.failed' | 'workflow.reviewer.started' | 'workflow.reviewer.ended' | 'workflow.reviewer.failed' | 'workflow.failed' | 'workflow.ended';
+interface WorkflowTraceEvent {
+    kind: WorkflowTraceEventKind;
+    runId: string;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface WorkflowTraceArtifact {
+    kind: string;
+    uri: string;
+    contentType?: string;
+    sha256?: string;
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowTraceExportLinks {
+    traceArtifactUri?: string;
+    exportBundleUri?: string;
+    partnerReportUri?: string;
+    intelligenceRunUri?: string;
+}
+interface WorkflowTraceEnvelope {
+    traceVersion: WorkflowTraceVersion;
+    runId: string;
+    topology?: WorkflowTopology;
+    events: WorkflowTraceEvent[];
+    artifacts?: WorkflowTraceArtifact[];
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowTraceSummary {
+    runId: string;
+    startedAt?: number;
+    endedAt?: number;
+    durationMs: number;
+    costUsd: number;
+    tokenUsage: RunTokenUsage;
+    phaseCount: number;
+    branchCount: number;
+    failedBranchCount: number;
+    agentCalls: number;
+    loopCalls: number;
+    verifierCalls: number;
+    analystCalls: number;
+    reviewerCalls: number;
+    agentFailures: number;
+    loopFailures: number;
+    verifierFailures: number;
+    analystFailures: number;
+    reviewerFailures: number;
+    eventCount: number;
+    failed: boolean;
+    failureMessage?: string;
+}
+interface WorkflowTraceProjectionMetadata {
+    experimentId: string;
+    candidateId: string;
+    seed: number;
+    model: string;
+    promptHash: string;
+    configHash: string;
+    commitSha: string;
+    splitTag: RunSplitTag;
+    scenarioId?: string;
+}
+
+declare const WORKFLOW_TRACE_EVENT_KINDS: readonly WorkflowTraceEventKind[];
+declare function validateWorkflowTraceEventKind(kind: string): WorkflowTraceEventKind;
+declare function validateWorkflowTraceEventPayload(kind: WorkflowTraceEventKind, payload: Record<string, unknown>): Record<string, unknown>;
+
+type WorkflowFeedbackPackVersion = 'workflow-feedback-pack-v1';
+type WorkflowFeedbackSeverity = AnalystSeverity;
+interface WorkflowToolUsageSummary {
+    totalCalls: number;
+    erroredCalls: number;
+    byTool: Record<string, {
+        calls: number;
+        errors: number;
+    }>;
+}
+interface WorkflowVerifierFindingSummary {
+    severity: WorkflowFeedbackSeverity;
+    message: string;
+    evidence?: string;
+    detail?: Record<string, unknown>;
+}
+interface WorkflowVerifierLayerSummary {
+    layer: string;
+    status: LayerResult['status'];
+    score?: number;
+    durationMs: number;
+    reason?: string;
+    findings: WorkflowVerifierFindingSummary[];
+    diagnostics?: Record<string, number | null>;
+}
+interface WorkflowVerifierSummary {
+    allPass: boolean;
+    blendedScore: number;
+    durationMs: number;
+    failedLayers: string[];
+    layers: WorkflowVerifierLayerSummary[];
+}
+interface WorkflowFailureClusterInput {
+    id: string;
+    name: string;
+    share?: number;
+    runCount?: number;
+    exemplars?: readonly string[];
+    suggestedFix?: string;
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowFailureClusterSummary {
+    id: string;
+    name: string;
+    share: number;
+    runCount?: number;
+    exemplars: string[];
+    suggestedFix?: string;
+    source: 'failure-cluster-view' | 'insight-report' | 'custom';
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowAnalystFindingSummary {
+    findingId: string;
+    analystId: string;
+    severity: WorkflowFeedbackSeverity;
+    area: string;
+    claim: string;
+    confidence: number;
+    subject?: string;
+    recommendedAction?: string;
+    evidenceRefs: EvidenceRef[];
+}
+interface WorkflowAnalystFeedbackPack {
+    schemaVersion: WorkflowFeedbackPackVersion;
+    runId: string;
+    generatedAt: string;
+    summary: WorkflowTraceSummary;
+    verifier?: WorkflowVerifierSummary;
+    toolUsage: WorkflowToolUsageSummary;
+    failureClusters: WorkflowFailureClusterSummary[];
+    findings: WorkflowAnalystFindingSummary[];
+    recommendations: string[];
+    driverContextLines: string[];
+}
+interface WorkflowFeedbackPackLimits {
+    findings?: number;
+    clusters?: number;
+    layerFindings?: number;
+    recommendations?: number;
+    contextLines?: number;
+}
+interface BuildWorkflowAnalystFeedbackPackOptions {
+    envelope: WorkflowTraceEnvelope | unknown;
+    verifier?: VerificationReport;
+    analystFindings?: readonly AnalystFinding[];
+    failureClusters?: FailureClusterReport | FailureClusterInsight | readonly WorkflowFailureClusterInput[];
+    generatedAt?: string;
+    limits?: WorkflowFeedbackPackLimits;
+}
+declare function buildWorkflowAnalystFeedbackPack(options: BuildWorkflowAnalystFeedbackPackOptions): WorkflowAnalystFeedbackPack;
+declare function renderWorkflowFeedbackPack(pack: WorkflowAnalystFeedbackPack, options?: {
+    maxChars?: number;
+}): string;
+
+interface WorkflowTraceSanitizationReport extends RedactionReport {
+    hashedArgs: number;
+    truncatedStrings: number;
+    droppedPayloadKeys: Record<string, number>;
+    droppedArtifactContents: number;
+}
+interface SanitizeWorkflowTraceEnvelopeOptions {
+    rules?: readonly RedactionRule[];
+    maxStringLength?: number;
+    hashSalt?: string;
+    approvedArtifactUris?: readonly string[];
+    approvedArtifactKinds?: readonly string[];
+}
+interface SanitizedWorkflowTraceEnvelopeResult {
+    envelope: WorkflowTraceEnvelope;
+    report: WorkflowTraceSanitizationReport;
+}
+declare function sanitizeWorkflowTraceEnvelope(input: WorkflowTraceEnvelope | unknown, options?: SanitizeWorkflowTraceEnvelopeOptions): SanitizedWorkflowTraceEnvelopeResult;
+
+interface WorkflowPhaseGraphNode {
+    id: string;
+    title: string;
+    startedAt?: number;
+    endedAt?: number;
+    eventCount: number;
+    branchCount: number;
+    failedBranchCount: number;
+    agentCalls: number;
+    loopCalls: number;
+    verifierCalls: number;
+    analystCalls: number;
+    reviewerCalls: number;
+    agentFailures: number;
+    loopFailures: number;
+    verifierFailures: number;
+    analystFailures: number;
+    reviewerFailures: number;
+    costUsd: number;
+    tokenUsage: RunTokenUsage;
+}
+interface WorkflowPhaseGraphBranch {
+    id: string;
+    operation: string;
+    branchIndex: number;
+    phase: string | null;
+    status: 'started' | 'ended' | 'failed';
+    startedAt?: number;
+    endedAt?: number;
+    durationMs?: number;
+    stageCount?: number;
+    stageIndex?: number;
+    message?: string;
+    code?: string;
+}
+interface WorkflowPhaseGraph {
+    nodes: WorkflowPhaseGraphNode[];
+    branches: WorkflowPhaseGraphBranch[];
+}
+declare function workflowPhaseGraph(events: readonly WorkflowTraceEvent[]): WorkflowPhaseGraph;
+
+interface WorkflowDelegateTraceSummary {
+    index: number | null;
+    label: string | null;
+    phase: string | null;
+    costUsd: number | null;
+    tokenUsage: RunTokenUsage | null;
+    trace: unknown;
+}
+interface WorkflowDelegateFailureSummary {
+    index: number | null;
+    label: string | null;
+    phase: string | null;
+    durationMs: number | null;
+    message: string | null;
+    code: string | null;
+}
+interface WorkflowCheckpointTraceSummary extends WorkflowDelegateTraceSummary {
+    output: unknown;
+}
+interface WorkflowExecutionSummary extends WorkflowTraceSummary {
+    source?: string;
+    eventKinds: Record<string, number>;
+    phases: string[];
+    phaseGraph: WorkflowPhaseGraph;
+    agentRuns: WorkflowDelegateTraceSummary[];
+    loopRuns: WorkflowDelegateTraceSummary[];
+    verifierOutputs: WorkflowCheckpointTraceSummary[];
+    analystOutputs: WorkflowCheckpointTraceSummary[];
+    reviewerOutputs: WorkflowCheckpointTraceSummary[];
+    agentFailureDetails: WorkflowDelegateFailureSummary[];
+    loopFailureDetails: WorkflowDelegateFailureSummary[];
+    verifierFailureDetails: WorkflowDelegateFailureSummary[];
+    analystFailureDetails: WorkflowDelegateFailureSummary[];
+    reviewerFailureDetails: WorkflowDelegateFailureSummary[];
+}
+interface SummarizeWorkflowExecutionOptions {
+    source?: string;
+}
+declare function summarizeWorkflowExecution(input: WorkflowTraceEnvelope | unknown, options?: SummarizeWorkflowExecutionOptions): WorkflowExecutionSummary;
+
+type WorkflowTraceIntelligenceEnvelopeVersion = 'workflow-trace-intelligence-envelope-v1';
+type WorkflowTraceExportGrantScope = 'workflow-trace:export' | 'workflow-trace:read' | '*';
+type WorkflowTraceExportGrantSubject = 'product' | 'partner' | 'tenant';
+interface WorkflowTraceExportGrant {
+    grantId: string;
+    subject: WorkflowTraceExportGrantSubject;
+    subjectId: string;
+    scopes: readonly WorkflowTraceExportGrantScope[];
+    grantedAt?: string;
+    expiresAt?: string;
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowTraceHashEvidence {
+    path: string;
+    sha256: string;
+    shape?: unknown;
+}
+interface WorkflowTraceArtifactEvidence {
+    kind: string;
+    uri: string;
+    contentType?: string;
+    sha256?: string;
+}
+interface WorkflowTraceCompactEvidence {
+    eventKinds: Record<string, number>;
+    phases: string[];
+    toolNames: string[];
+    redactedHashes: WorkflowTraceHashEvidence[];
+    artifacts: WorkflowTraceArtifactEvidence[];
+    failureMessage?: string;
+}
+interface WorkflowTraceIntelligenceEnvelope {
+    schemaVersion: WorkflowTraceIntelligenceEnvelopeVersion;
+    destination: string;
+    generatedAt: string;
+    productId: string;
+    partnerId?: string;
+    runId: string;
+    grantIds: string[];
+    traceEnvelope: WorkflowTraceEnvelope;
+    summary: WorkflowExecutionSummary;
+    compactEvidence: WorkflowTraceCompactEvidence;
+    sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
+    links?: WorkflowTraceExportLinks;
+}
+interface BuildWorkflowTraceIntelligenceEnvelopeOptions {
+    envelope: WorkflowTraceEnvelope | unknown;
+    productId: string;
+    partnerId?: string;
+    grants: readonly WorkflowTraceExportGrant[];
+    generatedAt?: string;
+    destination?: string;
+    sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
+    links?: WorkflowTraceExportLinks;
+    metadata?: Record<string, unknown>;
+}
+declare function buildWorkflowTraceIntelligenceEnvelope(options: BuildWorkflowTraceIntelligenceEnvelopeOptions): WorkflowTraceIntelligenceEnvelope;
+declare function validateWorkflowTraceIntelligenceEnvelope(input: unknown): WorkflowTraceIntelligenceEnvelope;
+
+interface WorkflowTraceRunRecordOptions extends WorkflowTraceProjectionMetadata {
+    runId?: string;
+    score?: number;
+    raw?: Record<string, number>;
+    failureMode?: string;
+    judgeMetadata?: RunRecord['judgeMetadata'];
+    agentProfile?: RunRecord['agentProfile'];
+}
+declare function workflowTraceToRunRecord(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceRunRecordOptions): RunRecord;
+
+interface WorkflowTraceTrajectoryOptions {
+    projectId?: string;
+    scenarioId?: string;
+    task: string;
+    split?: DatasetSplit;
+    tags?: Record<string, string>;
+    score?: number;
+    success?: boolean;
+    metadata?: Record<string, unknown>;
+}
+declare function workflowTraceToFeedbackTrajectory(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceTrajectoryOptions): FeedbackTrajectory;
+
+type WorkflowPartnerReportVersion = 'workflow-partner-report-v1';
+interface WorkflowPartnerFinding {
+    source: 'analyst' | 'verifier' | 'failure-cluster';
+    severity: AnalystSeverity;
+    area: string;
+    claim: string;
+    evidence: EvidenceRef[];
+    recommendedAction?: string;
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowPartnerReport {
+    schemaVersion: WorkflowPartnerReportVersion;
+    runId: string;
+    generatedAt: string;
+    summary: WorkflowTraceSummary;
+    docsApiGaps: WorkflowPartnerFinding[];
+    prReadyFindings: WorkflowPartnerFinding[];
+    failureClusters: ReturnType<typeof buildWorkflowAnalystFeedbackPack>['failureClusters'];
+    recommendations: string[];
+    traceArtifacts: WorkflowTraceEnvelope['artifacts'];
+    links?: WorkflowTraceExportLinks;
+    exportBundle: {
+        traceEnvelope: WorkflowTraceEnvelope;
+        sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
+        feedbackPack: ReturnType<typeof buildWorkflowAnalystFeedbackPack>;
+        trajectory: ReturnType<typeof workflowTraceToFeedbackTrajectory>;
+        runRecord?: RunRecord;
+    };
+}
+interface BuildWorkflowPartnerReportOptions extends Omit<BuildWorkflowAnalystFeedbackPackOptions, 'envelope'> {
+    envelope: WorkflowTraceEnvelope | unknown;
+    sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
+    trajectory: WorkflowTraceTrajectoryOptions;
+    runRecord?: WorkflowTraceRunRecordOptions;
+    links?: WorkflowTraceExportLinks;
+}
+declare function buildWorkflowPartnerReport(options: BuildWorkflowPartnerReportOptions): WorkflowPartnerReport;
+declare function validateWorkflowPartnerReport(input: unknown): WorkflowPartnerReport;
+declare function renderWorkflowPartnerReport(report: WorkflowPartnerReport, options?: {
+    maxFindings?: number;
+}): string;
+
+type WorkflowDriverPromotionDecisionVersion = 'workflow-driver-promotion-v1';
+type WorkflowDriverPromotionRejectionCode = 'missing_baseline_records' | 'missing_candidate_records' | 'missing_holdout_pairs' | 'few_pairs' | 'insufficient_lift' | 'cost_ceiling';
+interface WorkflowDriverPromotionPair {
+    key: string;
+    scenarioId: string;
+    seed: number;
+    baselineRunId: string;
+    candidateRunId: string;
+    baselineScore: number;
+    candidateScore: number;
+    delta: number;
+}
+interface WorkflowDriverPromotionEvidence {
+    pairedRuns: number;
+    expectedScenarioIds: string[];
+    pairedScenarioIds: string[];
+    missingScenarioIds: string[];
+    baselineMean: number;
+    candidateMean: number;
+    lift: number;
+    liftCi: {
+        low: number;
+        high: number;
+    };
+    bootstrap: PairedBootstrapResult;
+    confidence: number;
+    resamples: number;
+    statistic: 'mean' | 'median';
+    deltaThreshold: number;
+    baselineMedianCostUsd: number;
+    candidateMedianCostUsd: number;
+    pairs: WorkflowDriverPromotionPair[];
+}
+interface WorkflowDriverPromotionDecision {
+    schemaVersion: WorkflowDriverPromotionDecisionVersion;
+    generatedAt: string;
+    baselineStrategyId: string;
+    candidateStrategyId: string;
+    promote: boolean;
+    rejectionCode: WorkflowDriverPromotionRejectionCode | null;
+    reason: string;
+    evidence: WorkflowDriverPromotionEvidence;
+}
+interface DecideWorkflowDriverPromotionOptions {
+    records: readonly RunRecord[] | readonly unknown[];
+    baselineStrategyId?: string;
+    candidateStrategyId?: string;
+    expectedScenarioIds?: readonly string[];
+    minPairedHoldoutRuns?: number;
+    deltaThreshold?: number;
+    confidence?: number;
+    resamples?: number;
+    seed?: number;
+    statistic?: 'mean' | 'median';
+    costPerRunCeiling?: number;
+    generatedAt?: string;
+}
+declare function decideWorkflowDriverPromotion(options: DecideWorkflowDriverPromotionOptions): WorkflowDriverPromotionDecision;
+
+interface WorkflowTraceEnvelopeFromEventsOptions {
+    runId?: string;
+    topology?: WorkflowTopology;
+    artifacts?: readonly WorkflowTraceArtifact[];
+    metadata?: Record<string, unknown>;
+}
+interface WorkflowRuntimeResultLike {
+    runId?: string;
+    meta?: unknown;
+    output?: unknown;
+    events: readonly unknown[];
+}
+interface WorkflowRuntimeResultToTraceEnvelopeOptions extends Omit<WorkflowTraceEnvelopeFromEventsOptions, 'runId'> {
+    runId?: string;
+    includeOutputInMetadata?: boolean;
+}
+declare function workflowEventsToTraceEnvelope(events: readonly unknown[], options?: WorkflowTraceEnvelopeFromEventsOptions): WorkflowTraceEnvelope;
+declare function workflowRuntimeResultToTraceEnvelope(result: WorkflowRuntimeResultLike, options?: WorkflowRuntimeResultToTraceEnvelopeOptions): WorkflowTraceEnvelope;
+
+declare function validateWorkflowTraceEvent(input: unknown): WorkflowTraceEvent;
+declare function validateWorkflowTraceEnvelope(input: unknown): WorkflowTraceEnvelope;
+declare function summarizeWorkflowTrace(input: WorkflowTraceEnvelope | unknown): WorkflowTraceSummary;
+
+export { type BuildWorkflowAnalystFeedbackPackOptions, type BuildWorkflowPartnerReportOptions, type BuildWorkflowTraceIntelligenceEnvelopeOptions, type DecideWorkflowDriverPromotionOptions, type SanitizeWorkflowTraceEnvelopeOptions, type SanitizedWorkflowTraceEnvelopeResult, type SummarizeWorkflowExecutionOptions, WORKFLOW_TRACE_EVENT_KINDS, type WorkflowAnalystFeedbackPack, type WorkflowAnalystFindingSummary, type WorkflowCheckpointTraceSummary, type WorkflowDelegateTraceSummary, type WorkflowDriverPromotionDecision, type WorkflowDriverPromotionDecisionVersion, type WorkflowDriverPromotionEvidence, type WorkflowDriverPromotionPair, type WorkflowDriverPromotionRejectionCode, type WorkflowExecutionSummary, type WorkflowFailureClusterInput, type WorkflowFailureClusterSummary, type WorkflowFeedbackPackLimits, type WorkflowFeedbackPackVersion, type WorkflowFeedbackSeverity, type WorkflowPartnerFinding, type WorkflowPartnerReport, type WorkflowPartnerReportVersion, type WorkflowPhaseGraph, type WorkflowPhaseGraphBranch, type WorkflowPhaseGraphNode, type WorkflowRuntimeResultLike, type WorkflowRuntimeResultToTraceEnvelopeOptions, type WorkflowToolUsageSummary, WorkflowTopology, type WorkflowTraceArtifact, type WorkflowTraceArtifactEvidence, type WorkflowTraceCompactEvidence, type WorkflowTraceEnvelope, type WorkflowTraceEnvelopeFromEventsOptions, type WorkflowTraceEvent, type WorkflowTraceEventKind, type WorkflowTraceExportGrant, type WorkflowTraceExportGrantScope, type WorkflowTraceExportGrantSubject, type WorkflowTraceExportLinks, type WorkflowTraceHashEvidence, type WorkflowTraceIntelligenceEnvelope, type WorkflowTraceIntelligenceEnvelopeVersion, type WorkflowTraceProjectionMetadata, type WorkflowTraceRunRecordOptions, type WorkflowTraceSanitizationReport, type WorkflowTraceSummary, type WorkflowTraceTrajectoryOptions, type WorkflowTraceVersion, type WorkflowVerifierFindingSummary, type WorkflowVerifierLayerSummary, type WorkflowVerifierSummary, buildWorkflowAnalystFeedbackPack, buildWorkflowPartnerReport, buildWorkflowTraceIntelligenceEnvelope, decideWorkflowDriverPromotion, renderWorkflowFeedbackPack, renderWorkflowPartnerReport, sanitizeWorkflowTraceEnvelope, summarizeWorkflowExecution, summarizeWorkflowTrace, validateWorkflowPartnerReport, validateWorkflowTraceEnvelope, validateWorkflowTraceEvent, validateWorkflowTraceEventKind, validateWorkflowTraceEventPayload, validateWorkflowTraceIntelligenceEnvelope, workflowEventsToTraceEnvelope, workflowPhaseGraph, workflowRuntimeResultToTraceEnvelope, workflowTraceToFeedbackTrajectory, workflowTraceToRunRecord };