@tangle-network/agent-eval 0.38.0 → 0.40.1

package/dist/index-CN2agEaO.d.ts

@@ -1,191 +0,0 @@
-import { a as DatasetScenario, c as Dataset, b as DatasetManifest } from './dataset-ueRVTUoY.js';
-import { b as CalibrationResult } from './judge-calibration-DilmB3Ml.js';
-import { O as OutcomeStore } from './outcome-store-D6KWmYvj.js';
-import { T as TraceStore } from './store-Db2Bv8Cf.js';
-
-/**
- * Red-team battery — adversarial scenario corpus with per-category
- * scorers.
- *
- * Every scenario is a `DatasetScenario` with a `redTeam: {...}` payload
- * that declares: category, expected refusal, attack surface. Scorers
- * inspect the agent's output + trajectory and return a PASS/FAIL per
- * category. Aggregated via `redTeamReport`.
- *
- * The built-in corpus is intentionally conservative — 40 cases
- * covering the most-seen attack patterns. Extend via `extendCorpus`.
- */
-
-type RedTeamCategory = 'prompt_injection_direct' | 'prompt_injection_indirect' | 'jailbreak_persona' | 'jailbreak_dan' | 'pii_leak' | 'permission_escalation' | 'data_exfil' | 'policy_override';
-interface RedTeamPayload {
-    category: RedTeamCategory;
-    /** The hostile input the agent sees. */
-    input: string;
-    /**
-     * What a safe agent should do: "refuse" means explicit refusal,
-     * "ignore" means process benignly without executing the attack.
-     */
-    expected: 'refuse' | 'ignore';
-    /** Substring(s) that MUST NOT appear in output. */
-    forbiddenStrings?: string[];
-    /** Tool names the agent MUST NOT call for this scenario. */
-    forbiddenTools?: string[];
-}
-interface RedTeamCase extends DatasetScenario {
-    payload: RedTeamPayload;
-}
-interface RedTeamFinding {
-    scenarioId: string;
-    category: RedTeamCategory;
-    passed: boolean;
-    reason: string;
-    evidence?: string;
-}
-interface RedTeamReport {
-    findings: RedTeamFinding[];
-    passRateByCategory: Record<RedTeamCategory, number>;
-    overallPassRate: number;
-}
-/** Ship a canonical, small corpus. Consumers extend via `extendCorpus`. */
-declare const DEFAULT_RED_TEAM_CORPUS: RedTeamCase[];
-declare function redTeamDataset(extraCases?: RedTeamCase[]): Dataset;
-/**
- * Score a single agent response against the red-team case.
- * `toolCalls` should be the names of tools the agent invoked during the run.
- */
-declare function scoreRedTeamOutput(output: string, toolCalls: string[], rtCase: RedTeamCase): RedTeamFinding;
-/** Aggregate red-team findings into per-category pass rates. */
-declare function redTeamReport(findings: RedTeamFinding[]): RedTeamReport;
-/**
- * Extract the tool-call names from a corpus run — convenience for the
- * common pipeline (run the scenario → score the run).
- */
-declare function toolNamesForRun(store: TraceStore, runId: string): Promise<string[]>;
-
-/**
- * Governance reporting — shared types.
- *
- * The framework collects a `GovernanceContext` (traces + outcomes +
- * dataset manifests + red-team results + judge calibration) and each
- * specific template (NIST AI RMF, SOC2, EU AI Act) renders a
- * structured report from it.
- *
- * Reports are machine-readable JSON first; human-readable Markdown is a
- * pure transform on top. External auditors consume the Markdown; CI
- * consumes the JSON.
- */
-
-interface GovernanceContext {
-    /** Legal / org identity for the report. */
-    organization: string;
-    /** System / agent identifier. */
-    systemName: string;
-    /** ISO8601 period the report covers. */
-    periodStart: string;
-    periodEnd: string;
-    /** Versioned dataset manifests used during the period. */
-    datasets: DatasetManifest[];
-    traceStore: TraceStore;
-    outcomeStore?: OutcomeStore;
-    /** Cached red-team results for the period, if available. */
-    redTeam?: RedTeamReport;
-    /** Judge-vs-human calibration results, if measured. */
-    judgeCalibration?: CalibrationResult[];
-    /** Responsible owner for the system — role + name + email. */
-    owner: {
-        role: string;
-        name: string;
-        email: string;
-    };
-}
-interface GovernanceFinding {
-    id: string;
-    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
-    /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */
-    control: string;
-    summary: string;
-    evidence?: string;
-    remediation?: string;
-}
-interface GovernanceReport {
-    framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT';
-    version: string;
-    context: Pick<GovernanceContext, 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner'>;
-    summary: {
-        findings: number;
-        byeverity: Record<GovernanceFinding['severity'], number>;
-        overall: 'compliant' | 'compliant-with-findings' | 'non-compliant';
-    };
-    findings: GovernanceFinding[];
-    /** Framework-specific structured payload (mapped controls, risk class, etc.). */
-    payload: Record<string, unknown>;
-    generatedAt: string;
-}
-declare function renderMarkdown(report: GovernanceReport): string;
-declare function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'];
-
-/**
- * EU AI Act — risk-class classification + compliance checklist.
- *
- * Classification is declarative: caller supplies the domain/use-case
- * signals (biometric? critical infrastructure? education? employment?
- * access to services?) and we map to the Act's risk tiers:
- *   - "unacceptable" (prohibited)
- *   - "high"        (Annex III — strict obligations)
- *   - "limited"     (transparency obligations)
- *   - "minimal"     (voluntary codes of conduct)
- *
- * Then the compliance checklist enumerates Article 9 (risk mgmt),
- * 10 (data + data governance), 11 (technical documentation), 13
- * (transparency), 14 (human oversight), 15 (accuracy + robustness)
- * requirements and flags gaps.
- */
-
-type EuRiskClass = 'unacceptable' | 'high' | 'limited' | 'minimal';
-interface UseCaseSignals {
-    /** Used for biometric identification in public spaces? (Art. 5 — unacceptable). */
-    biometricPublic?: boolean;
-    /** Social scoring by public authorities? (Art. 5). */
-    socialScoring?: boolean;
-    /** Subliminal manipulation? (Art. 5). */
-    subliminal?: boolean;
-    /** Annex III sector: critical infrastructure / education / employment /
-     *  access to essential services / law enforcement / migration /
-     *  administration of justice / democratic processes? */
-    annexIII?: boolean;
-    /** Interacts directly with natural persons (chatbot, agent)? — limited risk. */
-    chatbot?: boolean;
-    /** Generates synthetic media (image/audio/video/text deepfakes)? — limited risk. */
-    generatesSyntheticMedia?: boolean;
-}
-declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
-declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
-
-/**
- * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping.
- *
- * Each subcategory derives its status from concrete framework state:
- *   MEASURE 2.x: do we have a calibration regime? contamination controls?
- *   MEASURE 2.7: are red-team results available?
- *   MANAGE 1.x: are outcome metrics captured? correlation measured?
- *   GOVERN 1.x: dataset + prompt provenance recorded?
- *
- * We ship the mapping and the derivation rules; consumers supply the
- * GovernanceContext.
- */
-
-declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceReport>;
-
-/**
- * SOC 2 — Common Criteria 7 (system operations + change management)
- * audit trail derived from the trace corpus.
- *
- * This is NOT a formal SOC2 report — that requires an external
- * auditor. What we ship is the machine-readable *evidence* package
- * that an auditor consumes: run counts, deploy events, access log
- * summary, anomaly tracking, response-time SLOs.
- */
-
-declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
-
-export { DEFAULT_RED_TEAM_CORPUS as D, type EuRiskClass as E, type GovernanceContext as G, type RedTeamCase as R, type UseCaseSignals as U, type GovernanceFinding as a, type GovernanceReport as b, type RedTeamCategory as c, type RedTeamFinding as d, type RedTeamPayload as e, type RedTeamReport as f, classifyEuAiRisk as g, euAiActReport as h, redTeamReport as i, renderMarkdown as j, soc2Report as k, summarize as l, nistAiRmfReport as n, redTeamDataset as r, scoreRedTeamOutput as s, toolNamesForRun as t };