@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/governance/index.d.ts

@@ -1,135 +1,5 @@
-import { c as DatasetManifest } from '../dataset-BbGkaN2I.js';
-import { a as CalibrationResult } from '../judge-calibration-0p2QcWNE.js';
-import { b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
-import { d as RedTeamReport } from '../red-team-BWdoyleI.js';
-import { T as TraceStore } from '../store-BcFXE6LG.js';
-import '../errors-CzMUYo7b.js';
-import '../schema-m0gsnbt3.js';
-
-/**
- * Governance reporting — shared types.
- *
- * The framework collects a `GovernanceContext` (traces + outcomes +
- * dataset manifests + red-team results + judge calibration) and each
- * specific template (NIST AI RMF, SOC2, EU AI Act) renders a
- * structured report from it.
- *
- * Reports are machine-readable JSON first; human-readable Markdown is a
- * pure transform on top. External auditors consume the Markdown; CI
- * consumes the JSON.
- */
-
-interface GovernanceContext {
-    /** Legal / org identity for the report. */
-    organization: string;
-    /** System / agent identifier. */
-    systemName: string;
-    /** ISO8601 period the report covers. */
-    periodStart: string;
-    periodEnd: string;
-    /** Versioned dataset manifests used during the period. */
-    datasets: DatasetManifest[];
-    traceStore: TraceStore;
-    outcomeStore?: OutcomeStore;
-    /** Cached red-team results for the period, if available. */
-    redTeam?: RedTeamReport;
-    /** Judge-vs-human calibration results, if measured. */
-    judgeCalibration?: CalibrationResult[];
-    /** Responsible owner for the system — role + name + email. */
-    owner: {
-        role: string;
-        name: string;
-        email: string;
-    };
-}
-interface GovernanceFinding {
-    id: string;
-    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
-    /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */
-    control: string;
-    summary: string;
-    evidence?: string;
-    remediation?: string;
-}
-interface GovernanceReport {
-    framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT';
-    version: string;
-    context: Pick<GovernanceContext, 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner'>;
-    summary: {
-        findings: number;
-        byeverity: Record<GovernanceFinding['severity'], number>;
-        overall: 'compliant' | 'compliant-with-findings' | 'non-compliant';
-    };
-    findings: GovernanceFinding[];
-    /** Framework-specific structured payload (mapped controls, risk class, etc.). */
-    payload: Record<string, unknown>;
-    generatedAt: string;
-}
-declare function renderMarkdown(report: GovernanceReport): string;
-declare function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'];
-
-/**
- * EU AI Act — risk-class classification + compliance checklist.
- *
- * Classification is declarative: caller supplies the domain/use-case
- * signals (biometric? critical infrastructure? education? employment?
- * access to services?) and we map to the Act's risk tiers:
- *   - "unacceptable" (prohibited)
- *   - "high"        (Annex III — strict obligations)
- *   - "limited"     (transparency obligations)
- *   - "minimal"     (voluntary codes of conduct)
- *
- * Then the compliance checklist enumerates Article 9 (risk mgmt),
- * 10 (data + data governance), 11 (technical documentation), 13
- * (transparency), 14 (human oversight), 15 (accuracy + robustness)
- * requirements and flags gaps.
- */
-
-type EuRiskClass = 'unacceptable' | 'high' | 'limited' | 'minimal';
-interface UseCaseSignals {
-    /** Used for biometric identification in public spaces? (Art. 5 — unacceptable). */
-    biometricPublic?: boolean;
-    /** Social scoring by public authorities? (Art. 5). */
-    socialScoring?: boolean;
-    /** Subliminal manipulation? (Art. 5). */
-    subliminal?: boolean;
-    /** Annex III sector: critical infrastructure / education / employment /
-     *  access to essential services / law enforcement / migration /
-     *  administration of justice / democratic processes? */
-    annexIII?: boolean;
-    /** Interacts directly with natural persons (chatbot, agent)? — limited risk. */
-    chatbot?: boolean;
-    /** Generates synthetic media (image/audio/video/text deepfakes)? — limited risk. */
-    generatesSyntheticMedia?: boolean;
-}
-declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
-declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
-
-/**
- * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping.
- *
- * Each subcategory derives its status from concrete framework state:
- *   MEASURE 2.x: do we have a calibration regime? contamination controls?
- *   MEASURE 2.7: are red-team results available?
- *   MANAGE 1.x: are outcome metrics captured? correlation measured?
- *   GOVERN 1.x: dataset + prompt provenance recorded?
- *
- * We ship the mapping and the derivation rules; consumers supply the
- * GovernanceContext.
- */
-
-declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceReport>;
-
-/**
- * SOC 2 — Common Criteria 7 (system operations + change management)
- * audit trail derived from the trace corpus.
- *
- * This is NOT a formal SOC2 report — that requires an external
- * auditor. What we ship is the machine-readable *evidence* package
- * that an auditor consumes: run counts, deploy events, access log
- * summary, anomaly tracking, response-time SLOs.
- */
-
-declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
-
-export { type EuRiskClass, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type UseCaseSignals, classifyEuAiRisk, euAiActReport, nistAiRmfReport, renderMarkdown, soc2Report, summarize };
+export * from './eu-ai-act';
+export * from './nist-ai-rmf';
+export * from './soc2';
+export * from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/governance/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/governance/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,eAAe,CAAA;AAC7B,cAAc,QAAQ,CAAA;AACtB,cAAc,SAAS,CAAA"}

package/dist/governance/nist-ai-rmf.d.ts

@@ -0,0 +1,15 @@
+/**
+ * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping.
+ *
+ * Each subcategory derives its status from concrete framework state:
+ *   MEASURE 2.x: do we have a calibration regime? contamination controls?
+ *   MEASURE 2.7: are red-team results available?
+ *   MANAGE 1.x: are outcome metrics captured? correlation measured?
+ *   GOVERN 1.x: dataset + prompt provenance recorded?
+ *
+ * We ship the mapping and the derivation rules; consumers supply the
+ * GovernanceContext.
+ */
+import type { GovernanceContext, GovernanceReport } from './types';
+export declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceReport>;
+//# sourceMappingURL=nist-ai-rmf.d.ts.map

package/dist/governance/nist-ai-rmf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nist-ai-rmf.d.ts","sourceRoot":"","sources":["../../src/governance/nist-ai-rmf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAqB,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAGrF,wBAAsB,eAAe,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAgKvF"}

package/dist/governance/soc2.d.ts

@@ -0,0 +1,12 @@
+/**
+ * SOC 2 — Common Criteria 7 (system operations + change management)
+ * audit trail derived from the trace corpus.
+ *
+ * This is NOT a formal SOC2 report — that requires an external
+ * auditor. What we ship is the machine-readable *evidence* package
+ * that an auditor consumes: run counts, deploy events, access log
+ * summary, anomaly tracking, response-time SLOs.
+ */
+import type { GovernanceContext, GovernanceReport } from './types';
+export declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
+//# sourceMappingURL=soc2.d.ts.map

package/dist/governance/soc2.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"soc2.d.ts","sourceRoot":"","sources":["../../src/governance/soc2.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAqB,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAGrF,wBAAsB,UAAU,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA6GlF"}

package/dist/governance/types.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Governance reporting — shared types.
+ *
+ * The framework collects a `GovernanceContext` (traces + outcomes +
+ * dataset manifests + red-team results + judge calibration) and each
+ * specific template (NIST AI RMF, SOC2, EU AI Act) renders a
+ * structured report from it.
+ *
+ * Reports are machine-readable JSON first; human-readable Markdown is a
+ * pure transform on top. External auditors consume the Markdown; CI
+ * consumes the JSON.
+ */
+import type { DatasetManifest } from '../dataset';
+import type { CalibrationResult } from '../judge-calibration';
+import type { OutcomeStore } from '../meta-eval/outcome-store';
+import type { RedTeamReport } from '../red-team';
+import type { TraceStore } from '../trace/store';
+export interface GovernanceContext {
+    /** Legal / org identity for the report. */
+    organization: string;
+    /** System / agent identifier. */
+    systemName: string;
+    /** ISO8601 period the report covers. */
+    periodStart: string;
+    periodEnd: string;
+    /** Versioned dataset manifests used during the period. */
+    datasets: DatasetManifest[];
+    traceStore: TraceStore;
+    outcomeStore?: OutcomeStore;
+    /** Cached red-team results for the period, if available. */
+    redTeam?: RedTeamReport;
+    /** Judge-vs-human calibration results, if measured. */
+    judgeCalibration?: CalibrationResult[];
+    /** Responsible owner for the system — role + name + email. */
+    owner: {
+        role: string;
+        name: string;
+        email: string;
+    };
+}
+export interface GovernanceFinding {
+    id: string;
+    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
+    /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */
+    control: string;
+    summary: string;
+    evidence?: string;
+    remediation?: string;
+}
+export interface GovernanceReport {
+    framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT';
+    version: string;
+    context: Pick<GovernanceContext, 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner'>;
+    summary: {
+        findings: number;
+        byeverity: Record<GovernanceFinding['severity'], number>;
+        overall: 'compliant' | 'compliant-with-findings' | 'non-compliant';
+    };
+    findings: GovernanceFinding[];
+    /** Framework-specific structured payload (mapped controls, risk class, etc.). */
+    payload: Record<string, unknown>;
+    generatedAt: string;
+}
+export declare function renderMarkdown(report: GovernanceReport): string;
+export declare function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'];
+//# sourceMappingURL=types.d.ts.map

package/dist/governance/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/governance/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAChD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,WAAW,iBAAiB;IAChC,2CAA2C;IAC3C,YAAY,EAAE,MAAM,CAAA;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,0DAA0D;IAC1D,QAAQ,EAAE,eAAe,EAAE,CAAA;IAC3B,UAAU,EAAE,UAAU,CAAA;IACtB,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,4DAA4D;IAC5D,OAAO,CAAC,EAAE,aAAa,CAAA;IACvB,uDAAuD;IACvD,gBAAgB,CAAC,EAAE,iBAAiB,EAAE,CAAA;IACtC,8DAA8D;IAC9D,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACrD;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAA;IACzD,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,aAAa,GAAG,MAAM,GAAG,WAAW,CAAA;IAC/C,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,IAAI,CACX,iBAAiB,EACjB,cAAc,GAAG,YAAY,GAAG,aAAa,GAAG,WAAW,GAAG,OAAO,CACtE,CAAA;IACD,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAA;QACxD,OAAO,EAAE,WAAW,GAAG,yBAAyB,GAAG,eAAe,CAAA;KACnE,CAAA;IACD,QAAQ,EAAE,iBAAiB,EAAE,CAAA;IAC7B,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAChC,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA4C/D;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAgBpF"}

package/dist/harness-optimizer.d.ts

@@ -0,0 +1,82 @@
+import { type Objective, type ParetoResult } from './pareto';
+import { type RunTrace } from './run-critic';
+import { type RunScore, type RunScoreWeights } from './run-score';
+import type { SteeringBundle } from './steering';
+export type HarnessIntervention = 'continue' | 'plan' | 'audit' | 'recover' | 'repair' | 'verify' | 'final_gate' | 'wait_for_measurement' | 'abort';
+export interface WorkflowTopology {
+    id: string;
+    interventions: HarnessIntervention[];
+    maxParallelBranches?: number;
+    metadata?: Record<string, unknown>;
+}
+export interface MeasurementPolicy {
+    required: string[];
+    optional?: string[];
+    promoteOn?: Array<keyof RunScore | 'aggregate'>;
+}
+export interface HarnessVariant {
+    id: string;
+    steering?: SteeringBundle;
+    topology?: WorkflowTopology;
+    measurement?: MeasurementPolicy;
+    budgets?: Record<string, number>;
+    models?: Record<string, string>;
+    reviewers?: Record<string, string>;
+    metadata?: Record<string, unknown>;
+}
+export interface HarnessScenario {
+    id: string;
+    task: string;
+    split?: 'train' | 'validation' | 'test' | string;
+    metadata?: Record<string, unknown>;
+}
+export interface HarnessRunRequest {
+    variant: HarnessVariant;
+    scenario: HarnessScenario;
+    trialIndex: number;
+}
+export interface HarnessAdapter {
+    run(request: HarnessRunRequest): Promise<RunTrace>;
+}
+export interface HarnessRunResult {
+    variant: HarnessVariant;
+    scenario: HarnessScenario;
+    trialIndex: number;
+    trace: RunTrace;
+    score: RunScore;
+    aggregate: number;
+}
+export interface HarnessVariantReport {
+    variant: HarnessVariant;
+    runs: HarnessRunResult[];
+    aggregateMean: number;
+    passRate: number;
+    costUsdMean: number;
+    wallSecondsMean: number;
+    scoreMean: RunScore;
+}
+export interface HarnessSelection {
+    winner: HarnessVariantReport;
+    frontier: ParetoResult<HarnessVariantReport>;
+    reports: HarnessVariantReport[];
+}
+export interface HarnessExperimentResult {
+    results: HarnessRunResult[];
+    selection: HarnessSelection;
+}
+export interface HarnessExperimentConfig {
+    adapter: HarnessAdapter;
+    variants: HarnessVariant[];
+    scenarios: HarnessScenario[];
+    trialsPerScenario?: number;
+    parallelism?: number;
+    weights?: Partial<RunScoreWeights>;
+    objectives?: Array<Objective<HarnessVariantReport>>;
+    score?: (trace: RunTrace, request: HarnessRunRequest) => RunScore | Promise<RunScore>;
+    onResult?: (result: HarnessRunResult) => void | Promise<void>;
+}
+export declare const DEFAULT_HARNESS_OBJECTIVES: Array<Objective<HarnessVariantReport>>;
+export declare function runHarnessExperiment(config: HarnessExperimentConfig): Promise<HarnessExperimentResult>;
+export declare function selectHarnessVariant(results: HarnessRunResult[], objectives?: Array<Objective<HarnessVariantReport>>): HarnessSelection;
+export declare function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVariantReport[];
+//# sourceMappingURL=harness-optimizer.d.ts.map

package/dist/harness-optimizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-optimizer.d.ts","sourceRoot":"","sources":["../src/harness-optimizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAkB,MAAM,UAAU,CAAA;AAC5E,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAqB,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AACpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,MAAM,MAAM,mBAAmB,GAC3B,UAAU,GACV,MAAM,GACN,OAAO,GACP,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,sBAAsB,GACtB,OAAO,CAAA;AAEX,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,aAAa,EAAE,mBAAmB,EAAE,CAAA;IACpC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAA;CAChD;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,WAAW,CAAC,EAAE,iBAAiB,CAAA;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,GAAG,MAAM,CAAA;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,eAAe,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACnD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,eAAe,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,QAAQ,CAAA;IACf,KAAK,EAAE,QAAQ,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,cAAc,CAAA;IACvB,IAAI,EAAE,gBAAgB,EAAE,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,SAAS,EAAE,QAAQ,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,oBAAoB,CAAA;IAC5B,QAAQ,EAAE,YAAY,CAAC,oBAAoB,CAAC,CAAA;IAC5C,OAAO,EAAE,oBAAoB,EAAE,CAAA;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,gBAAgB,EAAE,CAAA;IAC3B,SAAS,EAAE,gBAAgB,CAAA;CAC5B;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,cAAc,EAAE,CAAA;IAC1B,SAAS,EAAE,eAAe,EAAE,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC,CAAA;IACnD,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,KAAK,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IACrF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9D;AAED,eAAO,MAAM,0BAA0B,EAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAK7E,CAAA;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,uBAAuB,CAAC,CAmBlC;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,UAAU,GAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAA8B,GAC9E,gBAAgB,CAQlB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,oBAAoB,EAAE,CAoB3F"}

package/dist/held-out-gate.d.ts

@@ -0,0 +1,135 @@
+/**
+ * HeldOutGate — first-class held-out paired-delta promotion gate.
+ *
+ * Encodes the "honesty override" pattern that lived inline in
+ * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
+ * The optimizer's best-guess is one thing; what we should actually
+ * ship is another. The gate is the line between them.
+ *
+ * A candidate is promoted iff ALL three pass:
+ *
+ *   1. **Productive runs**: the candidate has at least
+ *      `minProductiveRuns` paired observations on items where BOTH
+ *      candidate and baseline produced a real (non-silent) score.
+ *   2. **Paired delta**: the lower bound of the bootstrap CI on the
+ *      median per-item delta (candidate − baseline) on the HOLDOUT
+ *      split is strictly greater than `pairedDeltaThreshold`.
+ *   3. **Overfit gap**: the candidate's gap between search-split
+ *      score and holdout-split score is no worse (more positive)
+ *      than the baseline's gap by more than `overfitGapThreshold`.
+ *      "Better on search, worse on holdout" is the canonical
+ *      overfit pattern; this catches it.
+ *
+ * The decision carries a machine-readable `rejectionCode` plus an
+ * `evidence` block with every number the gate looked at, so the
+ * downstream researcher / paper / dashboard can re-derive the
+ * verdict without re-running.
+ *
+ * See also:
+ *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
+ *   - `src/run-record.ts` for the input row schema
+ *   - `src/reference-replay.ts` for the older, reference-replay-
+ *     specific promotion path (still useful for replay-style evals).
+ */
+import type { RunRecord } from './run-record';
+export type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
+export interface HeldOutGateConfig {
+    /** Minimum number of paired (candidate, baseline) holdout observations
+     *  required before the gate will even consider promoting. Default 3. */
+    minProductiveRuns?: number;
+    /** The bootstrap-CI lower bound on the median paired holdout delta
+     *  must exceed this to promote. Default 0. */
+    pairedDeltaThreshold?: number;
+    /** Maximum allowed worsening of (search − holdout) gap relative to
+     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
+     *  absolute score points more than baseline before rejection). */
+    overfitGapThreshold?: number;
+    /** Stable label of the baseline candidate. Required — paper-grade
+     *  evaluation never compares two unlabelled candidates. */
+    baselineKey: string;
+    /** Confidence level for the bootstrap CI. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    bootstrapResamples?: number;
+    /** Optional deterministic seed for the bootstrap. Default undefined
+     *  (Math.random). */
+    seed?: number;
+    /**
+     * Hard ceiling on the candidate's median per-task USD cost. When the
+     * candidate clears the quality gates (paired-delta + overfit-gap) but
+     * its median cost exceeds this number, the gate rejects with
+     * `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
+     * exactly like the pre-cost gate.
+     *
+     * This exists because "we ship the better prompt" is only an honest
+     * pitch when the better prompt also fits a customer-stated budget.
+     * Cost is read from `RunRecord.costUsd` (already mandatory on every
+     * run) so no new schema is required.
+     */
+    costPerTaskCeiling?: number;
+}
+export interface GateEvidence {
+    /** Number of paired (candidate, baseline) holdout observations used. */
+    productiveRuns: number;
+    /** Median of (candidate − baseline) paired holdout deltas. */
+    medianPairedDelta: number;
+    /** Bootstrap CI on the median paired holdout delta. */
+    pairedCI: {
+        low: number;
+        high: number;
+    };
+    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
+    pairedPValue: number;
+    /** Mean candidate score on the search split (NaN if none). */
+    searchScore: number;
+    /** Mean candidate score on the holdout split (NaN if none). */
+    holdoutScore: number;
+    /** Candidate (search − holdout) gap. */
+    overfitGap: number;
+    /** Baseline (search − holdout) gap. */
+    baselineOverfitGap: number;
+    /** Median per-task USD cost across the candidate's runs. Recorded
+     *  even when no `costPerTaskCeiling` is configured so downstream
+     *  dashboards (intelligence.tangle.tools) can render \$/task per
+     *  generation regardless of gating policy. */
+    medianCandidateCost: number;
+    /** Median per-task USD cost across the baseline runs, for
+     *  symmetric reporting. */
+    medianBaselineCost: number;
+}
+export interface GateDecision {
+    /** Final promote/no-promote verdict. */
+    promote: boolean;
+    /** The candidate that was evaluated. */
+    candidateId: string;
+    /** The baseline it was compared against. */
+    baselineId: string;
+    /** Every number the gate looked at, for audit + paper export. */
+    evidence: GateEvidence;
+    /** Human-readable reason. */
+    reason: string;
+    /** Machine-readable rejection code, or null on promote. */
+    rejectionCode: HeldOutGateRejectionCode | null;
+}
+/**
+ * Held-out paired-delta promotion gate. Construct once with config,
+ * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
+ * baseline) pair. Stateless across calls.
+ */
+export declare class HeldOutGate {
+    private readonly minProductiveRuns;
+    private readonly pairedDeltaThreshold;
+    private readonly overfitGapThreshold;
+    private readonly baselineKey;
+    private readonly confidence;
+    private readonly resamples;
+    private readonly seed?;
+    private readonly costPerTaskCeiling?;
+    constructor(config: HeldOutGateConfig);
+    /** Decide whether `candidate` should replace `baseline`. Pairing
+     *  is by (experimentId, seed) — identical experiment + seed pairs
+     *  the candidate run with the matching baseline run. Pairs without
+     *  a holdout score on both sides are dropped. */
+    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
+}
+//# sourceMappingURL=held-out-gate.d.ts.map

package/dist/held-out-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"held-out-gate.d.ts","sourceRoot":"","sources":["../src/held-out-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG7C,MAAM,MAAM,wBAAwB,GAChC,UAAU,GACV,gBAAgB,GAChB,aAAa,GACb,cAAc,CAAA;AAElB,MAAM,WAAW,iBAAiB;IAChC;4EACwE;IACxE,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;kDAC8C;IAC9C,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B;;sEAEkE;IAClE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;+DAC2D;IAC3D,WAAW,EAAE,MAAM,CAAA;IACnB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,yCAAyC;IACzC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B;yBACqB;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;;;;;;;;OAWG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,YAAY;IAC3B,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAA;IACtB,8DAA8D;IAC9D,iBAAiB,EAAE,MAAM,CAAA;IACzB,uDAAuD;IACvD,QAAQ,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACvC,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAA;IACpB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAA;IACnB,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAA;IACpB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAA;IAClB,uCAAuC;IACvC,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;kDAG8C;IAC9C,mBAAmB,EAAE,MAAM,CAAA;IAC3B;+BAC2B;IAC3B,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,OAAO,EAAE,OAAO,CAAA;IAChB,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAA;IACnB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAA;IAClB,iEAAiE;IACjE,QAAQ,EAAE,YAAY,CAAA;IACtB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAA;IACd,2DAA2D;IAC3D,aAAa,EAAE,wBAAwB,GAAG,IAAI,CAAA;CAC/C;AAED;;;;GAIG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAQ;gBAEhC,MAAM,EAAE,iBAAiB;IAoBrC;;;qDAGiD;IACjD,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,YAAY;CAqJtE"}

package/dist/hosted/client.d.ts

@@ -0,0 +1,73 @@
+/**
+ * # Hosted-tier ingest client.
+ *
+ * Ships eval-run events + trace spans to any orchestrator (ours, a
+ * partner's self-hosted one, or a future open implementation) that
+ * speaks the wire format in `./types.ts`.
+ *
+ * Three modes:
+ *   - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
+ *     the client appends the versioned `/v1/ingest/...` path itself; a trailing
+ *     `/v1` on the endpoint is tolerated and normalized away). We handle ingest
+ *     + storage + dashboard.
+ *   - **Self-hosted:** point at whatever URL runs the reference receiver
+ *     from `examples/hosted-ingest-server/`.
+ *   - **Off (default):** when `hostedTenant` is unset, nothing is sent.
+ *     Everything stays local.
+ */
+import { type EvalRunEvent, type HostedWireVersion, type IngestResponse, type TraceSpanEvent } from './types';
+export interface HostedTenant {
+    /** Orchestrator endpoint base URL (no trailing slash). Required. */
+    endpoint: string;
+    /** Bearer token issued by the orchestrator. Required. */
+    apiKey: string;
+    /** Tenant id — the orchestrator's primary key for this consumer. Required. */
+    tenantId: string;
+    /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
+    fetchImpl?: typeof fetch;
+    /** Per-call timeout in ms. Default 30s. */
+    timeoutMs?: number;
+    /** Retries on 5xx / network errors. Default 2. */
+    retries?: number;
+}
+export interface HostedClient {
+    ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
+    ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    readonly tenant: HostedTenant;
+    readonly wireVersion: HostedWireVersion;
+}
+export declare function createHostedClient(tenant: HostedTenant): HostedClient;
+/**
+ * Build a `HostedClient` from environment, or `undefined` when ingest is not
+ * configured — the canonical, fail-soft wiring every product uses so eval-run +
+ * trace provenance lands in the Intelligence dashboard with ONE call:
+ *
+ *   const hosted = hostedClientFromEnv()
+ *   // ...run the loop...
+ *   await emitLoopProvenance({ ..., hostedClient: hosted })  // no-op if undefined
+ *
+ * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
+ * missing — so a product wires the ship call unconditionally and it stays a
+ * no-op until the env is set. Env precedence:
+ *   - endpoint:  `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
+ *   - apiKey:    `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
+ *   - tenantId:  `TANGLE_TENANT_ID`
+ * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
+ * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
+ */
+/**
+ * Build a {@link HostedTenant} config from env — the input `selfImprove`'s
+ * `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
+ * as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
+ * endpoint / apiKey / tenantId is missing, so a product wires
+ * `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
+ * and it stays off until the env is set.
+ */
+export declare function hostedTenantFromEnv(overrides?: Partial<HostedTenant> & {
+    env?: Record<string, string | undefined>;
+}): HostedTenant | undefined;
+export declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
+    env?: Record<string, string | undefined>;
+}): HostedClient | undefined;
+//# sourceMappingURL=client.d.ts.map

package/dist/hosted/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/hosted/client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,iBAAiB,EAEtB,KAAK,cAAc,EAEnB,KAAK,cAAc,EACpB,MAAM,SAAS,CAAA;AAEhB,MAAM,WAAW,YAAY;IAC3B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAA;IAChB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAA;IACd,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACpF,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACxF,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACvF,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAA;CACxC;AAuED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CAuBrE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,GAAE,OAAO,CAAC,YAAY,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;CAAO,GACnF,YAAY,GAAG,SAAS,CAe1B;AAED,wBAAgB,mBAAmB,CACjC,SAAS,GAAE,OAAO,CAAC,YAAY,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;CAAO,GACnF,YAAY,GAAG,SAAS,CAG1B"}

package/dist/hosted/from-env.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * `hostedTenantFromEnv` is the env→HostedTenant map every product passes to
+ * `selfImprove({ hostedTenant })`. It must fail SOFT (undefined, not throw) when
+ * unconfigured, so a product wires it unconditionally and it stays off until the
+ * env is set.
+ */
+export {};
+//# sourceMappingURL=from-env.test.d.ts.map

package/dist/hosted/from-env.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"from-env.test.d.ts","sourceRoot":"","sources":["../../src/hosted/from-env.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}