@elsium-ai/testing 0.1.6

package/dist/determinism.d.ts

@@ -0,0 +1,28 @@
+export interface DeterminismResult {
+    deterministic: boolean;
+    runs: number;
+    uniqueOutputs: number;
+    outputs: string[];
+    variance: number;
+}
+export interface StabilityResult {
+    stable: boolean;
+    runs: number;
+    uniqueOutputs: number;
+    outputs: Array<{
+        output: string;
+        timestamp: number;
+    }>;
+    variance: number;
+}
+export declare function assertDeterministic(fn: (seed?: number) => Promise<string>, options?: {
+    runs?: number;
+    seed?: number;
+    tolerance?: number;
+}): Promise<DeterminismResult>;
+export declare function assertStable(fn: (seed?: number) => Promise<string>, options?: {
+    intervalMs?: number;
+    runs?: number;
+    seed?: number;
+}): Promise<StabilityResult>;
+//# sourceMappingURL=determinism.d.ts.map

package/dist/determinism.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"determinism.d.ts","sourceRoot":"","sources":["../src/determinism.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,iBAAiB;IACjC,aAAa,EAAE,OAAO,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,OAAO,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACrD,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,wBAAsB,mBAAmB,CACxC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EACtC,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5D,OAAO,CAAC,iBAAiB,CAAC,CA8B5B;AAED,wBAAsB,YAAY,CACjC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EACtC,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAAC,eAAe,CAAC,CA0B1B"}

package/dist/eval.d.ts

@@ -0,0 +1,85 @@
+export interface EvalCase {
+    name: string;
+    input: string;
+    expected?: string;
+    criteria?: EvalCriterion[];
+    tags?: string[];
+}
+export type LLMJudge = (prompt: string) => Promise<{
+    score: number;
+    reasoning: string;
+}>;
+export type EvalCriterion = {
+    type: 'contains';
+    value: string;
+    caseSensitive?: boolean;
+} | {
+    type: 'not_contains';
+    value: string;
+    caseSensitive?: boolean;
+} | {
+    type: 'matches';
+    pattern: string;
+    flags?: string;
+} | {
+    type: 'length_min';
+    value: number;
+} | {
+    type: 'length_max';
+    value: number;
+} | {
+    type: 'json_valid';
+} | {
+    type: 'json_matches';
+    schema: Record<string, unknown>;
+} | {
+    type: 'custom';
+    name: string;
+    fn: (output: string) => boolean;
+} | {
+    type: 'llm_judge';
+    prompt: string;
+    judge: LLMJudge;
+    threshold?: number;
+} | {
+    type: 'semantic_similarity';
+    reference: string;
+    threshold?: number;
+} | {
+    type: 'factual_accuracy';
+    facts: string[];
+    threshold?: number;
+};
+export interface EvalResult {
+    name: string;
+    passed: boolean;
+    score: number;
+    criteria: CriterionResult[];
+    input: string;
+    output: string;
+    durationMs: number;
+    tags: string[];
+}
+export interface CriterionResult {
+    type: string;
+    passed: boolean;
+    message: string;
+}
+export interface EvalSuiteConfig {
+    name: string;
+    cases: EvalCase[];
+    runner: (input: string) => Promise<string>;
+    concurrency?: number;
+}
+export interface EvalSuiteResult {
+    name: string;
+    total: number;
+    passed: number;
+    failed: number;
+    score: number;
+    results: EvalResult[];
+    durationMs: number;
+}
+export declare function runEvalSuite(config: EvalSuiteConfig): Promise<EvalSuiteResult>;
+export declare function formatEvalReport(result: EvalSuiteResult): string;
+//# sourceMappingURL=eval.d.ts.map

package/dist/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,aAAa,EAAE,CAAA;IAC1B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CACf;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAA;AAExF,MAAM,MAAM,aAAa,GACtB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,GACtB;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAA;CAAE,GACjE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1E;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GACtE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAEpE,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,eAAe,EAAE,CAAA;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,EAAE,CAAA;CACd;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,QAAQ,EAAE,CAAA;IACjB,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,UAAU,EAAE,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;CAClB;AAkUD,wBAAsB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CA+BpF;AAID,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA0BhE"}

package/dist/fixtures.d.ts

@@ -0,0 +1,31 @@
+import { type MockProvider, type MockResponseConfig } from './mock-provider';
+export interface FixtureEntry {
+    request: {
+        messages: Array<{
+            role: string;
+            content: string;
+        }>;
+        model?: string;
+        system?: string;
+    };
+    response: MockResponseConfig;
+    timestamp?: string;
+}
+export interface Fixture {
+    readonly name: string;
+    readonly entries: FixtureEntry[];
+    toProvider(options?: {
+        matching?: 'sequential' | 'request-hash';
+    }): MockProvider;
+    toJSON(): string;
+}
+export declare function createFixture(name: string, entries: FixtureEntry[]): Fixture;
+export declare function loadFixture(json: string): Fixture;
+export interface FixtureRecorder {
+    wrap(provider: MockProvider): MockProvider;
+    getEntries(): FixtureEntry[];
+    toFixture(name: string): Fixture;
+    clear(): void;
+}
+export declare function createRecorder(): FixtureRecorder;
+//# sourceMappingURL=fixtures.d.ts.map

package/dist/fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,kBAAkB,EAAgB,MAAM,iBAAiB,CAAA;AAE1F,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE;QACR,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QAClD,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;KACf,CAAA;IACD,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAA;IAChC,UAAU,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,YAAY,GAAG,cAAc,CAAA;KAAE,GAAG,YAAY,CAAA;IAChF,MAAM,IAAI,MAAM,CAAA;CAChB;AAOD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAyD5E;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGjD;AAID,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,QAAQ,EAAE,YAAY,GAAG,YAAY,CAAA;IAC1C,UAAU,IAAI,YAAY,EAAE,CAAA;IAC5B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IAChC,KAAK,IAAI,IAAI,CAAA;CACb;AAED,wBAAgB,cAAc,IAAI,eAAe,CAgDhD"}

package/dist/index.d.ts

@@ -0,0 +1,19 @@
+export { mockProvider } from './mock-provider';
+export type { MockProvider, MockProviderOptions, MockResponseConfig } from './mock-provider';
+export { createFixture, loadFixture, createRecorder } from './fixtures';
+export type { Fixture, FixtureEntry, FixtureRecorder } from './fixtures';
+export { runEvalSuite, formatEvalReport } from './eval';
+export type { EvalCase, EvalCriterion, EvalResult, CriterionResult, EvalSuiteConfig, EvalSuiteResult, LLMJudge, } from './eval';
+export { createSnapshotStore, hashOutput, testSnapshot } from './snapshot';
+export type { PromptSnapshot, SnapshotStore, SnapshotTestResult } from './snapshot';
+export { createPromptRegistry, definePrompt } from './prompts';
+export type { PromptDefinition, PromptDiff, DiffLine, PromptRegistry } from './prompts';
+export { createRegressionSuite } from './regression';
+export type { RegressionBaseline, RegressionResult, RegressionDetail, RegressionSuite, } from './regression';
+export { createReplayRecorder, createReplayPlayer } from './replay';
+export type { ReplayEntry, ReplayRecorder, ReplayPlayer } from './replay';
+export { createPinStore, pinOutput } from './pinning';
+export type { Pin, PinStore, PinResult } from './pinning';
+export { assertDeterministic, assertStable } from './determinism';
+export type { DeterminismResult, StabilityResult } from './determinism';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAG5F,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACvE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAGxE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAA;AACvD,YAAY,EACX,QAAQ,EACR,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,eAAe,EACf,QAAQ,GACR,MAAM,QAAQ,CAAA;AAGf,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC1E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAGnF,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAC9D,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAGvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,YAAY,EACX,kBAAkB,EAClB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GACf,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AACnE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAGzE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrD,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAGzD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA"}