@elsium-ai/testing 0.9.1 → 0.11.0

package/dist/multi-turn.d.ts

@@ -0,0 +1,99 @@
+import type { Message } from '@elsium-ai/core';
+import type { ToolExecutionResult } from '@elsium-ai/tools';
+import type { ToolCallEntry } from './tool-assertions';
+export type TurnAssertion = {
+    type: 'response_contains';
+    value: string;
+} | {
+    type: 'response_not_contains';
+    value: string;
+} | {
+    type: 'response_matches';
+    pattern: string;
+    flags?: string;
+} | {
+    type: 'tool_called';
+    name: string;
+    times?: number;
+} | {
+    type: 'tool_not_called';
+    name: string;
+} | {
+    type: 'tool_args_match';
+    name: string;
+    args: Record<string, unknown>;
+} | {
+    type: 'max_iterations';
+    value: number;
+} | {
+    type: 'max_latency_ms';
+    value: number;
+} | {
+    type: 'custom';
+    name: string;
+    fn: (result: TurnResult) => boolean;
+};
+export interface TurnResult {
+    turnIndex: number;
+    name?: string;
+    input: string;
+    output: string;
+    toolCalls: ToolCallEntry[];
+    usage: {
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalTokens: number;
+        totalCost: number;
+        iterations: number;
+    };
+    durationMs: number;
+    assertions: Array<{
+        type: string;
+        passed: boolean;
+        message: string;
+    }>;
+    passed: boolean;
+}
+export interface ConversationTurn {
+    role: 'user';
+    content: string | ((history: TurnResult[]) => string);
+    assertions?: TurnAssertion[];
+    name?: string;
+}
+interface AgentResultLike {
+    message: Message;
+    usage: {
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        totalTokens: number;
+        totalCost: number;
+        iterations: number;
+    };
+    toolCalls: Array<{
+        name: string;
+        arguments: Record<string, unknown>;
+        result: ToolExecutionResult;
+    }>;
+    traceId: string;
+}
+export interface ConversationScenarioConfig {
+    name: string;
+    description?: string;
+    turns: ConversationTurn[];
+    runner: (messages: Message[]) => Promise<AgentResultLike>;
+    tags?: string[];
+}
+export interface ConversationResult {
+    name: string;
+    passed: boolean;
+    turns: TurnResult[];
+    totalDurationMs: number;
+    totalTokens: number;
+    totalCost: number;
+    totalToolCalls: number;
+    tags: string[];
+}
+export declare function runConversation(config: ConversationScenarioConfig): Promise<ConversationResult>;
+export declare function formatConversationReport(result: ConversationResult): string;
+export {};
+//# sourceMappingURL=multi-turn.d.ts.map

package/dist/multi-turn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi-turn.d.ts","sourceRoot":"","sources":["../src/multi-turn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAE9C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAGtD,MAAM,MAAM,aAAa,GACtB;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAA;CAAE,CAAA;AAExE,MAAM,WAAW,UAAU;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,aAAa,EAAE,CAAA;IAC1B,KAAK,EAAE;QACN,gBAAgB,EAAE,MAAM,CAAA;QACxB,iBAAiB,EAAE,MAAM,CAAA;QACzB,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,UAAU,EAAE,MAAM,CAAA;KAClB,CAAA;IACD,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACrE,MAAM,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,MAAM,CAAC,CAAA;IACrD,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;CACb;AAED,UAAU,eAAe;IACxB,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,EAAE;QACN,gBAAgB,EAAE,MAAM,CAAA;QACxB,iBAAiB,EAAE,MAAM,CAAA;QACzB,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,UAAU,EAAE,MAAM,CAAA;KAClB,CAAA;IACD,SAAS,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAClC,MAAM,EAAE,mBAAmB,CAAA;KAC3B,CAAC,CAAA;IACF,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,gBAAgB,EAAE,CAAA;IACzB,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,eAAe,CAAC,CAAA;IACzD,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,EAAE,MAAM,EAAE,CAAA;CACd;AAyFD,wBAAsB,eAAe,CACpC,MAAM,EAAE,0BAA0B,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA+D7B;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,kBAAkB,GAAG,MAAM,CA2B3E"}

package/dist/red-team.d.ts

@@ -0,0 +1,69 @@
+import type { Message } from '@elsium-ai/core';
+import type { ToolExecutionResult } from '@elsium-ai/tools';
+export type AttackCategory = 'prompt_injection' | 'jailbreak' | 'data_extraction' | 'persona_override' | 'instruction_bypass';
+export interface AttackProbe {
+    name: string;
+    category: AttackCategory;
+    input: string;
+    detectCompromise: (output: string) => boolean;
+    severity: 'low' | 'medium' | 'high' | 'critical';
+}
+export interface MultiTurnAttackProbe {
+    name: string;
+    category: AttackCategory;
+    turns: Array<string | ((history: string[]) => string)>;
+    detectCompromise: (outputs: string[]) => boolean;
+    severity: 'low' | 'medium' | 'high' | 'critical';
+}
+interface AgentResultLike {
+    message: Message;
+    toolCalls: Array<{
+        name: string;
+        arguments: Record<string, unknown>;
+        result: ToolExecutionResult;
+    }>;
+}
+export interface RedTeamConfig {
+    name: string;
+    runner: (input: string) => Promise<string>;
+    probes?: AttackProbe[];
+    categories?: AttackCategory[];
+    concurrency?: number;
+    multiTurnProbes?: MultiTurnAttackProbe[];
+    multiTurnRunner?: (messages: Message[]) => Promise<AgentResultLike>;
+}
+export interface ProbeResult {
+    probe: AttackProbe;
+    output: string;
+    compromised: boolean;
+    durationMs: number;
+    error?: string;
+}
+export interface MultiTurnProbeResult {
+    probe: MultiTurnAttackProbe;
+    outputs: string[];
+    compromised: boolean;
+    durationMs: number;
+    error?: string;
+}
+export interface RedTeamResult {
+    name: string;
+    total: number;
+    passed: number;
+    compromised: number;
+    errored: number;
+    results: ProbeResult[];
+    multiTurnResults: MultiTurnProbeResult[];
+    byCategory: Record<AttackCategory, {
+        total: number;
+        compromised: number;
+    }>;
+    score: number;
+    durationMs: number;
+}
+export declare function getBuiltInMultiTurnProbes(categories?: AttackCategory[]): MultiTurnAttackProbe[];
+export declare function getBuiltInProbes(categories?: AttackCategory[]): AttackProbe[];
+export declare function runRedTeam(config: RedTeamConfig): Promise<RedTeamResult>;
+export declare function formatRedTeamReport(result: RedTeamResult): string;
+export {};
+//# sourceMappingURL=red-team.d.ts.map

package/dist/red-team.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"red-team.d.ts","sourceRoot":"","sources":["../src/red-team.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAE9C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAE3D,MAAM,MAAM,cAAc,GACvB,kBAAkB,GAClB,WAAW,GACX,iBAAiB,GACjB,kBAAkB,GAClB,oBAAoB,CAAA;AAEvB,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,cAAc,CAAA;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAA;IAC7C,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAA;CAChD;AAED,MAAM,WAAW,oBAAoB;IACpC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,cAAc,CAAA;IACxB,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC,CAAC,CAAA;IACtD,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,OAAO,CAAA;IAChD,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAA;CAChD;AAED,UAAU,eAAe;IACxB,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAClC,MAAM,EAAE,mBAAmB,CAAA;KAC3B,CAAC,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;IAC1C,MAAM,CAAC,EAAE,WAAW,EAAE,CAAA;IACtB,UAAU,CAAC,EAAE,cAAc,EAAE,CAAA;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,oBAAoB,EAAE,CAAA;IACxC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,eAAe,CAAC,CAAA;CACnE;AAED,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,WAAW,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,OAAO,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,oBAAoB,CAAA;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,WAAW,EAAE,OAAO,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,WAAW,EAAE,CAAA;IACtB,gBAAgB,EAAE,oBAAoB,EAAE,CAAA;IACxC,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC1E,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;CAClB;AA8gBD,wBAAgB,yBAAyB,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,oBAAoB,EAAE,CAG/F;AAUD,wBAAgB,gBAAgB,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,CAG7E;AAoID,wBAAsB,UAAU,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAoB9E;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAoCjE"}

package/dist/tool-assertions.d.ts

@@ -0,0 +1,46 @@
+import type { ToolExecutionResult } from '@elsium-ai/tools';
+import type { EvalCriterion } from './eval';
+export type ToolCallEntry = {
+    name: string;
+    arguments: Record<string, unknown>;
+    result: ToolExecutionResult;
+};
+export type ToolAssertion = {
+    type: 'called';
+    name: string;
+    times?: number;
+} | {
+    type: 'not_called';
+    name: string;
+} | {
+    type: 'called_with';
+    name: string;
+    args: Record<string, unknown>;
+    partial?: boolean;
+} | {
+    type: 'called_in_order';
+    names: string[];
+} | {
+    type: 'all_succeeded';
+} | {
+    type: 'none_failed';
+} | {
+    type: 'call_count';
+    min?: number;
+    max?: number;
+} | {
+    type: 'no_repeated_calls';
+    name?: string;
+} | {
+    type: 'custom';
+    name: string;
+    fn: (calls: ToolCallEntry[]) => boolean;
+};
+export interface ToolAssertionResult {
+    type: string;
+    passed: boolean;
+    message: string;
+}
+export declare function assertToolCalls(calls: ToolCallEntry[], assertions: ToolAssertion[]): ToolAssertionResult[];
+export declare function toolCallsToEvalCriteria(assertions: ToolAssertion[], calls: ToolCallEntry[]): EvalCriterion[];
+//# sourceMappingURL=tool-assertions.d.ts.map

package/dist/tool-assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-assertions.d.ts","sourceRoot":"","sources":["../src/tool-assertions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAE3C,MAAM,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,mBAAmB,CAAA;CAC3B,CAAA;AAED,MAAM,MAAM,aAAa,GACtB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GACvF;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,GACzB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,OAAO,CAAA;CAAE,CAAA;AAE5E,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA+KD,wBAAgB,eAAe,CAC9B,KAAK,EAAE,aAAa,EAAE,EACtB,UAAU,EAAE,aAAa,EAAE,GACzB,mBAAmB,EAAE,CAuBvB;AAED,wBAAgB,uBAAuB,CACtC,UAAU,EAAE,aAAa,EAAE,EAC3B,KAAK,EAAE,aAAa,EAAE,GACpB,aAAa,EAAE,CASjB"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@elsium-ai/testing",
-	"version": "0.9.1",
+	"version": "0.11.0",
 	"description": "Testing utilities, mock providers, fixtures, and eval framework for ElsiumAI",
 	"license": "MIT",
 	"author": "Eric Utrera <ebutrera9103@gmail.com>",
@@ -26,10 +26,10 @@
 		"dev": "bun --watch src/index.ts"
 	},
 	"dependencies": {
-		"@elsium-ai/core": "^0.9.1",
-		"@elsium-ai/gateway": "^0.9.1",
-		"@elsium-ai/agents": "^0.9.1",
-		"@elsium-ai/tools": "^0.9.1"
+		"@elsium-ai/core": "^0.11.0",
+		"@elsium-ai/gateway": "^0.11.0",
+		"@elsium-ai/agents": "^0.11.0",
+		"@elsium-ai/tools": "^0.11.0"
 	},
 	"devDependencies": {
 		"typescript": "^5.7.0"