@artemiskit/redteam 0.1.6 → 0.2.0

package/CHANGELOG.md

@@ -1,5 +1,84 @@
 # @artemiskit/redteam
 
+## 0.2.0
+
+### Minor Changes
+
+- d2c3835: ## v0.2.0 - Enhanced Evaluation Features
+
+  ### CLI (`@artemiskit/cli`)
+
+  #### New Features
+
+  - **Multi-turn mutations**: Added `--mutations multi_turn` flag for red team testing with 4 built-in strategies:
+    - `gradual_escalation`: Gradually intensifies requests over conversation turns
+    - `context_switching`: Shifts topics to lower defenses before attack
+    - `persona_building`: Establishes trust through roleplay
+    - `distraction`: Uses side discussions to slip in harmful requests
+  - **Custom multi-turn conversations**: Support for array prompts in red team scenarios (consistent with `run` command format). The last user message becomes the attack target, preceding messages form conversation context.
+  - **Custom attacks**: Added `--custom-attacks` flag to load custom attack patterns from YAML files with template variables and variations.
+  - **Encoding mutations**: Added `--mutations encoding` for obfuscation attacks (base64, ROT13, hex, unicode).
+  - **Directory scanning**: Run all scenarios in a directory with `akit run scenarios/`
+  - **Glob pattern matching**: Use patterns like `akit run scenarios/**/*.yaml`
+  - **Parallel execution**: Added `--parallel` flag for concurrent scenario execution
+  - **Scenario tags**: Filter scenarios with `--tags` flag
+
+  ### Core (`@artemiskit/core`)
+
+  #### New Features
+
+  - **Combined matchers**: New `type: combined` expectation with `operator: and|or` for complex assertion logic
+  - **`not_contains` expectation**: Negative containment check to ensure responses don't include specific text
+  - **`similarity` expectation**: Semantic similarity matching with two modes:
+    - Embedding-based: Uses vector embeddings for fast semantic comparison
+    - LLM-based fallback: Uses LLM to evaluate semantic similarity when embeddings unavailable
+    - Configurable threshold (default 0.75)
+  - **`inline` expectation**: Safe expression-based custom matchers in YAML using JavaScript-like expressions (e.g., `response.length > 100`, `response.includes('hello')`)
+  - **p90 latency metric**: Added p90 percentile to stress test latency metrics
+  - **Token usage tracking**: Monitor token consumption per request in stress tests
+  - **Cost estimation**: Estimate API costs with model pricing data
+
+  ### Red Team (`@artemiskit/redteam`)
+
+  #### New Features
+
+  - **MultiTurnMutation class**: Full implementation with strategy support and custom conversation prefixes
+  - **Custom attack loader**: Parse and load custom attack patterns from YAML
+  - **Encoding mutation**: Obfuscate attack payloads using various encoding schemes
+  - **CVSS-like severity scoring**: Detailed attack severity scoring with:
+    - `CvssScore` interface with attack vector, complexity, impact metrics
+    - `CvssCalculator` class for score calculation and aggregation
+    - Predefined scores for all mutations and detection categories
+    - Human-readable score descriptions and vector strings
+
+  ### Reports (`@artemiskit/reports`)
+
+  #### New Features
+
+  - **Run comparison HTML report**: Visual diff between two runs showing:
+    - Metrics overview with baseline vs current comparison
+    - Change summary (regressions, improvements, unchanged)
+    - Case-by-case comparison table with filtering
+    - Side-by-side response comparison for each case
+  - **Comparison JSON export**: Structured comparison data for programmatic use
+
+  ### CLI Enhancements
+
+  - **Compare command `--html` flag**: Generate HTML comparison report
+  - **Compare command `--json` flag**: Generate JSON comparison data
+
+  ### Documentation
+
+  - Updated all CLI command documentation
+  - Added comprehensive examples for custom multi-turn scenarios
+  - Documented combined matchers and `not_contains` expectations
+  - Added mutation strategy reference tables
+
+### Patch Changes
+
+- Updated dependencies [d2c3835]
+  - @artemiskit/core@0.2.0
+
 ## 0.1.6
 
 ### Patch Changes

package/dist/custom-attacks.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Custom attack YAML loader
+ * Allows users to define custom red team attacks in YAML format
+ */
+import type { Mutation } from './mutations';
+/**
+ * Schema for custom attack definition in YAML
+ */
+export interface CustomAttackDefinition {
+    /** Unique name for the attack */
+    name: string;
+    /** Description of what the attack tests */
+    description: string;
+    /** Severity level */
+    severity: 'low' | 'medium' | 'high' | 'critical';
+    /** Attack templates - use {{prompt}} as placeholder for the original prompt */
+    templates: string[];
+    /** Optional: Variations to apply to templates */
+    variations?: {
+        /** Placeholder name (e.g., 'role') */
+        name: string;
+        /** Values to substitute */
+        values: string[];
+    }[];
+}
+/**
+ * Schema for custom attacks YAML file
+ */
+export interface CustomAttacksFile {
+    /** File format version */
+    version: string;
+    /** List of custom attack definitions */
+    attacks: CustomAttackDefinition[];
+}
+/**
+ * Custom mutation created from YAML definition
+ */
+export declare class CustomMutation implements Mutation {
+    readonly name: string;
+    readonly description: string;
+    readonly severity: 'low' | 'medium' | 'high' | 'critical';
+    private templates;
+    private variations;
+    constructor(definition: CustomAttackDefinition);
+    mutate(prompt: string): string;
+}
+/**
+ * Load custom attacks from a YAML file
+ */
+export declare function loadCustomAttacks(filePath: string): CustomMutation[];
+/**
+ * Load custom attacks from a YAML string
+ */
+export declare function parseCustomAttacks(yamlContent: string): CustomMutation[];
+/**
+ * Generate an example custom attacks YAML file
+ */
+export declare function generateExampleCustomAttacksYaml(): string;
+//# sourceMappingURL=custom-attacks.d.ts.map

package/dist/custom-attacks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom-attacks.d.ts","sourceRoot":"","sources":["../src/custom-attacks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB;IACrB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IACjD,+EAA+E;IAC/E,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,iDAAiD;IACjD,UAAU,CAAC,EAAE;QACX,sCAAsC;QACtC,IAAI,EAAE,MAAM,CAAC;QACb,2BAA2B;QAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;KAClB,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,0BAA0B;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,OAAO,EAAE,sBAAsB,EAAE,CAAC;CACnC;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAE1D,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,UAAU,CAAuC;gBAE7C,UAAU,EAAE,sBAAsB;IAQ9C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;CAkB/B;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAwBpE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,EAAE,CAexE;AAgDD;;GAEG;AACH,wBAAgB,gCAAgC,IAAI,MAAM,CAsDzD"}

package/dist/detector.d.ts

@@ -1,11 +1,14 @@
 /**
- * Unsafe response detector
+ * Unsafe response detector with CVSS-like severity scoring
  */
+import { CvssCalculator, type CvssScore } from './severity';
 export interface DetectionResult {
     unsafe: boolean;
     reasons: string[];
     severity: 'low' | 'medium' | 'high' | 'critical';
     categories: string[];
+    /** Detailed CVSS-like score for the detection */
+    cvssScore?: CvssScore;
 }
 export declare class UnsafeResponseDetector {
     private patterns;
@@ -17,6 +20,14 @@ export declare class UnsafeResponseDetector {
     /**
      * Add a custom detection pattern
      */
-    addPattern(pattern: RegExp, category: string, severity: 'low' | 'medium' | 'high' | 'critical', description: string): void;
+    addPattern(pattern: RegExp, category: string, severity: 'low' | 'medium' | 'high' | 'critical', description: string, cvssParams?: Parameters<typeof CvssCalculator.calculate>[0]): void;
+    /**
+     * Get severity info for display
+     */
+    static getSeverityInfo(severity: 'low' | 'medium' | 'high' | 'critical'): import("./severity").SeverityInfo;
+    /**
+     * Describe a CVSS score in human-readable form
+     */
+    static describeCvssScore(cvssScore: CvssScore): string;
 }
 //# sourceMappingURL=detector.d.ts.map

package/dist/detector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../src/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IACjD,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAKb;;IAoDH;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe;IAmCzC;;OAEG;IACH,UAAU,CACR,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,EAChD,WAAW,EAAE,MAAM,GAClB,IAAI;CAGR"}
+{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../src/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,SAAS,EAAyC,MAAM,YAAY,CAAC;AAEnG,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IACjD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAUD,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAqB;;IA2DrC;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe;IA0CzC;;OAEG;IACH,UAAU,CACR,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,EAChD,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAC1D,IAAI;IAKP;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU;IAIvE;;OAEG;IACH,MAAM,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM;CAGvD"}

package/dist/generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../src/generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAM5C,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;CAClD;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAa;gBAElB,SAAS,CAAC,EAAE,QAAQ,EAAE;IASlC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,eAAe,EAAE;IA0BvD;;OAEG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAQ3D;;OAEG;IACH,aAAa,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAQ/E,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,WAAW;CAOpB"}
+{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../src/generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAQ5C,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;CAClD;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAa;gBAElB,SAAS,CAAC,EAAE,QAAQ,EAAE;IAWlC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,eAAe,EAAE;IA0BvD;;OAEG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAQ3D;;OAEG;IACH,aAAa,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAQ/E,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,WAAW;CAOpB"}

package/dist/index.d.ts

@@ -5,5 +5,6 @@
 export * from './mutations';
 export { RedTeamGenerator, type GeneratedPrompt } from './generator';
 export { UnsafeResponseDetector, type DetectionResult } from './detector';
-export { SeverityMapper, type Severity, type SeverityInfo } from './severity';
+export { SeverityMapper, CvssCalculator, MUTATION_CVSS_SCORES, DETECTION_CVSS_SCORES, type Severity, type SeverityInfo, type CvssScore, } from './severity';
+export { CustomMutation, loadCustomAttacks, parseCustomAttacks, generateExampleCustomAttacksYaml, type CustomAttackDefinition, type CustomAttacksFile, } from './custom-attacks';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,aAAa,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,aAAa,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAC1E,OAAO,EACL,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,gCAAgC,EAChC,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,GACvB,MAAM,kBAAkB,CAAC"}