@artemiskit/reports 0.1.6 → 0.2.0

package/CHANGELOG.md

@@ -1,5 +1,84 @@
 # @artemiskit/reports
 
+## 0.2.0
+
+### Minor Changes
+
+- d2c3835: ## v0.2.0 - Enhanced Evaluation Features
+
+  ### CLI (`@artemiskit/cli`)
+
+  #### New Features
+
+  - **Multi-turn mutations**: Added `--mutations multi_turn` flag for red team testing with 4 built-in strategies:
+    - `gradual_escalation`: Gradually intensifies requests over conversation turns
+    - `context_switching`: Shifts topics to lower defenses before attack
+    - `persona_building`: Establishes trust through roleplay
+    - `distraction`: Uses side discussions to slip in harmful requests
+  - **Custom multi-turn conversations**: Support for array prompts in red team scenarios (consistent with `run` command format). The last user message becomes the attack target, preceding messages form conversation context.
+  - **Custom attacks**: Added `--custom-attacks` flag to load custom attack patterns from YAML files with template variables and variations.
+  - **Encoding mutations**: Added `--mutations encoding` for obfuscation attacks (base64, ROT13, hex, unicode).
+  - **Directory scanning**: Run all scenarios in a directory with `akit run scenarios/`
+  - **Glob pattern matching**: Use patterns like `akit run scenarios/**/*.yaml`
+  - **Parallel execution**: Added `--parallel` flag for concurrent scenario execution
+  - **Scenario tags**: Filter scenarios with `--tags` flag
+
+  ### Core (`@artemiskit/core`)
+
+  #### New Features
+
+  - **Combined matchers**: New `type: combined` expectation with `operator: and|or` for complex assertion logic
+  - **`not_contains` expectation**: Negative containment check to ensure responses don't include specific text
+  - **`similarity` expectation**: Semantic similarity matching with two modes:
+    - Embedding-based: Uses vector embeddings for fast semantic comparison
+    - LLM-based fallback: Uses LLM to evaluate semantic similarity when embeddings unavailable
+    - Configurable threshold (default 0.75)
+  - **`inline` expectation**: Safe expression-based custom matchers in YAML using JavaScript-like expressions (e.g., `response.length > 100`, `response.includes('hello')`)
+  - **p90 latency metric**: Added p90 percentile to stress test latency metrics
+  - **Token usage tracking**: Monitor token consumption per request in stress tests
+  - **Cost estimation**: Estimate API costs with model pricing data
+
+  ### Red Team (`@artemiskit/redteam`)
+
+  #### New Features
+
+  - **MultiTurnMutation class**: Full implementation with strategy support and custom conversation prefixes
+  - **Custom attack loader**: Parse and load custom attack patterns from YAML
+  - **Encoding mutation**: Obfuscate attack payloads using various encoding schemes
+  - **CVSS-like severity scoring**: Detailed attack severity scoring with:
+    - `CvssScore` interface with attack vector, complexity, impact metrics
+    - `CvssCalculator` class for score calculation and aggregation
+    - Predefined scores for all mutations and detection categories
+    - Human-readable score descriptions and vector strings
+
+  ### Reports (`@artemiskit/reports`)
+
+  #### New Features
+
+  - **Run comparison HTML report**: Visual diff between two runs showing:
+    - Metrics overview with baseline vs current comparison
+    - Change summary (regressions, improvements, unchanged)
+    - Case-by-case comparison table with filtering
+    - Side-by-side response comparison for each case
+  - **Comparison JSON export**: Structured comparison data for programmatic use
+
+  ### CLI Enhancements
+
+  - **Compare command `--html` flag**: Generate HTML comparison report
+  - **Compare command `--json` flag**: Generate JSON comparison data
+
+  ### Documentation
+
+  - Updated all CLI command documentation
+  - Added comprehensive examples for custom multi-turn scenarios
+  - Documented combined matchers and `not_contains` expectations
+  - Added mutation strategy reference tables
+
+### Patch Changes
+
+- Updated dependencies [d2c3835]
+  - @artemiskit/core@0.2.0
+
 ## 0.1.6
 
 ### Patch Changes

package/dist/html/compare-generator.d.ts

@@ -0,0 +1,52 @@
+/**
+ * HTML Comparison Report Generator
+ * Generates a visual comparison between two runs
+ */
+import type { CaseResult, RunManifest } from '@artemiskit/core';
+/**
+ * Case-level comparison data
+ */
+export interface CaseComparison {
+    caseId: string;
+    name?: string;
+    baselineStatus: 'passed' | 'failed' | null;
+    currentStatus: 'passed' | 'failed' | null;
+    baselineScore: number | null;
+    currentScore: number | null;
+    scoreDelta: number;
+    baselineLatency: number | null;
+    currentLatency: number | null;
+    latencyDelta: number;
+    changeType: 'regressed' | 'improved' | 'unchanged' | 'new' | 'removed';
+    baselineCase?: CaseResult;
+    currentCase?: CaseResult;
+}
+/**
+ * Comparison data structure
+ */
+export interface ComparisonData {
+    baseline: RunManifest;
+    current: RunManifest;
+    metrics: {
+        successRateDelta: number;
+        medianLatencyDelta: number;
+        totalTokensDelta: number;
+    };
+    caseComparisons: CaseComparison[];
+    summary: {
+        totalRegressions: number;
+        totalImprovements: number;
+        totalUnchanged: number;
+        casesRemoved: number;
+        casesAdded: number;
+    };
+}
+/**
+ * Build comparison data from two manifests
+ */
+export declare function buildComparisonData(baseline: RunManifest, current: RunManifest): ComparisonData;
+/**
+ * Generate an HTML comparison report
+ */
+export declare function generateCompareHTMLReport(baseline: RunManifest, current: RunManifest): string;
+//# sourceMappingURL=compare-generator.d.ts.map

package/dist/html/compare-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare-generator.d.ts","sourceRoot":"","sources":["../../src/html/compare-generator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGhE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAC3C,aAAa,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAC1C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,SAAS,CAAC;IACvE,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,WAAW,CAAC,EAAE,UAAU,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAwfD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,GAAG,cAAc,CAsG/F;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,GAAG,MAAM,CA0H7F"}

package/dist/html/generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/html/generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAkOpD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,CA2BhE"}
+{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/html/generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAmgBpD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,CA+BhE"}

package/dist/html/redteam-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"redteam-generator.d.ts","sourceRoot":"","sources":["../../src/html/redteam-generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAkVxD,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CA0C3E"}
+{"version":3,"file":"redteam-generator.d.ts","sourceRoot":"","sources":["../../src/html/redteam-generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AA6nBxD,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CA8C3E"}

package/dist/html/stress-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stress-generator.d.ts","sourceRoot":"","sources":["../../src/html/stress-generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AA0UvD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAsCzE"}
+{"version":3,"file":"stress-generator.d.ts","sourceRoot":"","sources":["../../src/html/stress-generator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAiavD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAsCzE"}

package/dist/index.d.ts

@@ -6,4 +6,5 @@ export { generateHTMLReport } from './html/generator';
 export { generateJSONReport, type JSONReportOptions } from './json/generator';
 export { generateRedTeamHTMLReport } from './html/redteam-generator';
 export { generateStressHTMLReport } from './html/stress-generator';
+export { generateCompareHTMLReport, buildComparisonData, type ComparisonData, type CaseComparison, } from './html/compare-generator';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAGrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAGrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAGnE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,cAAc,GACpB,MAAM,0BAA0B,CAAC"}