@artemiskit/core 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/dist/adapters/types.d.ts +3 -1
  3. package/dist/adapters/types.d.ts.map +1 -1
  4. package/dist/artifacts/types.d.ts +39 -0
  5. package/dist/artifacts/types.d.ts.map +1 -1
  6. package/dist/cost/index.d.ts +5 -0
  7. package/dist/cost/index.d.ts.map +1 -0
  8. package/dist/cost/pricing.d.ts +66 -0
  9. package/dist/cost/pricing.d.ts.map +1 -0
  10. package/dist/evaluators/combined.d.ts +10 -0
  11. package/dist/evaluators/combined.d.ts.map +1 -0
  12. package/dist/evaluators/index.d.ts +4 -0
  13. package/dist/evaluators/index.d.ts.map +1 -1
  14. package/dist/evaluators/inline.d.ts +22 -0
  15. package/dist/evaluators/inline.d.ts.map +1 -0
  16. package/dist/evaluators/not-contains.d.ts +10 -0
  17. package/dist/evaluators/not-contains.d.ts.map +1 -0
  18. package/dist/evaluators/similarity.d.ts +16 -0
  19. package/dist/evaluators/similarity.d.ts.map +1 -0
  20. package/dist/events/emitter.d.ts +111 -0
  21. package/dist/events/emitter.d.ts.map +1 -0
  22. package/dist/events/index.d.ts +6 -0
  23. package/dist/events/index.d.ts.map +1 -0
  24. package/dist/events/types.d.ts +177 -0
  25. package/dist/events/types.d.ts.map +1 -0
  26. package/dist/index.d.ts +1 -0
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +13056 -12093
  29. package/dist/scenario/discovery.d.ts +72 -0
  30. package/dist/scenario/discovery.d.ts.map +1 -0
  31. package/dist/scenario/index.d.ts +1 -0
  32. package/dist/scenario/index.d.ts.map +1 -1
  33. package/dist/scenario/schema.d.ts +1245 -9
  34. package/dist/scenario/schema.d.ts.map +1 -1
  35. package/package.json +1 -1
  36. package/src/adapters/types.ts +3 -1
  37. package/src/artifacts/types.ts +39 -0
  38. package/src/cost/index.ts +14 -0
  39. package/src/cost/pricing.ts +273 -0
  40. package/src/evaluators/combined.test.ts +172 -0
  41. package/src/evaluators/combined.ts +95 -0
  42. package/src/evaluators/index.ts +12 -0
  43. package/src/evaluators/inline.test.ts +409 -0
  44. package/src/evaluators/inline.ts +393 -0
  45. package/src/evaluators/not-contains.test.ts +105 -0
  46. package/src/evaluators/not-contains.ts +45 -0
  47. package/src/evaluators/similarity.test.ts +333 -0
  48. package/src/evaluators/similarity.ts +258 -0
  49. package/src/index.ts +3 -0
  50. package/src/scenario/discovery.test.ts +153 -0
  51. package/src/scenario/discovery.ts +277 -0
  52. package/src/scenario/index.ts +1 -0
  53. package/src/scenario/schema.ts +43 -2
package/CHANGELOG.md CHANGED
@@ -1,5 +1,79 @@
1
1
  # @artemiskit/core
2
2
 
3
+ ## 0.2.0
4
+
5
+ ### Minor Changes
6
+
7
+ - d2c3835: ## v0.2.0 - Enhanced Evaluation Features
8
+
9
+ ### CLI (`@artemiskit/cli`)
10
+
11
+ #### New Features
12
+
13
+ - **Multi-turn mutations**: Added `--mutations multi_turn` flag for red team testing with 4 built-in strategies:
14
+ - `gradual_escalation`: Gradually intensifies requests over conversation turns
15
+ - `context_switching`: Shifts topics to lower defenses before attack
16
+ - `persona_building`: Establishes trust through roleplay
17
+ - `distraction`: Uses side discussions to slip in harmful requests
18
+ - **Custom multi-turn conversations**: Support for array prompts in red team scenarios (consistent with `run` command format). The last user message becomes the attack target, preceding messages form conversation context.
19
+ - **Custom attacks**: Added `--custom-attacks` flag to load custom attack patterns from YAML files with template variables and variations.
20
+ - **Encoding mutations**: Added `--mutations encoding` for obfuscation attacks (base64, ROT13, hex, unicode).
21
+ - **Directory scanning**: Run all scenarios in a directory with `akit run scenarios/`
22
+ - **Glob pattern matching**: Use patterns like `akit run scenarios/**/*.yaml`
23
+ - **Parallel execution**: Added `--parallel` flag for concurrent scenario execution
24
+ - **Scenario tags**: Filter scenarios with `--tags` flag
25
+
26
+ ### Core (`@artemiskit/core`)
27
+
28
+ #### New Features
29
+
30
+ - **Combined matchers**: New `type: combined` expectation with `operator: and|or` for complex assertion logic
31
+ - **`not_contains` expectation**: Negative containment check to ensure responses don't include specific text
32
+ - **`similarity` expectation**: Semantic similarity matching with two modes:
33
+ - Embedding-based: Uses vector embeddings for fast semantic comparison
34
+ - LLM-based fallback: Uses LLM to evaluate semantic similarity when embeddings unavailable
35
+ - Configurable threshold (default 0.75)
36
+ - **`inline` expectation**: Safe expression-based custom matchers in YAML using JavaScript-like expressions (e.g., `response.length > 100`, `response.includes('hello')`)
37
+ - **p90 latency metric**: Added p90 percentile to stress test latency metrics
38
+ - **Token usage tracking**: Monitor token consumption per request in stress tests
39
+ - **Cost estimation**: Estimate API costs with model pricing data
40
+
41
+ ### Red Team (`@artemiskit/redteam`)
42
+
43
+ #### New Features
44
+
45
+ - **MultiTurnMutation class**: Full implementation with strategy support and custom conversation prefixes
46
+ - **Custom attack loader**: Parse and load custom attack patterns from YAML
47
+ - **Encoding mutation**: Obfuscate attack payloads using various encoding schemes
48
+ - **CVSS-like severity scoring**: Detailed attack severity scoring with:
49
+ - `CvssScore` interface with attack vector, complexity, impact metrics
50
+ - `CvssCalculator` class for score calculation and aggregation
51
+ - Predefined scores for all mutations and detection categories
52
+ - Human-readable score descriptions and vector strings
53
+
54
+ ### Reports (`@artemiskit/reports`)
55
+
56
+ #### New Features
57
+
58
+ - **Run comparison HTML report**: Visual diff between two runs showing:
59
+ - Metrics overview with baseline vs current comparison
60
+ - Change summary (regressions, improvements, unchanged)
61
+ - Case-by-case comparison table with filtering
62
+ - Side-by-side response comparison for each case
63
+ - **Comparison JSON export**: Structured comparison data for programmatic use
64
+
65
+ ### CLI Enhancements
66
+
67
+ - **Compare command `--html` flag**: Generate HTML comparison report
68
+ - **Compare command `--json` flag**: Generate JSON comparison data
69
+
70
+ ### Documentation
71
+
72
+ - Updated all CLI command documentation
73
+ - Added comprehensive examples for custom multi-turn scenarios
74
+ - Documented combined matchers and `not_contains` expectations
75
+ - Added mutation strategy reference tables
76
+
3
77
  ## 0.1.6
4
78
 
5
79
  ### Patch Changes
@@ -95,7 +95,7 @@ export interface ModelClient {
95
95
  readonly provider: string;
96
96
  generate(options: GenerateOptions): Promise<GenerateResult>;
97
97
  stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
98
- embed?(text: string): Promise<number[]>;
98
+ embed?(text: string, model?: string): Promise<number[]>;
99
99
  capabilities(): Promise<ModelCapabilities>;
100
100
  close?(): Promise<void>;
101
101
  }
@@ -129,6 +129,8 @@ export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
129
129
  resourceName: string;
130
130
  deploymentName: string;
131
131
  apiVersion: string;
132
+ /** Optional separate deployment name for embedding models */
133
+ embeddingDeploymentName?: string;
132
134
  }
133
135
  /**
134
136
  * Vercel AI SDK configuration
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExC,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}
@@ -280,6 +280,15 @@ export interface StressRequestResult {
280
280
  error?: string;
281
281
  /** Timestamp of the request */
282
282
  timestamp: number;
283
+ /** Token usage for this request */
284
+ tokens?: {
285
+ /** Prompt/input tokens */
286
+ prompt: number;
287
+ /** Completion/output tokens */
288
+ completion: number;
289
+ /** Total tokens */
290
+ total: number;
291
+ };
283
292
  }
284
293
  /**
285
294
  * Stress test metrics
@@ -309,6 +318,36 @@ export interface StressMetrics {
309
318
  p95_latency_ms: number;
310
319
  /** 99th percentile latency */
311
320
  p99_latency_ms: number;
321
+ /** Token usage metrics (optional - only if provider returns token counts) */
322
+ tokens?: {
323
+ /** Total prompt/input tokens across all requests */
324
+ total_prompt_tokens: number;
325
+ /** Total completion/output tokens across all requests */
326
+ total_completion_tokens: number;
327
+ /** Total tokens (prompt + completion) */
328
+ total_tokens: number;
329
+ /** Average tokens per request */
330
+ avg_tokens_per_request: number;
331
+ };
332
+ /** Estimated cost metrics (optional - only if cost estimation is available) */
333
+ cost?: {
334
+ /** Estimated total cost in USD */
335
+ estimated_total_usd: number;
336
+ /** Cost breakdown by token type */
337
+ breakdown: {
338
+ /** Cost for prompt/input tokens */
339
+ prompt_cost_usd: number;
340
+ /** Cost for completion/output tokens */
341
+ completion_cost_usd: number;
342
+ };
343
+ /** Model used for cost calculation */
344
+ model: string;
345
+ /** Pricing used (per 1K tokens) */
346
+ pricing: {
347
+ prompt_per_1k: number;
348
+ completion_per_1k: number;
349
+ };
350
+ };
312
351
  }
313
352
  /**
314
353
  * Stress test configuration
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Cost estimation module
3
+ */
4
+ export { MODEL_PRICING, DEFAULT_PRICING, getModelPricing, estimateCost, formatCost, listKnownModels, type ModelPricing, type CostEstimate, } from './pricing.js';
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cost/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,aAAa,EACb,eAAe,EACf,eAAe,EACf,YAAY,EACZ,UAAU,EACV,eAAe,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC"}
@@ -0,0 +1,66 @@
1
+ /**
2
+ * LLM Pricing Data and Cost Estimation
3
+ *
4
+ * Pricing is per 1,000 tokens (1K tokens) in USD
5
+ * Data is updated periodically - always verify with provider's official pricing
6
+ */
7
+ export interface ModelPricing {
8
+ /** Price per 1K prompt/input tokens in USD */
9
+ promptPer1K: number;
10
+ /** Price per 1K completion/output tokens in USD */
11
+ completionPer1K: number;
12
+ /** Last updated date */
13
+ lastUpdated: string;
14
+ /** Notes about the pricing */
15
+ notes?: string;
16
+ }
17
+ export interface CostEstimate {
18
+ /** Estimated total cost in USD */
19
+ totalUsd: number;
20
+ /** Cost for prompt tokens */
21
+ promptCostUsd: number;
22
+ /** Cost for completion tokens */
23
+ completionCostUsd: number;
24
+ /** The model used for pricing */
25
+ model: string;
26
+ /** Pricing used */
27
+ pricing: ModelPricing;
28
+ }
29
+ /**
30
+ * Known model pricing data
31
+ * Prices are in USD per 1,000 tokens
32
+ */
33
+ export declare const MODEL_PRICING: Record<string, ModelPricing>;
34
+ /**
35
+ * Default pricing for unknown models
36
+ * Uses conservative estimates
37
+ */
38
+ export declare const DEFAULT_PRICING: ModelPricing;
39
+ /**
40
+ * Get pricing for a model
41
+ * @param model Model identifier
42
+ * @returns Pricing data or default if unknown
43
+ */
44
+ export declare function getModelPricing(model: string): ModelPricing;
45
+ /**
46
+ * Estimate cost for token usage
47
+ * @param promptTokens Number of prompt/input tokens
48
+ * @param completionTokens Number of completion/output tokens
49
+ * @param model Model identifier
50
+ * @returns Cost estimate
51
+ */
52
+ export declare function estimateCost(promptTokens: number, completionTokens: number, model: string): CostEstimate;
53
+ /**
54
+ * Format cost for display
55
+ * @param costUsd Cost in USD
56
+ * @returns Formatted string
57
+ */
58
+ export declare function formatCost(costUsd: number): string;
59
+ /**
60
+ * List all known models with pricing
61
+ */
62
+ export declare function listKnownModels(): Array<{
63
+ model: string;
64
+ pricing: ModelPricing;
65
+ }>;
66
+ //# sourceMappingURL=pricing.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../../src/cost/pricing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAqHtD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,YAK7B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAkD3D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,YAAY,CAcd;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAQlD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAKjF"}
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Combined evaluator - evaluates multiple expectations with and/or logic
3
+ */
4
+ import type { Expected } from '../scenario/schema';
5
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
6
+ export declare class CombinedEvaluator implements Evaluator {
7
+ readonly type = "combined";
8
+ evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
9
+ }
10
+ //# sourceMappingURL=combined.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"combined.d.ts","sourceRoot":"","sources":["../../src/evaluators/combined.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAW5E,qBAAa,iBAAkB,YAAW,SAAS;IACjD,QAAQ,CAAC,IAAI,cAAc;IAErB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;CAuE5B"}
@@ -19,6 +19,10 @@ export { ExactEvaluator } from './exact';
19
19
  export { RegexEvaluator } from './regex';
20
20
  export { FuzzyEvaluator } from './fuzzy';
21
21
  export { ContainsEvaluator } from './contains';
22
+ export { NotContainsEvaluator } from './not-contains';
23
+ export { CombinedEvaluator } from './combined';
22
24
  export { JsonSchemaEvaluator } from './json-schema';
23
25
  export { LLMGraderEvaluator } from './llm-grader';
26
+ export { SimilarityEvaluator } from './similarity';
27
+ export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';
24
28
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAUzC;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAOpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC;AAED,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAczC;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAOpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC;AAED,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC"}
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Inline custom matcher evaluator
3
+ * Allows users to define simple matching expressions directly in YAML
4
+ *
5
+ * Supports a safe subset of JavaScript-like expressions:
6
+ * - String methods: includes, startsWith, endsWith, length, toLowerCase, toUpperCase, trim
7
+ * - Comparisons: ==, !=, >, <, >=, <=
8
+ * - Logical operators: &&, ||, !
9
+ * - Regex matching: response.match(/pattern/)
10
+ * - JSON parsing: JSON.parse(response)
11
+ */
12
+ import type { Expected } from '../scenario/schema';
13
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
14
+ export declare class InlineEvaluator implements Evaluator {
15
+ readonly type = "inline";
16
+ evaluate(response: string, expected: Expected, _context?: EvaluatorContext): Promise<EvaluatorResult>;
17
+ }
18
+ /**
19
+ * List of supported expression patterns for documentation
20
+ */
21
+ export declare const SUPPORTED_EXPRESSIONS: string[];
22
+ //# sourceMappingURL=inline.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inline.d.ts","sourceRoot":"","sources":["../../src/evaluators/inline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAyS5E,qBAAa,eAAgB,YAAW,SAAS;IAC/C,QAAQ,CAAC,IAAI,YAAY;IAEnB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,eAAe,CAAC;CAqD5B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,UAiBjC,CAAC"}
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Not Contains evaluator - checks if response does NOT contain specific values
3
+ */
4
+ import type { Expected } from '../scenario/schema';
5
+ import type { Evaluator, EvaluatorResult } from './types';
6
+ export declare class NotContainsEvaluator implements Evaluator {
7
+ readonly type = "not_contains";
8
+ evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
9
+ }
10
+ //# sourceMappingURL=not-contains.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"not-contains.d.ts","sourceRoot":"","sources":["../../src/evaluators/not-contains.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,oBAAqB,YAAW,SAAS;IACpD,QAAQ,CAAC,IAAI,kBAAkB;IAEzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CAkC/E"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Semantic similarity evaluator
3
+ * Uses vector embeddings for semantic similarity matching when available,
4
+ * falls back to LLM-based semantic comparison otherwise.
5
+ */
6
+ import type { Expected } from '../scenario/schema';
7
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
8
+ export declare class SimilarityEvaluator implements Evaluator {
9
+ readonly type = "similarity";
10
+ evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
11
+ /**
12
+ * Evaluate similarity using LLM-based comparison
13
+ */
14
+ private evaluateWithLLM;
15
+ }
16
+ //# sourceMappingURL=similarity.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"similarity.d.ts","sourceRoot":"","sources":["../../src/evaluators/similarity.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA6D5E,qBAAa,mBAAoB,YAAW,SAAS;IACnD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IAkF3B;;OAEG;YACW,eAAe;CAiG9B"}
@@ -0,0 +1,111 @@
1
+ /**
2
+ * Event emitter for observability
3
+ *
4
+ * A lightweight typed event emitter that allows the observability package
5
+ * to subscribe to events from the core runner.
6
+ */
7
+ import type { ArtemisEvent, ArtemisEventMap, ArtemisEventType } from './types';
8
+ /**
9
+ * Event handler function type
10
+ */
11
+ export type EventHandler<T extends ArtemisEventType> = (event: ArtemisEventMap[T]) => void;
12
+ /**
13
+ * Async event handler function type
14
+ */
15
+ export type AsyncEventHandler<T extends ArtemisEventType> = (event: ArtemisEventMap[T]) => void | Promise<void>;
16
+ /**
17
+ * Subscriber interface for observability integrations
18
+ */
19
+ export interface EventSubscriber {
20
+ /** Unique identifier for this subscriber */
21
+ id: string;
22
+ /** Handle an event */
23
+ handleEvent(event: ArtemisEvent): void | Promise<void>;
24
+ /** Optional: called when subscriber is removed */
25
+ dispose?(): void | Promise<void>;
26
+ }
27
+ /**
28
+ * Global event emitter instance
29
+ *
30
+ * This singleton allows the observability package to subscribe to events
31
+ * without tight coupling to the core runner.
32
+ */
33
+ declare class ArtemisEventEmitter {
34
+ private subscribers;
35
+ private handlers;
36
+ private enabled;
37
+ /**
38
+ * Enable or disable event emission
39
+ * When disabled, no events are emitted (zero overhead)
40
+ */
41
+ setEnabled(enabled: boolean): void;
42
+ /**
43
+ * Check if any subscribers are registered
44
+ */
45
+ hasSubscribers(): boolean;
46
+ /**
47
+ * Register a subscriber that receives all events
48
+ */
49
+ subscribe(subscriber: EventSubscriber): () => void;
50
+ /**
51
+ * Remove a subscriber by ID
52
+ */
53
+ unsubscribe(subscriberId: string): Promise<void>;
54
+ /**
55
+ * Register a handler for a specific event type
56
+ */
57
+ on<T extends ArtemisEventType>(eventType: T, handler: AsyncEventHandler<T>): () => void;
58
+ /**
59
+ * Remove a handler for a specific event type
60
+ */
61
+ off<T extends ArtemisEventType>(eventType: T, handler: AsyncEventHandler<T>): void;
62
+ /**
63
+ * Emit an event to all subscribers and type-specific handlers
64
+ *
65
+ * Events are emitted asynchronously but errors are caught and logged
66
+ * to prevent observability from breaking the test runner.
67
+ */
68
+ emit<T extends ArtemisEventType>(event: ArtemisEventMap[T]): void;
69
+ /**
70
+ * Emit an event and wait for all handlers to complete
71
+ * Useful for testing or when you need to ensure handlers have run
72
+ */
73
+ emitAsync<T extends ArtemisEventType>(event: ArtemisEventMap[T]): Promise<void>;
74
+ /**
75
+ * Remove all subscribers and handlers
76
+ */
77
+ clear(): Promise<void>;
78
+ /**
79
+ * Get the number of registered subscribers
80
+ */
81
+ get subscriberCount(): number;
82
+ /**
83
+ * Get the number of registered handlers (across all event types)
84
+ */
85
+ get handlerCount(): number;
86
+ }
87
+ /**
88
+ * Global event emitter singleton
89
+ *
90
+ * Usage in core:
91
+ * ```typescript
92
+ * import { events } from '@artemiskit/core';
93
+ * events.emit({ type: 'test:start', ... });
94
+ * ```
95
+ *
96
+ * Usage in observability:
97
+ * ```typescript
98
+ * import { events } from '@artemiskit/core';
99
+ * events.subscribe({
100
+ * id: 'metrics',
101
+ * handleEvent: (event) => { ... }
102
+ * });
103
+ * ```
104
+ */
105
+ export declare const events: ArtemisEventEmitter;
106
+ /**
107
+ * Helper to create a timestamp for events
108
+ */
109
+ export declare function createTimestamp(): number;
110
+ export {};
111
+ //# sourceMappingURL=emitter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"emitter.d.ts","sourceRoot":"","sources":["../../src/events/emitter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE/E;;GAEG;AACH,MAAM,MAAM,YAAY,CAAC,CAAC,SAAS,gBAAgB,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;AAE3F;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,SAAS,gBAAgB,IAAI,CAC1D,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KACtB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,kDAAkD;IAClD,OAAO,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClC;AAED;;;;;GAKG;AACH,cAAM,mBAAmB;IACvB,OAAO,CAAC,WAAW,CAA2C;IAC9D,OAAO,CAAC,QAAQ,CAAiE;IACjF,OAAO,CAAC,OAAO,CAAiB;IAEhC;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIlC;;OAEG;IACH,cAAc,IAAI,OAAO;IAIzB;;OAEG;IACH,SAAS,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM,IAAI;IASlD;;OAEG;IACG,WAAW,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,gBAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI;IAYvF;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,gBAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI;IAUlF;;;;;OAKG;IACH,IAAI,CAAC,CAAC,SAAS,gBAAgB,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,IAAI;IAWjE;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,gBAAgB,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCrF;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B;;OAEG;IACH,IAAI,eAAe,IAAI,MAAM,CAE5B;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,MAAM,CAMzB;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,MAAM,qBAA4B,CAAC;AAEhD;;GAEG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAExC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Events module - for observability integration
3
+ */
4
+ export * from './types';
5
+ export { events, createTimestamp, type EventSubscriber, type EventHandler, type AsyncEventHandler } from './emitter';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/events/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,WAAW,CAAC"}