@artemiskit/core 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/dist/adapters/types.d.ts +3 -1
- package/dist/adapters/types.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +39 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/cost/index.d.ts +5 -0
- package/dist/cost/index.d.ts.map +1 -0
- package/dist/cost/pricing.d.ts +66 -0
- package/dist/cost/pricing.d.ts.map +1 -0
- package/dist/evaluators/combined.d.ts +10 -0
- package/dist/evaluators/combined.d.ts.map +1 -0
- package/dist/evaluators/index.d.ts +4 -0
- package/dist/evaluators/index.d.ts.map +1 -1
- package/dist/evaluators/inline.d.ts +22 -0
- package/dist/evaluators/inline.d.ts.map +1 -0
- package/dist/evaluators/not-contains.d.ts +10 -0
- package/dist/evaluators/not-contains.d.ts.map +1 -0
- package/dist/evaluators/similarity.d.ts +16 -0
- package/dist/evaluators/similarity.d.ts.map +1 -0
- package/dist/events/emitter.d.ts +111 -0
- package/dist/events/emitter.d.ts.map +1 -0
- package/dist/events/index.d.ts +6 -0
- package/dist/events/index.d.ts.map +1 -0
- package/dist/events/types.d.ts +177 -0
- package/dist/events/types.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13056 -12093
- package/dist/scenario/discovery.d.ts +72 -0
- package/dist/scenario/discovery.d.ts.map +1 -0
- package/dist/scenario/index.d.ts +1 -0
- package/dist/scenario/index.d.ts.map +1 -1
- package/dist/scenario/schema.d.ts +1245 -9
- package/dist/scenario/schema.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/adapters/types.ts +3 -1
- package/src/artifacts/types.ts +39 -0
- package/src/cost/index.ts +14 -0
- package/src/cost/pricing.ts +273 -0
- package/src/evaluators/combined.test.ts +172 -0
- package/src/evaluators/combined.ts +95 -0
- package/src/evaluators/index.ts +12 -0
- package/src/evaluators/inline.test.ts +409 -0
- package/src/evaluators/inline.ts +393 -0
- package/src/evaluators/not-contains.test.ts +105 -0
- package/src/evaluators/not-contains.ts +45 -0
- package/src/evaluators/similarity.test.ts +333 -0
- package/src/evaluators/similarity.ts +258 -0
- package/src/index.ts +3 -0
- package/src/scenario/discovery.test.ts +153 -0
- package/src/scenario/discovery.ts +277 -0
- package/src/scenario/index.ts +1 -0
- package/src/scenario/schema.ts +43 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,79 @@
|
|
|
1
1
|
# @artemiskit/core
|
|
2
2
|
|
|
3
|
+
## 0.2.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- d2c3835: ## v0.2.0 - Enhanced Evaluation Features
|
|
8
|
+
|
|
9
|
+
### CLI (`@artemiskit/cli`)
|
|
10
|
+
|
|
11
|
+
#### New Features
|
|
12
|
+
|
|
13
|
+
- **Multi-turn mutations**: Added `--mutations multi_turn` flag for red team testing with 4 built-in strategies:
|
|
14
|
+
- `gradual_escalation`: Gradually intensifies requests over conversation turns
|
|
15
|
+
- `context_switching`: Shifts topics to lower defenses before attack
|
|
16
|
+
- `persona_building`: Establishes trust through roleplay
|
|
17
|
+
- `distraction`: Uses side discussions to slip in harmful requests
|
|
18
|
+
- **Custom multi-turn conversations**: Support for array prompts in red team scenarios (consistent with `run` command format). The last user message becomes the attack target, preceding messages form conversation context.
|
|
19
|
+
- **Custom attacks**: Added `--custom-attacks` flag to load custom attack patterns from YAML files with template variables and variations.
|
|
20
|
+
- **Encoding mutations**: Added `--mutations encoding` for obfuscation attacks (base64, ROT13, hex, unicode).
|
|
21
|
+
- **Directory scanning**: Run all scenarios in a directory with `akit run scenarios/`
|
|
22
|
+
- **Glob pattern matching**: Use patterns like `akit run scenarios/**/*.yaml`
|
|
23
|
+
- **Parallel execution**: Added `--parallel` flag for concurrent scenario execution
|
|
24
|
+
- **Scenario tags**: Filter scenarios with `--tags` flag
|
|
25
|
+
|
|
26
|
+
### Core (`@artemiskit/core`)
|
|
27
|
+
|
|
28
|
+
#### New Features
|
|
29
|
+
|
|
30
|
+
- **Combined matchers**: New `type: combined` expectation with `operator: and|or` for complex assertion logic
|
|
31
|
+
- **`not_contains` expectation**: Negative containment check to ensure responses don't include specific text
|
|
32
|
+
- **`similarity` expectation**: Semantic similarity matching with two modes:
|
|
33
|
+
- Embedding-based: Uses vector embeddings for fast semantic comparison
|
|
34
|
+
- LLM-based fallback: Uses LLM to evaluate semantic similarity when embeddings unavailable
|
|
35
|
+
- Configurable threshold (default 0.75)
|
|
36
|
+
- **`inline` expectation**: Safe expression-based custom matchers in YAML using JavaScript-like expressions (e.g., `response.length > 100`, `response.includes('hello')`)
|
|
37
|
+
- **p90 latency metric**: Added p90 percentile to stress test latency metrics
|
|
38
|
+
- **Token usage tracking**: Monitor token consumption per request in stress tests
|
|
39
|
+
- **Cost estimation**: Estimate API costs with model pricing data
|
|
40
|
+
|
|
41
|
+
### Red Team (`@artemiskit/redteam`)
|
|
42
|
+
|
|
43
|
+
#### New Features
|
|
44
|
+
|
|
45
|
+
- **MultiTurnMutation class**: Full implementation with strategy support and custom conversation prefixes
|
|
46
|
+
- **Custom attack loader**: Parse and load custom attack patterns from YAML
|
|
47
|
+
- **Encoding mutation**: Obfuscate attack payloads using various encoding schemes
|
|
48
|
+
- **CVSS-like severity scoring**: Detailed attack severity scoring with:
|
|
49
|
+
- `CvssScore` interface with attack vector, complexity, impact metrics
|
|
50
|
+
- `CvssCalculator` class for score calculation and aggregation
|
|
51
|
+
- Predefined scores for all mutations and detection categories
|
|
52
|
+
- Human-readable score descriptions and vector strings
|
|
53
|
+
|
|
54
|
+
### Reports (`@artemiskit/reports`)
|
|
55
|
+
|
|
56
|
+
#### New Features
|
|
57
|
+
|
|
58
|
+
- **Run comparison HTML report**: Visual diff between two runs showing:
|
|
59
|
+
- Metrics overview with baseline vs current comparison
|
|
60
|
+
- Change summary (regressions, improvements, unchanged)
|
|
61
|
+
- Case-by-case comparison table with filtering
|
|
62
|
+
- Side-by-side response comparison for each case
|
|
63
|
+
- **Comparison JSON export**: Structured comparison data for programmatic use
|
|
64
|
+
|
|
65
|
+
### CLI Enhancements
|
|
66
|
+
|
|
67
|
+
- **Compare command `--html` flag**: Generate HTML comparison report
|
|
68
|
+
- **Compare command `--json` flag**: Generate JSON comparison data
|
|
69
|
+
|
|
70
|
+
### Documentation
|
|
71
|
+
|
|
72
|
+
- Updated all CLI command documentation
|
|
73
|
+
- Added comprehensive examples for custom multi-turn scenarios
|
|
74
|
+
- Documented combined matchers and `not_contains` expectations
|
|
75
|
+
- Added mutation strategy reference tables
|
|
76
|
+
|
|
3
77
|
## 0.1.6
|
|
4
78
|
|
|
5
79
|
### Patch Changes
|
package/dist/adapters/types.d.ts
CHANGED
|
@@ -95,7 +95,7 @@ export interface ModelClient {
|
|
|
95
95
|
readonly provider: string;
|
|
96
96
|
generate(options: GenerateOptions): Promise<GenerateResult>;
|
|
97
97
|
stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
|
|
98
|
-
embed?(text: string): Promise<number[]>;
|
|
98
|
+
embed?(text: string, model?: string): Promise<number[]>;
|
|
99
99
|
capabilities(): Promise<ModelCapabilities>;
|
|
100
100
|
close?(): Promise<void>;
|
|
101
101
|
}
|
|
@@ -129,6 +129,8 @@ export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
|
|
|
129
129
|
resourceName: string;
|
|
130
130
|
deploymentName: string;
|
|
131
131
|
apiVersion: string;
|
|
132
|
+
/** Optional separate deployment name for embedding models */
|
|
133
|
+
embeddingDeploymentName?: string;
|
|
132
134
|
}
|
|
133
135
|
/**
|
|
134
136
|
* Vercel AI SDK configuration
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}
|
|
@@ -280,6 +280,15 @@ export interface StressRequestResult {
|
|
|
280
280
|
error?: string;
|
|
281
281
|
/** Timestamp of the request */
|
|
282
282
|
timestamp: number;
|
|
283
|
+
/** Token usage for this request */
|
|
284
|
+
tokens?: {
|
|
285
|
+
/** Prompt/input tokens */
|
|
286
|
+
prompt: number;
|
|
287
|
+
/** Completion/output tokens */
|
|
288
|
+
completion: number;
|
|
289
|
+
/** Total tokens */
|
|
290
|
+
total: number;
|
|
291
|
+
};
|
|
283
292
|
}
|
|
284
293
|
/**
|
|
285
294
|
* Stress test metrics
|
|
@@ -309,6 +318,36 @@ export interface StressMetrics {
|
|
|
309
318
|
p95_latency_ms: number;
|
|
310
319
|
/** 99th percentile latency */
|
|
311
320
|
p99_latency_ms: number;
|
|
321
|
+
/** Token usage metrics (optional - only if provider returns token counts) */
|
|
322
|
+
tokens?: {
|
|
323
|
+
/** Total prompt/input tokens across all requests */
|
|
324
|
+
total_prompt_tokens: number;
|
|
325
|
+
/** Total completion/output tokens across all requests */
|
|
326
|
+
total_completion_tokens: number;
|
|
327
|
+
/** Total tokens (prompt + completion) */
|
|
328
|
+
total_tokens: number;
|
|
329
|
+
/** Average tokens per request */
|
|
330
|
+
avg_tokens_per_request: number;
|
|
331
|
+
};
|
|
332
|
+
/** Estimated cost metrics (optional - only if cost estimation is available) */
|
|
333
|
+
cost?: {
|
|
334
|
+
/** Estimated total cost in USD */
|
|
335
|
+
estimated_total_usd: number;
|
|
336
|
+
/** Cost breakdown by token type */
|
|
337
|
+
breakdown: {
|
|
338
|
+
/** Cost for prompt/input tokens */
|
|
339
|
+
prompt_cost_usd: number;
|
|
340
|
+
/** Cost for completion/output tokens */
|
|
341
|
+
completion_cost_usd: number;
|
|
342
|
+
};
|
|
343
|
+
/** Model used for cost calculation */
|
|
344
|
+
model: string;
|
|
345
|
+
/** Pricing used (per 1K tokens) */
|
|
346
|
+
pricing: {
|
|
347
|
+
prompt_per_1k: number;
|
|
348
|
+
completion_per_1k: number;
|
|
349
|
+
};
|
|
350
|
+
};
|
|
312
351
|
}
|
|
313
352
|
/**
|
|
314
353
|
* Stress test configuration
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cost/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,aAAa,EACb,eAAe,EACf,eAAe,EACf,YAAY,EACZ,UAAU,EACV,eAAe,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM Pricing Data and Cost Estimation
|
|
3
|
+
*
|
|
4
|
+
* Pricing is per 1,000 tokens (1K tokens) in USD
|
|
5
|
+
* Data is updated periodically - always verify with provider's official pricing
|
|
6
|
+
*/
|
|
7
|
+
export interface ModelPricing {
|
|
8
|
+
/** Price per 1K prompt/input tokens in USD */
|
|
9
|
+
promptPer1K: number;
|
|
10
|
+
/** Price per 1K completion/output tokens in USD */
|
|
11
|
+
completionPer1K: number;
|
|
12
|
+
/** Last updated date */
|
|
13
|
+
lastUpdated: string;
|
|
14
|
+
/** Notes about the pricing */
|
|
15
|
+
notes?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface CostEstimate {
|
|
18
|
+
/** Estimated total cost in USD */
|
|
19
|
+
totalUsd: number;
|
|
20
|
+
/** Cost for prompt tokens */
|
|
21
|
+
promptCostUsd: number;
|
|
22
|
+
/** Cost for completion tokens */
|
|
23
|
+
completionCostUsd: number;
|
|
24
|
+
/** The model used for pricing */
|
|
25
|
+
model: string;
|
|
26
|
+
/** Pricing used */
|
|
27
|
+
pricing: ModelPricing;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Known model pricing data
|
|
31
|
+
* Prices are in USD per 1,000 tokens
|
|
32
|
+
*/
|
|
33
|
+
export declare const MODEL_PRICING: Record<string, ModelPricing>;
|
|
34
|
+
/**
|
|
35
|
+
* Default pricing for unknown models
|
|
36
|
+
* Uses conservative estimates
|
|
37
|
+
*/
|
|
38
|
+
export declare const DEFAULT_PRICING: ModelPricing;
|
|
39
|
+
/**
|
|
40
|
+
* Get pricing for a model
|
|
41
|
+
* @param model Model identifier
|
|
42
|
+
* @returns Pricing data or default if unknown
|
|
43
|
+
*/
|
|
44
|
+
export declare function getModelPricing(model: string): ModelPricing;
|
|
45
|
+
/**
|
|
46
|
+
* Estimate cost for token usage
|
|
47
|
+
* @param promptTokens Number of prompt/input tokens
|
|
48
|
+
* @param completionTokens Number of completion/output tokens
|
|
49
|
+
* @param model Model identifier
|
|
50
|
+
* @returns Cost estimate
|
|
51
|
+
*/
|
|
52
|
+
export declare function estimateCost(promptTokens: number, completionTokens: number, model: string): CostEstimate;
|
|
53
|
+
/**
|
|
54
|
+
* Format cost for display
|
|
55
|
+
* @param costUsd Cost in USD
|
|
56
|
+
* @returns Formatted string
|
|
57
|
+
*/
|
|
58
|
+
export declare function formatCost(costUsd: number): string;
|
|
59
|
+
/**
|
|
60
|
+
* List all known models with pricing
|
|
61
|
+
*/
|
|
62
|
+
export declare function listKnownModels(): Array<{
|
|
63
|
+
model: string;
|
|
64
|
+
pricing: ModelPricing;
|
|
65
|
+
}>;
|
|
66
|
+
//# sourceMappingURL=pricing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../../src/cost/pricing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAqHtD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,YAK7B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAkD3D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,YAAY,CAcd;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAQlD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAKjF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Combined evaluator - evaluates multiple expectations with and/or logic
|
|
3
|
+
*/
|
|
4
|
+
import type { Expected } from '../scenario/schema';
|
|
5
|
+
import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
|
|
6
|
+
export declare class CombinedEvaluator implements Evaluator {
|
|
7
|
+
readonly type = "combined";
|
|
8
|
+
evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=combined.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"combined.d.ts","sourceRoot":"","sources":["../../src/evaluators/combined.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAW5E,qBAAa,iBAAkB,YAAW,SAAS;IACjD,QAAQ,CAAC,IAAI,cAAc;IAErB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;CAuE5B"}
|
|
@@ -19,6 +19,10 @@ export { ExactEvaluator } from './exact';
|
|
|
19
19
|
export { RegexEvaluator } from './regex';
|
|
20
20
|
export { FuzzyEvaluator } from './fuzzy';
|
|
21
21
|
export { ContainsEvaluator } from './contains';
|
|
22
|
+
export { NotContainsEvaluator } from './not-contains';
|
|
23
|
+
export { CombinedEvaluator } from './combined';
|
|
22
24
|
export { JsonSchemaEvaluator } from './json-schema';
|
|
23
25
|
export { LLMGraderEvaluator } from './llm-grader';
|
|
26
|
+
export { SimilarityEvaluator } from './similarity';
|
|
27
|
+
export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';
|
|
24
28
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAczC;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAOpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC;AAED,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inline custom matcher evaluator
|
|
3
|
+
* Allows users to define simple matching expressions directly in YAML
|
|
4
|
+
*
|
|
5
|
+
* Supports a safe subset of JavaScript-like expressions:
|
|
6
|
+
* - String methods: includes, startsWith, endsWith, length, toLowerCase, toUpperCase, trim
|
|
7
|
+
* - Comparisons: ==, !=, >, <, >=, <=
|
|
8
|
+
* - Logical operators: &&, ||, !
|
|
9
|
+
* - Regex matching: response.match(/pattern/)
|
|
10
|
+
* - JSON parsing: JSON.parse(response)
|
|
11
|
+
*/
|
|
12
|
+
import type { Expected } from '../scenario/schema';
|
|
13
|
+
import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
|
|
14
|
+
export declare class InlineEvaluator implements Evaluator {
|
|
15
|
+
readonly type = "inline";
|
|
16
|
+
evaluate(response: string, expected: Expected, _context?: EvaluatorContext): Promise<EvaluatorResult>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* List of supported expression patterns for documentation
|
|
20
|
+
*/
|
|
21
|
+
export declare const SUPPORTED_EXPRESSIONS: string[];
|
|
22
|
+
//# sourceMappingURL=inline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inline.d.ts","sourceRoot":"","sources":["../../src/evaluators/inline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAyS5E,qBAAa,eAAgB,YAAW,SAAS;IAC/C,QAAQ,CAAC,IAAI,YAAY;IAEnB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,eAAe,CAAC;CAqD5B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,UAiBjC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not Contains evaluator - checks if response does NOT contain specific values
|
|
3
|
+
*/
|
|
4
|
+
import type { Expected } from '../scenario/schema';
|
|
5
|
+
import type { Evaluator, EvaluatorResult } from './types';
|
|
6
|
+
export declare class NotContainsEvaluator implements Evaluator {
|
|
7
|
+
readonly type = "not_contains";
|
|
8
|
+
evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=not-contains.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"not-contains.d.ts","sourceRoot":"","sources":["../../src/evaluators/not-contains.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE1D,qBAAa,oBAAqB,YAAW,SAAS;IACpD,QAAQ,CAAC,IAAI,kBAAkB;IAEzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;CAkC/E"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic similarity evaluator
|
|
3
|
+
* Uses vector embeddings for semantic similarity matching when available,
|
|
4
|
+
* falls back to LLM-based semantic comparison otherwise.
|
|
5
|
+
*/
|
|
6
|
+
import type { Expected } from '../scenario/schema';
|
|
7
|
+
import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
|
|
8
|
+
export declare class SimilarityEvaluator implements Evaluator {
|
|
9
|
+
readonly type = "similarity";
|
|
10
|
+
evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
|
|
11
|
+
/**
|
|
12
|
+
* Evaluate similarity using LLM-based comparison
|
|
13
|
+
*/
|
|
14
|
+
private evaluateWithLLM;
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=similarity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"similarity.d.ts","sourceRoot":"","sources":["../../src/evaluators/similarity.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA6D5E,qBAAa,mBAAoB,YAAW,SAAS;IACnD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IAkF3B;;OAEG;YACW,eAAe;CAiG9B"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Event emitter for observability
|
|
3
|
+
*
|
|
4
|
+
* A lightweight typed event emitter that allows the observability package
|
|
5
|
+
* to subscribe to events from the core runner.
|
|
6
|
+
*/
|
|
7
|
+
import type { ArtemisEvent, ArtemisEventMap, ArtemisEventType } from './types';
|
|
8
|
+
/**
|
|
9
|
+
* Event handler function type
|
|
10
|
+
*/
|
|
11
|
+
export type EventHandler<T extends ArtemisEventType> = (event: ArtemisEventMap[T]) => void;
|
|
12
|
+
/**
|
|
13
|
+
* Async event handler function type
|
|
14
|
+
*/
|
|
15
|
+
export type AsyncEventHandler<T extends ArtemisEventType> = (event: ArtemisEventMap[T]) => void | Promise<void>;
|
|
16
|
+
/**
|
|
17
|
+
* Subscriber interface for observability integrations
|
|
18
|
+
*/
|
|
19
|
+
export interface EventSubscriber {
|
|
20
|
+
/** Unique identifier for this subscriber */
|
|
21
|
+
id: string;
|
|
22
|
+
/** Handle an event */
|
|
23
|
+
handleEvent(event: ArtemisEvent): void | Promise<void>;
|
|
24
|
+
/** Optional: called when subscriber is removed */
|
|
25
|
+
dispose?(): void | Promise<void>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Global event emitter instance
|
|
29
|
+
*
|
|
30
|
+
* This singleton allows the observability package to subscribe to events
|
|
31
|
+
* without tight coupling to the core runner.
|
|
32
|
+
*/
|
|
33
|
+
declare class ArtemisEventEmitter {
|
|
34
|
+
private subscribers;
|
|
35
|
+
private handlers;
|
|
36
|
+
private enabled;
|
|
37
|
+
/**
|
|
38
|
+
* Enable or disable event emission
|
|
39
|
+
* When disabled, no events are emitted (zero overhead)
|
|
40
|
+
*/
|
|
41
|
+
setEnabled(enabled: boolean): void;
|
|
42
|
+
/**
|
|
43
|
+
* Check if any subscribers are registered
|
|
44
|
+
*/
|
|
45
|
+
hasSubscribers(): boolean;
|
|
46
|
+
/**
|
|
47
|
+
* Register a subscriber that receives all events
|
|
48
|
+
*/
|
|
49
|
+
subscribe(subscriber: EventSubscriber): () => void;
|
|
50
|
+
/**
|
|
51
|
+
* Remove a subscriber by ID
|
|
52
|
+
*/
|
|
53
|
+
unsubscribe(subscriberId: string): Promise<void>;
|
|
54
|
+
/**
|
|
55
|
+
* Register a handler for a specific event type
|
|
56
|
+
*/
|
|
57
|
+
on<T extends ArtemisEventType>(eventType: T, handler: AsyncEventHandler<T>): () => void;
|
|
58
|
+
/**
|
|
59
|
+
* Remove a handler for a specific event type
|
|
60
|
+
*/
|
|
61
|
+
off<T extends ArtemisEventType>(eventType: T, handler: AsyncEventHandler<T>): void;
|
|
62
|
+
/**
|
|
63
|
+
* Emit an event to all subscribers and type-specific handlers
|
|
64
|
+
*
|
|
65
|
+
* Events are emitted asynchronously but errors are caught and logged
|
|
66
|
+
* to prevent observability from breaking the test runner.
|
|
67
|
+
*/
|
|
68
|
+
emit<T extends ArtemisEventType>(event: ArtemisEventMap[T]): void;
|
|
69
|
+
/**
|
|
70
|
+
* Emit an event and wait for all handlers to complete
|
|
71
|
+
* Useful for testing or when you need to ensure handlers have run
|
|
72
|
+
*/
|
|
73
|
+
emitAsync<T extends ArtemisEventType>(event: ArtemisEventMap[T]): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Remove all subscribers and handlers
|
|
76
|
+
*/
|
|
77
|
+
clear(): Promise<void>;
|
|
78
|
+
/**
|
|
79
|
+
* Get the number of registered subscribers
|
|
80
|
+
*/
|
|
81
|
+
get subscriberCount(): number;
|
|
82
|
+
/**
|
|
83
|
+
* Get the number of registered handlers (across all event types)
|
|
84
|
+
*/
|
|
85
|
+
get handlerCount(): number;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Global event emitter singleton
|
|
89
|
+
*
|
|
90
|
+
* Usage in core:
|
|
91
|
+
* ```typescript
|
|
92
|
+
* import { events } from '@artemiskit/core';
|
|
93
|
+
* events.emit({ type: 'test:start', ... });
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* Usage in observability:
|
|
97
|
+
* ```typescript
|
|
98
|
+
* import { events } from '@artemiskit/core';
|
|
99
|
+
* events.subscribe({
|
|
100
|
+
* id: 'metrics',
|
|
101
|
+
* handleEvent: (event) => { ... }
|
|
102
|
+
* });
|
|
103
|
+
* ```
|
|
104
|
+
*/
|
|
105
|
+
export declare const events: ArtemisEventEmitter;
|
|
106
|
+
/**
|
|
107
|
+
* Helper to create a timestamp for events
|
|
108
|
+
*/
|
|
109
|
+
export declare function createTimestamp(): number;
|
|
110
|
+
export {};
|
|
111
|
+
//# sourceMappingURL=emitter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"emitter.d.ts","sourceRoot":"","sources":["../../src/events/emitter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE/E;;GAEG;AACH,MAAM,MAAM,YAAY,CAAC,CAAC,SAAS,gBAAgB,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;AAE3F;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,SAAS,gBAAgB,IAAI,CAC1D,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KACtB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,kDAAkD;IAClD,OAAO,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClC;AAED;;;;;GAKG;AACH,cAAM,mBAAmB;IACvB,OAAO,CAAC,WAAW,CAA2C;IAC9D,OAAO,CAAC,QAAQ,CAAiE;IACjF,OAAO,CAAC,OAAO,CAAiB;IAEhC;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIlC;;OAEG;IACH,cAAc,IAAI,OAAO;IAIzB;;OAEG;IACH,SAAS,CAAC,UAAU,EAAE,eAAe,GAAG,MAAM,IAAI;IASlD;;OAEG;IACG,WAAW,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,gBAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI;IAYvF;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,gBAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI;IAUlF;;;;;OAKG;IACH,IAAI,CAAC,CAAC,SAAS,gBAAgB,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,IAAI;IAWjE;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,gBAAgB,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCrF;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B;;OAEG;IACH,IAAI,eAAe,IAAI,MAAM,CAE5B;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,MAAM,CAMzB;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,MAAM,qBAA4B,CAAC;AAEhD;;GAEG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAExC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/events/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,WAAW,CAAC"}
|