@artemiskit/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +213 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +350 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +398 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +99 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +24847 -13653
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/jest.d.ts +46 -0
  39. package/dist/matchers/jest.d.ts.map +1 -0
  40. package/dist/matchers/vitest.d.ts +24 -0
  41. package/dist/matchers/vitest.d.ts.map +1 -0
  42. package/dist/types-only.d.ts +31 -0
  43. package/dist/types-only.d.ts.map +1 -0
  44. package/dist/types-only.js +1 -0
  45. package/dist/types.d.ts +377 -0
  46. package/dist/types.d.ts.map +1 -0
  47. package/dist/utils/index.d.ts +112 -0
  48. package/dist/utils/index.d.ts.map +1 -0
  49. package/dist/utils/index.js +104 -0
  50. package/package.json +17 -2
  51. package/src/__tests__/builders.test.ts +445 -0
  52. package/src/__tests__/circuit-breaker.test.ts +704 -0
  53. package/src/__tests__/compare.test.ts +469 -0
  54. package/src/__tests__/contracts.test.ts +354 -0
  55. package/src/__tests__/guardian-modes.test.ts +215 -0
  56. package/src/__tests__/guardrails.test.ts +404 -0
  57. package/src/__tests__/semantic-validator.test.ts +421 -0
  58. package/src/__tests__/utils.test.ts +324 -0
  59. package/src/__tests__/validate.test.ts +307 -0
  60. package/src/artemiskit.ts +478 -5
  61. package/src/builders/index.ts +698 -0
  62. package/src/contracts/index.ts +357 -0
  63. package/src/guardian/circuit-breaker.ts +74 -28
  64. package/src/guardian/guardian.ts +307 -20
  65. package/src/guardian/guardrails.ts +143 -1
  66. package/src/guardian/index.ts +35 -1
  67. package/src/guardian/interceptor.ts +87 -11
  68. package/src/guardian/semantic-validator.ts +399 -0
  69. package/src/guardian/types.ts +413 -2
  70. package/src/index.ts +258 -14
  71. package/src/types-only.ts +276 -0
  72. package/src/types.ts +183 -0
  73. package/src/utils/index.ts +225 -0
  74. package/tsconfig.json +2 -1
@@ -0,0 +1,357 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Contract types for custom implementations
4
+ *
5
+ * These types define the contracts that custom adapters, evaluators,
6
+ * and storage backends must implement to integrate with ArtemisKit.
7
+ */
8
+
9
+ import type {
10
+ Evaluator,
11
+ EvaluatorContext,
12
+ EvaluatorResult,
13
+ Expected,
14
+ GenerateOptions,
15
+ GenerateResult,
16
+ ModelCapabilities,
17
+ ModelClient,
18
+ StorageAdapter,
19
+ } from '@artemiskit/core';
20
+
21
+ // ============================================================================
22
+ // Adapter Contract
23
+ // ============================================================================
24
+
25
+ /**
26
+ * Contract for implementing custom model adapters
27
+ *
28
+ * @example
29
+ * ```typescript
30
+ * import { defineAdapter, type AdapterContract } from '@artemiskit/sdk'
31
+ *
32
+ * const myAdapter = defineAdapter({
33
+ * provider: 'my-provider',
34
+ * async generate(options) {
35
+ * // Implementation
36
+ * return { id: '...', model: '...', text: '...', tokens: {...}, latencyMs: 0 }
37
+ * },
38
+ * async capabilities() {
39
+ * return { streaming: false, functionCalling: false, toolUse: false, maxContext: 4096 }
40
+ * }
41
+ * })
42
+ * ```
43
+ */
44
+ export interface AdapterContract extends ModelClient {
45
+ /**
46
+ * Provider identifier - must be unique
47
+ */
48
+ readonly provider: string;
49
+
50
+ /**
51
+ * Generate a completion from the model
52
+ */
53
+ generate(options: GenerateOptions): Promise<GenerateResult>;
54
+
55
+ /**
56
+ * Get the capabilities of this adapter
57
+ */
58
+ capabilities(): Promise<ModelCapabilities>;
59
+
60
+ /**
61
+ * Optional: Stream a completion from the model
62
+ */
63
+ stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
64
+
65
+ /**
66
+ * Optional: Generate embeddings
67
+ */
68
+ embed?(text: string, model?: string): Promise<number[]>;
69
+
70
+ /**
71
+ * Optional: Close/cleanup resources
72
+ */
73
+ close?(): Promise<void>;
74
+ }
75
+
76
+ /**
77
+ * Define a custom adapter with type checking
78
+ *
79
+ * @example
80
+ * ```typescript
81
+ * const myAdapter = defineAdapter({
82
+ * provider: 'my-llm',
83
+ * async generate(options) {
84
+ * const response = await myLLMClient.complete(options.prompt)
85
+ * return {
86
+ * id: response.id,
87
+ * model: 'my-model',
88
+ * text: response.text,
89
+ * tokens: { prompt: 0, completion: 0, total: 0 },
90
+ * latencyMs: response.duration
91
+ * }
92
+ * },
93
+ * async capabilities() {
94
+ * return { streaming: true, functionCalling: false, toolUse: false, maxContext: 8192 }
95
+ * }
96
+ * })
97
+ * ```
98
+ */
99
+ export function defineAdapter<T extends AdapterContract>(adapter: T): T {
100
+ // Validate required methods
101
+ if (!adapter.provider) {
102
+ throw new Error('Adapter must have a provider property');
103
+ }
104
+ if (typeof adapter.generate !== 'function') {
105
+ throw new Error('Adapter must have a generate method');
106
+ }
107
+ if (typeof adapter.capabilities !== 'function') {
108
+ throw new Error('Adapter must have a capabilities method');
109
+ }
110
+ return adapter;
111
+ }
112
+
113
+ // ============================================================================
114
+ // Evaluator Contract
115
+ // ============================================================================
116
+
117
+ /**
118
+ * Contract for implementing custom evaluators
119
+ *
120
+ * @example
121
+ * ```typescript
122
+ * import { defineEvaluator, type EvaluatorContract } from '@artemiskit/sdk'
123
+ *
124
+ * const customEvaluator = defineEvaluator({
125
+ * type: 'custom-semantic',
126
+ * async evaluate(response, expected, context) {
127
+ * // Custom evaluation logic
128
+ * const score = calculateSemanticScore(response, expected.value)
129
+ * return { passed: score > 0.8, score, reason: `Semantic score: ${score}` }
130
+ * }
131
+ * })
132
+ * ```
133
+ */
134
+ export interface EvaluatorContract extends Evaluator {
135
+ /**
136
+ * Unique type identifier for this evaluator
137
+ */
138
+ readonly type: string;
139
+
140
+ /**
141
+ * Evaluate a response against expected criteria
142
+ *
143
+ * @param response - The model's response text
144
+ * @param expected - The expected result configuration
145
+ * @param context - Optional context including model client and test case
146
+ * @returns Evaluation result with pass/fail, score, and reason
147
+ */
148
+ evaluate(
149
+ response: string,
150
+ expected: Expected,
151
+ context?: EvaluatorContext
152
+ ): Promise<EvaluatorResult>;
153
+ }
154
+
155
+ /**
156
+ * Define a custom evaluator with type checking
157
+ *
158
+ * @example
159
+ * ```typescript
160
+ * const wordCountEvaluator = defineEvaluator({
161
+ * type: 'word-count',
162
+ * async evaluate(response, expected, _context) {
163
+ * const wordCount = response.split(/\s+/).length
164
+ * const targetCount = (expected as any).wordCount ?? 100
165
+ * const tolerance = (expected as any).tolerance ?? 10
166
+ *
167
+ * const diff = Math.abs(wordCount - targetCount)
168
+ * const passed = diff <= tolerance
169
+ * const score = passed ? 1 : Math.max(0, 1 - diff / targetCount)
170
+ *
171
+ * return {
172
+ * passed,
173
+ * score,
174
+ * reason: `Word count: ${wordCount}, target: ${targetCount} (±${tolerance})`,
175
+ * details: { wordCount, targetCount, tolerance, diff }
176
+ * }
177
+ * }
178
+ * })
179
+ * ```
180
+ */
181
+ export function defineEvaluator<T extends EvaluatorContract>(evaluator: T): T {
182
+ // Validate required properties
183
+ if (!evaluator.type) {
184
+ throw new Error('Evaluator must have a type property');
185
+ }
186
+ if (typeof evaluator.evaluate !== 'function') {
187
+ throw new Error('Evaluator must have an evaluate method');
188
+ }
189
+ return evaluator;
190
+ }
191
+
192
+ // ============================================================================
193
+ // Storage Contract
194
+ // ============================================================================
195
+
196
+ /**
197
+ * Contract for implementing custom storage adapters
198
+ *
199
+ * **IMPORTANT: Sort Order Requirement**
200
+ *
201
+ * The `list()` method MUST return results sorted by `startTime` in descending order
202
+ * (most recent first). This is required by `ArtemisKit.compare()` when using
203
+ * `baseline: 'latest'` - it relies on `list({ limit: 1 })[0]` being the most recent run.
204
+ *
205
+ * If your storage backend doesn't natively support this sort order, you must
206
+ * implement sorting in the adapter's `list()` method.
207
+ *
208
+ * @example
209
+ * ```typescript
210
+ * import { defineStorage, type StorageContract } from '@artemiskit/sdk'
211
+ *
212
+ * const redisStorage = defineStorage({
213
+ * async save(manifest) {
214
+ * await redis.set(`run:${manifest.run_id}`, JSON.stringify(manifest))
215
+ * return manifest.run_id
216
+ * },
217
+ * async load(runId) {
218
+ * const data = await redis.get(`run:${runId}`)
219
+ * return JSON.parse(data)
220
+ * },
221
+ * async list(options) {
222
+ * // IMPORTANT: Results must be sorted by startTime descending (most recent first)
223
+ * const runs = await this.getAllRuns()
224
+ * runs.sort((a, b) => b.startTime.getTime() - a.startTime.getTime())
225
+ * return runs.slice(0, options?.limit ?? runs.length)
226
+ * },
227
+ * async delete(runId) {
228
+ * await redis.del(`run:${runId}`)
229
+ * }
230
+ * })
231
+ * ```
232
+ */
233
+ export interface StorageContract extends StorageAdapter {}
234
+
235
+ /**
236
+ * Define a custom storage adapter with type checking
237
+ *
238
+ * @example
239
+ * ```typescript
240
+ * const s3Storage = defineStorage({
241
+ * async save(manifest) {
242
+ * const key = `runs/${manifest.run_id}.json`
243
+ * await s3.putObject({ Bucket: 'my-bucket', Key: key, Body: JSON.stringify(manifest) })
244
+ * return manifest.run_id
245
+ * },
246
+ * async load(runId) {
247
+ * const key = `runs/${runId}.json`
248
+ * const { Body } = await s3.getObject({ Bucket: 'my-bucket', Key: key })
249
+ * return JSON.parse(Body)
250
+ * },
251
+ * async list(options) {
252
+ * // List objects from S3
253
+ * return []
254
+ * },
255
+ * async delete(runId) {
256
+ * const key = `runs/${runId}.json`
257
+ * await s3.deleteObject({ Bucket: 'my-bucket', Key: key })
258
+ * }
259
+ * })
260
+ * ```
261
+ */
262
+ export function defineStorage<T extends StorageContract>(storage: T): T {
263
+ // Validate required methods
264
+ if (typeof storage.save !== 'function') {
265
+ throw new Error('Storage must have a save method');
266
+ }
267
+ if (typeof storage.load !== 'function') {
268
+ throw new Error('Storage must have a load method');
269
+ }
270
+ if (typeof storage.list !== 'function') {
271
+ throw new Error('Storage must have a list method');
272
+ }
273
+ if (typeof storage.delete !== 'function') {
274
+ throw new Error('Storage must have a delete method');
275
+ }
276
+ return storage;
277
+ }
278
+
279
+ // ============================================================================
280
+ // Factory Types
281
+ // ============================================================================
282
+
283
+ /**
284
+ * Factory function type for creating adapters
285
+ */
286
+ export type AdapterFactory<TConfig = unknown> = (
287
+ config: TConfig
288
+ ) => AdapterContract | Promise<AdapterContract>;
289
+
290
+ /**
291
+ * Factory function type for creating evaluators
292
+ */
293
+ export type EvaluatorFactory<TConfig = unknown> = (config: TConfig) => EvaluatorContract;
294
+
295
+ /**
296
+ * Factory function type for creating storage adapters
297
+ */
298
+ export type StorageFactory<TConfig = unknown> = (
299
+ config: TConfig
300
+ ) => StorageContract | Promise<StorageContract>;
301
+
302
+ // ============================================================================
303
+ // Plugin System Types
304
+ // ============================================================================
305
+
306
+ /**
307
+ * Plugin interface for extending ArtemisKit
308
+ */
309
+ export interface ArtemisKitPlugin {
310
+ /**
311
+ * Plugin name (unique identifier)
312
+ */
313
+ name: string;
314
+
315
+ /**
316
+ * Plugin version
317
+ */
318
+ version: string;
319
+
320
+ /**
321
+ * Optional: Custom adapters to register
322
+ */
323
+ adapters?: Record<string, AdapterFactory>;
324
+
325
+ /**
326
+ * Optional: Custom evaluators to register
327
+ */
328
+ evaluators?: Record<string, EvaluatorFactory>;
329
+
330
+ /**
331
+ * Optional: Custom storage adapters to register
332
+ */
333
+ storage?: Record<string, StorageFactory>;
334
+
335
+ /**
336
+ * Optional: Initialize the plugin
337
+ */
338
+ init?(): Promise<void>;
339
+
340
+ /**
341
+ * Optional: Cleanup the plugin
342
+ */
343
+ cleanup?(): Promise<void>;
344
+ }
345
+
346
+ /**
347
+ * Define a plugin with type checking
348
+ */
349
+ export function definePlugin(plugin: ArtemisKitPlugin): ArtemisKitPlugin {
350
+ if (!plugin.name) {
351
+ throw new Error('Plugin must have a name');
352
+ }
353
+ if (!plugin.version) {
354
+ throw new Error('Plugin must have a version');
355
+ }
356
+ return plugin;
357
+ }
@@ -77,23 +77,11 @@ export class CircuitBreaker {
77
77
 
78
78
  /**
79
79
  * Check if the circuit allows requests
80
+ * @deprecated Use isOpen() instead - this method duplicates functionality.
81
+ * Guardian uses isOpen() exclusively for circuit breaker checks.
80
82
  */
81
83
  allowRequest(): boolean {
82
- if (!this.config.enabled) {
83
- return true;
84
- }
85
-
86
- if (this.state === 'open') {
87
- // Check if cooldown has passed
88
- const now = Date.now();
89
- if (now - this.lastOpenTime >= this.config.cooldownMs) {
90
- this.transition('half-open');
91
- return true;
92
- }
93
- return false;
94
- }
95
-
96
- return true;
84
+ return !this.isOpen();
97
85
  }
98
86
 
99
87
  /**
@@ -165,9 +153,14 @@ export class CircuitBreaker {
165
153
  }
166
154
 
167
155
  /**
168
- * Force the circuit to a specific state
156
+ * Force the circuit to a specific state.
157
+ * When forcing to 'open', also updates lastOpenTime so the cooldown
158
+ * period starts from now (matching trip() behavior).
169
159
  */
170
160
  forceState(state: CircuitBreakerState): void {
161
+ if (state === 'open') {
162
+ this.lastOpenTime = Date.now();
163
+ }
171
164
  this.transition(state);
172
165
  }
173
166
 
@@ -371,11 +364,13 @@ export class MetricsCollector {
371
364
  const rps = totalTimeMs > 0 ? (totalRequests * 1000) / totalTimeMs : 0;
372
365
 
373
366
  // Calculate cost tracking
367
+ // costPerMinute is computed as an actual rate (cost per 60s) based on recent activity
368
+ const costPerMinute = this.getCostPerMinute();
374
369
  const costTracking: CostTracking | undefined =
375
370
  totalCost > 0
376
371
  ? {
377
372
  totalCost,
378
- costPerMinute: this.currentWindow.costSum,
373
+ costPerMinute,
379
374
  costPerHour: this.getHourlyCost(),
380
375
  costPerDay: this.getDailyCost(),
381
376
  currency: this.currency,
@@ -420,6 +415,38 @@ export class MetricsCollector {
420
415
  this.currentWindow = this.createWindow();
421
416
  }
422
417
 
418
+ /**
419
+ * Get cost per minute (actual rate based on elapsed time)
420
+ *
421
+ * To avoid extreme extrapolated values immediately after window rotation,
422
+ * we require a minimum duration of data before extrapolating. If insufficient
423
+ * data is available in the current window, we fall back to historical data.
424
+ */
425
+ private getCostPerMinute(): number {
426
+ const now = Date.now();
427
+ const windowElapsed = now - this.currentWindow.startTime;
428
+ // Require at least 5 seconds of data before extrapolating
429
+ const minDurationMs = 5000;
430
+
431
+ if (windowElapsed >= minDurationMs) {
432
+ // Sufficient data in current window - extrapolate to a full minute
433
+ const effectiveDuration = Math.min(windowElapsed, 60000);
434
+ return (this.currentWindow.costSum * 60000) / effectiveDuration;
435
+ }
436
+
437
+ // Not enough data in current window - use historical data if available
438
+ if (this.history.length > 0) {
439
+ const lastWindow = this.history[this.history.length - 1];
440
+ if (lastWindow.costSum > 0) {
441
+ // Use the last complete window's rate
442
+ return (lastWindow.costSum * 60000) / this.windowDurationMs;
443
+ }
444
+ }
445
+
446
+ // No historical data either - return 0 to avoid misleading extrapolation
447
+ return 0;
448
+ }
449
+
423
450
  /**
424
451
  * Get hourly cost
425
452
  */
@@ -629,27 +656,46 @@ export class RateLimiter {
629
656
 
630
657
  if (tokensToAdd > 0) {
631
658
  this.burstTokens = Math.min(this.config.burstLimit, this.burstTokens + tokensToAdd);
632
- this.lastBurstRefill = now;
659
+ // Preserve sub-second precision to avoid refill rate drift
660
+ this.lastBurstRefill += tokensToAdd * 1000;
633
661
  }
634
662
  }
635
663
 
636
664
  /**
637
- * Rotate time windows
665
+ * Rotate time windows using sliding window approximation.
666
+ *
667
+ * To prevent the fixed-window burst vulnerability (allowing 2× the limit at
668
+ * window boundaries), we use a sliding window approximation: when a window
669
+ * rotates, we carry over a proportional count from the previous window based
670
+ * on how much of the new window overlaps with the old one.
638
671
  */
639
672
  private rotateWindows(now: number): void {
640
- // Rotate minute window
641
- if (now - this.minuteWindow.startTime >= 60000) {
642
- this.minuteWindow = { count: 0, startTime: now };
673
+ // Rotate minute window with sliding approximation
674
+ const minuteElapsed = now - this.minuteWindow.startTime;
675
+ if (minuteElapsed >= 60000) {
676
+ // Calculate overlap ratio: how much of the previous window is still relevant
677
+ const overlapMs = Math.max(0, 120000 - minuteElapsed); // 2 windows - elapsed
678
+ const overlapRatio = Math.min(1, overlapMs / 60000);
679
+ const carryOver = Math.floor(this.minuteWindow.count * overlapRatio);
680
+ this.minuteWindow = { count: carryOver, startTime: now - (minuteElapsed % 60000) };
643
681
  }
644
682
 
645
- // Rotate hour window
646
- if (now - this.hourWindow.startTime >= 3600000) {
647
- this.hourWindow = { count: 0, startTime: now };
683
+ // Rotate hour window with sliding approximation
684
+ const hourElapsed = now - this.hourWindow.startTime;
685
+ if (hourElapsed >= 3600000) {
686
+ const overlapMs = Math.max(0, 7200000 - hourElapsed);
687
+ const overlapRatio = Math.min(1, overlapMs / 3600000);
688
+ const carryOver = Math.floor(this.hourWindow.count * overlapRatio);
689
+ this.hourWindow = { count: carryOver, startTime: now - (hourElapsed % 3600000) };
648
690
  }
649
691
 
650
- // Rotate day window
651
- if (now - this.dayWindow.startTime >= 86400000) {
652
- this.dayWindow = { count: 0, startTime: now };
692
+ // Rotate day window with sliding approximation
693
+ const dayElapsed = now - this.dayWindow.startTime;
694
+ if (dayElapsed >= 86400000) {
695
+ const overlapMs = Math.max(0, 172800000 - dayElapsed);
696
+ const overlapRatio = Math.min(1, overlapMs / 86400000);
697
+ const carryOver = Math.floor(this.dayWindow.count * overlapRatio);
698
+ this.dayWindow = { count: carryOver, startTime: now - (dayElapsed % 86400000) };
653
699
  }
654
700
  }
655
701
  }