@artemiskit/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = import.meta.require;
|
|
47
|
+
|
|
48
|
+
// src/builders/index.ts
|
|
49
|
+
class TestCaseBuilder {
|
|
50
|
+
_id;
|
|
51
|
+
_name;
|
|
52
|
+
_description;
|
|
53
|
+
_prompt = "";
|
|
54
|
+
_expected;
|
|
55
|
+
_tags = [];
|
|
56
|
+
_metadata = {};
|
|
57
|
+
_timeout;
|
|
58
|
+
_retries = 0;
|
|
59
|
+
_provider;
|
|
60
|
+
_model;
|
|
61
|
+
_variables;
|
|
62
|
+
_redaction;
|
|
63
|
+
constructor(id) {
|
|
64
|
+
this._id = id;
|
|
65
|
+
}
|
|
66
|
+
name(name) {
|
|
67
|
+
this._name = name;
|
|
68
|
+
return this;
|
|
69
|
+
}
|
|
70
|
+
description(description) {
|
|
71
|
+
this._description = description;
|
|
72
|
+
return this;
|
|
73
|
+
}
|
|
74
|
+
prompt(prompt) {
|
|
75
|
+
this._prompt = prompt;
|
|
76
|
+
return this;
|
|
77
|
+
}
|
|
78
|
+
messages(messages) {
|
|
79
|
+
this._prompt = messages;
|
|
80
|
+
return this;
|
|
81
|
+
}
|
|
82
|
+
systemAndUser(systemPrompt, userPrompt) {
|
|
83
|
+
this._prompt = [
|
|
84
|
+
{ role: "system", content: systemPrompt },
|
|
85
|
+
{ role: "user", content: userPrompt }
|
|
86
|
+
];
|
|
87
|
+
return this;
|
|
88
|
+
}
|
|
89
|
+
expectExact(value, caseSensitive = true) {
|
|
90
|
+
this._expected = { type: "exact", value, caseSensitive };
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
expectContains(values, mode = "all") {
|
|
94
|
+
this._expected = { type: "contains", values, mode };
|
|
95
|
+
return this;
|
|
96
|
+
}
|
|
97
|
+
expectNotContains(values, mode = "all") {
|
|
98
|
+
this._expected = { type: "not_contains", values, mode };
|
|
99
|
+
return this;
|
|
100
|
+
}
|
|
101
|
+
expectRegex(pattern, flags) {
|
|
102
|
+
this._expected = { type: "regex", pattern, flags };
|
|
103
|
+
return this;
|
|
104
|
+
}
|
|
105
|
+
expectFuzzy(value, threshold = 0.8) {
|
|
106
|
+
this._expected = { type: "fuzzy", value, threshold };
|
|
107
|
+
return this;
|
|
108
|
+
}
|
|
109
|
+
expectJsonSchema(schema) {
|
|
110
|
+
this._expected = { type: "json_schema", schema };
|
|
111
|
+
return this;
|
|
112
|
+
}
|
|
113
|
+
expectLLMGrade(rubric, options) {
|
|
114
|
+
this._expected = {
|
|
115
|
+
type: "llm_grader",
|
|
116
|
+
rubric,
|
|
117
|
+
threshold: options?.threshold ?? 0.7,
|
|
118
|
+
model: options?.model,
|
|
119
|
+
provider: options?.provider
|
|
120
|
+
};
|
|
121
|
+
return this;
|
|
122
|
+
}
|
|
123
|
+
expectSimilarity(value, options) {
|
|
124
|
+
this._expected = {
|
|
125
|
+
type: "similarity",
|
|
126
|
+
value,
|
|
127
|
+
threshold: options?.threshold ?? 0.75,
|
|
128
|
+
mode: options?.mode,
|
|
129
|
+
model: options?.model,
|
|
130
|
+
embeddingModel: options?.embeddingModel
|
|
131
|
+
};
|
|
132
|
+
return this;
|
|
133
|
+
}
|
|
134
|
+
expectInline(expression, value) {
|
|
135
|
+
this._expected = { type: "inline", expression, value };
|
|
136
|
+
return this;
|
|
137
|
+
}
|
|
138
|
+
expectAll(...expectations) {
|
|
139
|
+
this._expected = { type: "combined", operator: "and", expectations };
|
|
140
|
+
return this;
|
|
141
|
+
}
|
|
142
|
+
expectAny(...expectations) {
|
|
143
|
+
this._expected = { type: "combined", operator: "or", expectations };
|
|
144
|
+
return this;
|
|
145
|
+
}
|
|
146
|
+
expect(expected) {
|
|
147
|
+
this._expected = expected;
|
|
148
|
+
return this;
|
|
149
|
+
}
|
|
150
|
+
tags(...tags) {
|
|
151
|
+
this._tags.push(...tags);
|
|
152
|
+
return this;
|
|
153
|
+
}
|
|
154
|
+
metadata(data) {
|
|
155
|
+
this._metadata = { ...this._metadata, ...data };
|
|
156
|
+
return this;
|
|
157
|
+
}
|
|
158
|
+
timeout(ms) {
|
|
159
|
+
this._timeout = ms;
|
|
160
|
+
return this;
|
|
161
|
+
}
|
|
162
|
+
retries(count) {
|
|
163
|
+
this._retries = count;
|
|
164
|
+
return this;
|
|
165
|
+
}
|
|
166
|
+
provider(provider) {
|
|
167
|
+
this._provider = provider;
|
|
168
|
+
return this;
|
|
169
|
+
}
|
|
170
|
+
model(model) {
|
|
171
|
+
this._model = model;
|
|
172
|
+
return this;
|
|
173
|
+
}
|
|
174
|
+
variables(vars) {
|
|
175
|
+
this._variables = vars;
|
|
176
|
+
return this;
|
|
177
|
+
}
|
|
178
|
+
redact(config) {
|
|
179
|
+
this._redaction = config;
|
|
180
|
+
return this;
|
|
181
|
+
}
|
|
182
|
+
build() {
|
|
183
|
+
if (!this._expected) {
|
|
184
|
+
throw new Error(`TestCase "${this._id}" must have an expectation. Call one of the expect* methods.`);
|
|
185
|
+
}
|
|
186
|
+
const hasEmptyPrompt = !this._prompt || typeof this._prompt === "string" && this._prompt.length === 0 || Array.isArray(this._prompt) && this._prompt.length === 0;
|
|
187
|
+
if (hasEmptyPrompt) {
|
|
188
|
+
throw new Error(`TestCase "${this._id}" must have a prompt.`);
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
id: this._id,
|
|
192
|
+
name: this._name,
|
|
193
|
+
description: this._description,
|
|
194
|
+
prompt: this._prompt,
|
|
195
|
+
expected: this._expected,
|
|
196
|
+
tags: this._tags,
|
|
197
|
+
metadata: this._metadata,
|
|
198
|
+
timeout: this._timeout,
|
|
199
|
+
retries: this._retries,
|
|
200
|
+
provider: this._provider,
|
|
201
|
+
model: this._model,
|
|
202
|
+
variables: this._variables,
|
|
203
|
+
redaction: this._redaction
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
class ScenarioBuilder {
|
|
209
|
+
_name;
|
|
210
|
+
_description;
|
|
211
|
+
_version = "1.0";
|
|
212
|
+
_provider;
|
|
213
|
+
_model;
|
|
214
|
+
_providerConfig;
|
|
215
|
+
_seed;
|
|
216
|
+
_temperature;
|
|
217
|
+
_maxTokens;
|
|
218
|
+
_tags = [];
|
|
219
|
+
_variables;
|
|
220
|
+
_redaction;
|
|
221
|
+
_systemPrompt;
|
|
222
|
+
_cases = [];
|
|
223
|
+
constructor(name) {
|
|
224
|
+
this._name = name;
|
|
225
|
+
}
|
|
226
|
+
description(description) {
|
|
227
|
+
this._description = description;
|
|
228
|
+
return this;
|
|
229
|
+
}
|
|
230
|
+
version(version) {
|
|
231
|
+
this._version = version;
|
|
232
|
+
return this;
|
|
233
|
+
}
|
|
234
|
+
provider(provider) {
|
|
235
|
+
this._provider = provider;
|
|
236
|
+
return this;
|
|
237
|
+
}
|
|
238
|
+
model(model) {
|
|
239
|
+
this._model = model;
|
|
240
|
+
return this;
|
|
241
|
+
}
|
|
242
|
+
providerConfig(config) {
|
|
243
|
+
this._providerConfig = config;
|
|
244
|
+
return this;
|
|
245
|
+
}
|
|
246
|
+
seed(seed) {
|
|
247
|
+
this._seed = seed;
|
|
248
|
+
return this;
|
|
249
|
+
}
|
|
250
|
+
temperature(temp) {
|
|
251
|
+
this._temperature = temp;
|
|
252
|
+
return this;
|
|
253
|
+
}
|
|
254
|
+
maxTokens(tokens) {
|
|
255
|
+
this._maxTokens = tokens;
|
|
256
|
+
return this;
|
|
257
|
+
}
|
|
258
|
+
tags(...tags) {
|
|
259
|
+
this._tags.push(...tags);
|
|
260
|
+
return this;
|
|
261
|
+
}
|
|
262
|
+
variables(vars) {
|
|
263
|
+
this._variables = vars;
|
|
264
|
+
return this;
|
|
265
|
+
}
|
|
266
|
+
redact(config) {
|
|
267
|
+
this._redaction = config;
|
|
268
|
+
return this;
|
|
269
|
+
}
|
|
270
|
+
systemPrompt(prompt) {
|
|
271
|
+
this._systemPrompt = prompt;
|
|
272
|
+
return this;
|
|
273
|
+
}
|
|
274
|
+
case(caseBuilder) {
|
|
275
|
+
this._cases.push(caseBuilder.build());
|
|
276
|
+
return this;
|
|
277
|
+
}
|
|
278
|
+
addCase(testCase) {
|
|
279
|
+
this._cases.push(testCase);
|
|
280
|
+
return this;
|
|
281
|
+
}
|
|
282
|
+
cases(...caseBuilders) {
|
|
283
|
+
for (const builder of caseBuilders) {
|
|
284
|
+
this._cases.push(builder.build());
|
|
285
|
+
}
|
|
286
|
+
return this;
|
|
287
|
+
}
|
|
288
|
+
build() {
|
|
289
|
+
if (this._cases.length === 0) {
|
|
290
|
+
throw new Error(`Scenario "${this._name}" must have at least one test case.`);
|
|
291
|
+
}
|
|
292
|
+
return {
|
|
293
|
+
name: this._name,
|
|
294
|
+
description: this._description,
|
|
295
|
+
version: this._version,
|
|
296
|
+
provider: this._provider,
|
|
297
|
+
model: this._model,
|
|
298
|
+
providerConfig: this._providerConfig,
|
|
299
|
+
seed: this._seed,
|
|
300
|
+
temperature: this._temperature,
|
|
301
|
+
maxTokens: this._maxTokens,
|
|
302
|
+
tags: this._tags,
|
|
303
|
+
variables: this._variables,
|
|
304
|
+
redaction: this._redaction,
|
|
305
|
+
setup: this._systemPrompt ? { systemPrompt: this._systemPrompt } : undefined,
|
|
306
|
+
cases: this._cases
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
function scenario(name) {
|
|
311
|
+
return new ScenarioBuilder(name);
|
|
312
|
+
}
|
|
313
|
+
function testCase(id) {
|
|
314
|
+
return new TestCaseBuilder(id);
|
|
315
|
+
}
|
|
316
|
+
function containsCase(id, prompt, values, mode = "all") {
|
|
317
|
+
return testCase(id).prompt(prompt).expectContains(values, mode).build();
|
|
318
|
+
}
|
|
319
|
+
function exactCase(id, prompt, value) {
|
|
320
|
+
return testCase(id).prompt(prompt).expectExact(value).build();
|
|
321
|
+
}
|
|
322
|
+
function regexCase(id, prompt, pattern, flags) {
|
|
323
|
+
return testCase(id).prompt(prompt).expectRegex(pattern, flags).build();
|
|
324
|
+
}
|
|
325
|
+
function jsonCase(id, prompt, schema) {
|
|
326
|
+
return testCase(id).prompt(prompt).expectJsonSchema(schema).build();
|
|
327
|
+
}
|
|
328
|
+
function gradedCase(id, prompt, rubric, threshold = 0.7) {
|
|
329
|
+
return testCase(id).prompt(prompt).expectLLMGrade(rubric, { threshold }).build();
|
|
330
|
+
}
|
|
331
|
+
function exact(value, caseSensitive = true) {
|
|
332
|
+
return { type: "exact", value, caseSensitive };
|
|
333
|
+
}
|
|
334
|
+
function contains(values, mode = "all") {
|
|
335
|
+
return { type: "contains", values, mode };
|
|
336
|
+
}
|
|
337
|
+
function notContains(values, mode = "all") {
|
|
338
|
+
return { type: "not_contains", values, mode };
|
|
339
|
+
}
|
|
340
|
+
function regex(pattern, flags) {
|
|
341
|
+
return { type: "regex", pattern, flags };
|
|
342
|
+
}
|
|
343
|
+
function fuzzy(value, threshold = 0.8) {
|
|
344
|
+
return { type: "fuzzy", value, threshold };
|
|
345
|
+
}
|
|
346
|
+
function jsonSchema(schema) {
|
|
347
|
+
return { type: "json_schema", schema };
|
|
348
|
+
}
|
|
349
|
+
function llmGrade(rubric, options) {
|
|
350
|
+
return {
|
|
351
|
+
type: "llm_grader",
|
|
352
|
+
rubric,
|
|
353
|
+
threshold: options?.threshold ?? 0.7,
|
|
354
|
+
model: options?.model,
|
|
355
|
+
provider: options?.provider
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
function similarity(value, options) {
|
|
359
|
+
return {
|
|
360
|
+
type: "similarity",
|
|
361
|
+
value,
|
|
362
|
+
threshold: options?.threshold ?? 0.75,
|
|
363
|
+
mode: options?.mode,
|
|
364
|
+
model: options?.model,
|
|
365
|
+
embeddingModel: options?.embeddingModel
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
function inline(expression, value) {
|
|
369
|
+
return { type: "inline", expression, value };
|
|
370
|
+
}
|
|
371
|
+
function allOf(...expectations) {
|
|
372
|
+
return { type: "combined", operator: "and", expectations };
|
|
373
|
+
}
|
|
374
|
+
function anyOf(...expectations) {
|
|
375
|
+
return { type: "combined", operator: "or", expectations };
|
|
376
|
+
}
|
|
377
|
+
export {
|
|
378
|
+
testCase,
|
|
379
|
+
similarity,
|
|
380
|
+
scenario,
|
|
381
|
+
regexCase,
|
|
382
|
+
regex,
|
|
383
|
+
notContains,
|
|
384
|
+
llmGrade,
|
|
385
|
+
jsonSchema,
|
|
386
|
+
jsonCase,
|
|
387
|
+
inline,
|
|
388
|
+
gradedCase,
|
|
389
|
+
fuzzy,
|
|
390
|
+
exactCase,
|
|
391
|
+
exact,
|
|
392
|
+
containsCase,
|
|
393
|
+
contains,
|
|
394
|
+
anyOf,
|
|
395
|
+
allOf,
|
|
396
|
+
TestCaseBuilder,
|
|
397
|
+
ScenarioBuilder
|
|
398
|
+
};
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Contract types for custom implementations
|
|
4
|
+
*
|
|
5
|
+
* These types define the contracts that custom adapters, evaluators,
|
|
6
|
+
* and storage backends must implement to integrate with ArtemisKit.
|
|
7
|
+
*/
|
|
8
|
+
import type { Evaluator, EvaluatorContext, EvaluatorResult, Expected, GenerateOptions, GenerateResult, ModelCapabilities, ModelClient, StorageAdapter } from '@artemiskit/core';
|
|
9
|
+
/**
|
|
10
|
+
* Contract for implementing custom model adapters
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import { defineAdapter, type AdapterContract } from '@artemiskit/sdk'
|
|
15
|
+
*
|
|
16
|
+
* const myAdapter = defineAdapter({
|
|
17
|
+
* provider: 'my-provider',
|
|
18
|
+
* async generate(options) {
|
|
19
|
+
* // Implementation
|
|
20
|
+
* return { id: '...', model: '...', text: '...', tokens: {...}, latencyMs: 0 }
|
|
21
|
+
* },
|
|
22
|
+
* async capabilities() {
|
|
23
|
+
* return { streaming: false, functionCalling: false, toolUse: false, maxContext: 4096 }
|
|
24
|
+
* }
|
|
25
|
+
* })
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export interface AdapterContract extends ModelClient {
|
|
29
|
+
/**
|
|
30
|
+
* Provider identifier - must be unique
|
|
31
|
+
*/
|
|
32
|
+
readonly provider: string;
|
|
33
|
+
/**
|
|
34
|
+
* Generate a completion from the model
|
|
35
|
+
*/
|
|
36
|
+
generate(options: GenerateOptions): Promise<GenerateResult>;
|
|
37
|
+
/**
|
|
38
|
+
* Get the capabilities of this adapter
|
|
39
|
+
*/
|
|
40
|
+
capabilities(): Promise<ModelCapabilities>;
|
|
41
|
+
/**
|
|
42
|
+
* Optional: Stream a completion from the model
|
|
43
|
+
*/
|
|
44
|
+
stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string>;
|
|
45
|
+
/**
|
|
46
|
+
* Optional: Generate embeddings
|
|
47
|
+
*/
|
|
48
|
+
embed?(text: string, model?: string): Promise<number[]>;
|
|
49
|
+
/**
|
|
50
|
+
* Optional: Close/cleanup resources
|
|
51
|
+
*/
|
|
52
|
+
close?(): Promise<void>;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Define a custom adapter with type checking
|
|
56
|
+
*
|
|
57
|
+
* @example
|
|
58
|
+
* ```typescript
|
|
59
|
+
* const myAdapter = defineAdapter({
|
|
60
|
+
* provider: 'my-llm',
|
|
61
|
+
* async generate(options) {
|
|
62
|
+
* const response = await myLLMClient.complete(options.prompt)
|
|
63
|
+
* return {
|
|
64
|
+
* id: response.id,
|
|
65
|
+
* model: 'my-model',
|
|
66
|
+
* text: response.text,
|
|
67
|
+
* tokens: { prompt: 0, completion: 0, total: 0 },
|
|
68
|
+
* latencyMs: response.duration
|
|
69
|
+
* }
|
|
70
|
+
* },
|
|
71
|
+
* async capabilities() {
|
|
72
|
+
* return { streaming: true, functionCalling: false, toolUse: false, maxContext: 8192 }
|
|
73
|
+
* }
|
|
74
|
+
* })
|
|
75
|
+
* ```
|
|
76
|
+
*/
|
|
77
|
+
export declare function defineAdapter<T extends AdapterContract>(adapter: T): T;
|
|
78
|
+
/**
|
|
79
|
+
* Contract for implementing custom evaluators
|
|
80
|
+
*
|
|
81
|
+
* @example
|
|
82
|
+
* ```typescript
|
|
83
|
+
* import { defineEvaluator, type EvaluatorContract } from '@artemiskit/sdk'
|
|
84
|
+
*
|
|
85
|
+
* const customEvaluator = defineEvaluator({
|
|
86
|
+
* type: 'custom-semantic',
|
|
87
|
+
* async evaluate(response, expected, context) {
|
|
88
|
+
* // Custom evaluation logic
|
|
89
|
+
* const score = calculateSemanticScore(response, expected.value)
|
|
90
|
+
* return { passed: score > 0.8, score, reason: `Semantic score: ${score}` }
|
|
91
|
+
* }
|
|
92
|
+
* })
|
|
93
|
+
* ```
|
|
94
|
+
*/
|
|
95
|
+
export interface EvaluatorContract extends Evaluator {
|
|
96
|
+
/**
|
|
97
|
+
* Unique type identifier for this evaluator
|
|
98
|
+
*/
|
|
99
|
+
readonly type: string;
|
|
100
|
+
/**
|
|
101
|
+
* Evaluate a response against expected criteria
|
|
102
|
+
*
|
|
103
|
+
* @param response - The model's response text
|
|
104
|
+
* @param expected - The expected result configuration
|
|
105
|
+
* @param context - Optional context including model client and test case
|
|
106
|
+
* @returns Evaluation result with pass/fail, score, and reason
|
|
107
|
+
*/
|
|
108
|
+
evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Define a custom evaluator with type checking
|
|
112
|
+
*
|
|
113
|
+
* @example
|
|
114
|
+
* ```typescript
|
|
115
|
+
* const wordCountEvaluator = defineEvaluator({
|
|
116
|
+
* type: 'word-count',
|
|
117
|
+
* async evaluate(response, expected, _context) {
|
|
118
|
+
* const wordCount = response.split(/\s+/).length
|
|
119
|
+
* const targetCount = (expected as any).wordCount ?? 100
|
|
120
|
+
* const tolerance = (expected as any).tolerance ?? 10
|
|
121
|
+
*
|
|
122
|
+
* const diff = Math.abs(wordCount - targetCount)
|
|
123
|
+
* const passed = diff <= tolerance
|
|
124
|
+
* const score = passed ? 1 : Math.max(0, 1 - diff / targetCount)
|
|
125
|
+
*
|
|
126
|
+
* return {
|
|
127
|
+
* passed,
|
|
128
|
+
* score,
|
|
129
|
+
* reason: `Word count: ${wordCount}, target: ${targetCount} (±${tolerance})`,
|
|
130
|
+
* details: { wordCount, targetCount, tolerance, diff }
|
|
131
|
+
* }
|
|
132
|
+
* }
|
|
133
|
+
* })
|
|
134
|
+
* ```
|
|
135
|
+
*/
|
|
136
|
+
export declare function defineEvaluator<T extends EvaluatorContract>(evaluator: T): T;
|
|
137
|
+
/**
|
|
138
|
+
* Contract for implementing custom storage adapters
|
|
139
|
+
*
|
|
140
|
+
* **IMPORTANT: Sort Order Requirement**
|
|
141
|
+
*
|
|
142
|
+
* The `list()` method MUST return results sorted by `startTime` in descending order
|
|
143
|
+
* (most recent first). This is required by `ArtemisKit.compare()` when using
|
|
144
|
+
* `baseline: 'latest'` - it relies on `list({ limit: 1 })[0]` being the most recent run.
|
|
145
|
+
*
|
|
146
|
+
* If your storage backend doesn't natively support this sort order, you must
|
|
147
|
+
* implement sorting in the adapter's `list()` method.
|
|
148
|
+
*
|
|
149
|
+
* @example
|
|
150
|
+
* ```typescript
|
|
151
|
+
* import { defineStorage, type StorageContract } from '@artemiskit/sdk'
|
|
152
|
+
*
|
|
153
|
+
* const redisStorage = defineStorage({
|
|
154
|
+
* async save(manifest) {
|
|
155
|
+
* await redis.set(`run:${manifest.run_id}`, JSON.stringify(manifest))
|
|
156
|
+
* return manifest.run_id
|
|
157
|
+
* },
|
|
158
|
+
* async load(runId) {
|
|
159
|
+
* const data = await redis.get(`run:${runId}`)
|
|
160
|
+
* return JSON.parse(data)
|
|
161
|
+
* },
|
|
162
|
+
* async list(options) {
|
|
163
|
+
* // IMPORTANT: Results must be sorted by startTime descending (most recent first)
|
|
164
|
+
* const runs = await this.getAllRuns()
|
|
165
|
+
* runs.sort((a, b) => b.startTime.getTime() - a.startTime.getTime())
|
|
166
|
+
* return runs.slice(0, options?.limit ?? runs.length)
|
|
167
|
+
* },
|
|
168
|
+
* async delete(runId) {
|
|
169
|
+
* await redis.del(`run:${runId}`)
|
|
170
|
+
* }
|
|
171
|
+
* })
|
|
172
|
+
* ```
|
|
173
|
+
*/
|
|
174
|
+
export interface StorageContract extends StorageAdapter {
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Define a custom storage adapter with type checking
|
|
178
|
+
*
|
|
179
|
+
* @example
|
|
180
|
+
* ```typescript
|
|
181
|
+
* const s3Storage = defineStorage({
|
|
182
|
+
* async save(manifest) {
|
|
183
|
+
* const key = `runs/${manifest.run_id}.json`
|
|
184
|
+
* await s3.putObject({ Bucket: 'my-bucket', Key: key, Body: JSON.stringify(manifest) })
|
|
185
|
+
* return manifest.run_id
|
|
186
|
+
* },
|
|
187
|
+
* async load(runId) {
|
|
188
|
+
* const key = `runs/${runId}.json`
|
|
189
|
+
* const { Body } = await s3.getObject({ Bucket: 'my-bucket', Key: key })
|
|
190
|
+
* return JSON.parse(Body)
|
|
191
|
+
* },
|
|
192
|
+
* async list(options) {
|
|
193
|
+
* // List objects from S3
|
|
194
|
+
* return []
|
|
195
|
+
* },
|
|
196
|
+
* async delete(runId) {
|
|
197
|
+
* const key = `runs/${runId}.json`
|
|
198
|
+
* await s3.deleteObject({ Bucket: 'my-bucket', Key: key })
|
|
199
|
+
* }
|
|
200
|
+
* })
|
|
201
|
+
* ```
|
|
202
|
+
*/
|
|
203
|
+
export declare function defineStorage<T extends StorageContract>(storage: T): T;
|
|
204
|
+
/**
|
|
205
|
+
* Factory function type for creating adapters
|
|
206
|
+
*/
|
|
207
|
+
export type AdapterFactory<TConfig = unknown> = (config: TConfig) => AdapterContract | Promise<AdapterContract>;
|
|
208
|
+
/**
|
|
209
|
+
* Factory function type for creating evaluators
|
|
210
|
+
*/
|
|
211
|
+
export type EvaluatorFactory<TConfig = unknown> = (config: TConfig) => EvaluatorContract;
|
|
212
|
+
/**
|
|
213
|
+
* Factory function type for creating storage adapters
|
|
214
|
+
*/
|
|
215
|
+
export type StorageFactory<TConfig = unknown> = (config: TConfig) => StorageContract | Promise<StorageContract>;
|
|
216
|
+
/**
|
|
217
|
+
* Plugin interface for extending ArtemisKit
|
|
218
|
+
*/
|
|
219
|
+
export interface ArtemisKitPlugin {
|
|
220
|
+
/**
|
|
221
|
+
* Plugin name (unique identifier)
|
|
222
|
+
*/
|
|
223
|
+
name: string;
|
|
224
|
+
/**
|
|
225
|
+
* Plugin version
|
|
226
|
+
*/
|
|
227
|
+
version: string;
|
|
228
|
+
/**
|
|
229
|
+
* Optional: Custom adapters to register
|
|
230
|
+
*/
|
|
231
|
+
adapters?: Record<string, AdapterFactory>;
|
|
232
|
+
/**
|
|
233
|
+
* Optional: Custom evaluators to register
|
|
234
|
+
*/
|
|
235
|
+
evaluators?: Record<string, EvaluatorFactory>;
|
|
236
|
+
/**
|
|
237
|
+
* Optional: Custom storage adapters to register
|
|
238
|
+
*/
|
|
239
|
+
storage?: Record<string, StorageFactory>;
|
|
240
|
+
/**
|
|
241
|
+
* Optional: Initialize the plugin
|
|
242
|
+
*/
|
|
243
|
+
init?(): Promise<void>;
|
|
244
|
+
/**
|
|
245
|
+
* Optional: Cleanup the plugin
|
|
246
|
+
*/
|
|
247
|
+
cleanup?(): Promise<void>;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Define a plugin with type checking
|
|
251
|
+
*/
|
|
252
|
+
export declare function definePlugin(plugin: ArtemisKitPlugin): ArtemisKitPlugin;
|
|
253
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/contracts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,eAAe,EACf,QAAQ,EACR,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,WAAW,EACX,cAAc,EACf,MAAM,kBAAkB,CAAC;AAM1B;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,eAAgB,SAAQ,WAAW;IAClD;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C;;OAEG;IACH,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F;;OAEG;IACH,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD;;OAEG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,aAAa,CAAC,CAAC,SAAS,eAAe,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,CAYtE;AAMD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,iBAAkB,SAAQ,SAAS;IAClD;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;OAOG;IACH,QAAQ,CACN,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS,iBAAiB,EAAE,SAAS,EAAE,CAAC,GAAG,CAAC,CAS5E;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;CAAG;AAE1D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,aAAa,CAAC,CAAC,SAAS,eAAe,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,CAetE;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,OAAO,GAAG,OAAO,IAAI,CAC9C,MAAM,EAAE,OAAO,KACZ,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAEhD;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,iBAAiB,CAAC;AAEzF;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,OAAO,GAAG,OAAO,IAAI,CAC9C,MAAM,EAAE,OAAO,KACZ,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAMhD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAE1C;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAE9C;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAEzC;;OAEG;IACH,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;OAEG;IACH,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,gBAAgB,CAQvE"}
|