@artemiskit/sdk 0.3.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/builders/index.d.ts +2 -0
- package/dist/builders/index.d.ts.map +1 -1
- package/dist/builders/index.js +23 -67
- package/dist/contracts/index.js +3 -49
- package/dist/index.js +25805 -20620
- package/dist/matchers/index.js +13 -59
- package/dist/matchers/jest.js +0 -46
- package/dist/matchers/vitest.js +0 -46
- package/dist/types.d.ts +3 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/index.js +11 -57
- package/package.json +3 -3
- package/src/__tests__/builders.test.ts +9 -4
- package/src/__tests__/validate.test.ts +4 -1
- package/src/artemiskit.ts +1 -0
- package/src/builders/index.ts +4 -2
- package/src/types.ts +3 -0
- package/adapters/openai/dist/index.js +0 -5625
package/dist/matchers/index.js
CHANGED
|
@@ -1,50 +1,4 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = import.meta.require;
|
|
47
|
-
|
|
48
2
|
// src/matchers/core.ts
|
|
49
3
|
function formatManifestSummary(manifest) {
|
|
50
4
|
const { metrics } = manifest;
|
|
@@ -207,18 +161,18 @@ var artemiskitMatchers = {
|
|
|
207
161
|
toPassStressTest
|
|
208
162
|
};
|
|
209
163
|
export {
|
|
210
|
-
|
|
211
|
-
toPassRedTeam,
|
|
212
|
-
toPassCasesWithTag,
|
|
213
|
-
toPassAllCases,
|
|
214
|
-
toHaveSuccessRate,
|
|
215
|
-
toHaveStressSuccessRate,
|
|
216
|
-
toHaveStressP95LatencyBelow,
|
|
217
|
-
toHaveP95LatencyBelow,
|
|
218
|
-
toHaveNoHighSeverityVulnerabilities,
|
|
219
|
-
toHaveNoCriticalVulnerabilities,
|
|
220
|
-
toHaveMedianLatencyBelow,
|
|
221
|
-
toHaveDefenseRate,
|
|
164
|
+
artemiskitMatchers,
|
|
222
165
|
toAchieveRPS,
|
|
223
|
-
|
|
166
|
+
toHaveDefenseRate,
|
|
167
|
+
toHaveMedianLatencyBelow,
|
|
168
|
+
toHaveNoCriticalVulnerabilities,
|
|
169
|
+
toHaveNoHighSeverityVulnerabilities,
|
|
170
|
+
toHaveP95LatencyBelow,
|
|
171
|
+
toHaveStressP95LatencyBelow,
|
|
172
|
+
toHaveStressSuccessRate,
|
|
173
|
+
toHaveSuccessRate,
|
|
174
|
+
toPassAllCases,
|
|
175
|
+
toPassCasesWithTag,
|
|
176
|
+
toPassRedTeam,
|
|
177
|
+
toPassStressTest
|
|
224
178
|
};
|
package/dist/matchers/jest.js
CHANGED
|
@@ -1,50 +1,4 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = import.meta.require;
|
|
47
|
-
|
|
48
2
|
// src/matchers/core.ts
|
|
49
3
|
function formatManifestSummary(manifest) {
|
|
50
4
|
const { metrics } = manifest;
|
package/dist/matchers/vitest.js
CHANGED
|
@@ -1,50 +1,4 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = import.meta.require;
|
|
47
|
-
|
|
48
2
|
// src/matchers/core.ts
|
|
49
3
|
function formatManifestSummary(manifest) {
|
|
50
4
|
const { metrics } = manifest;
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Type definitions for the programmatic SDK
|
|
4
4
|
*/
|
|
5
|
-
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult } from '@artemiskit/core';
|
|
5
|
+
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
6
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
7
|
import type { Scenario } from '@artemiskit/core';
|
|
8
8
|
/**
|
|
@@ -116,6 +116,8 @@ export interface RunOptions {
|
|
|
116
116
|
retries?: number;
|
|
117
117
|
/** Redaction configuration */
|
|
118
118
|
redaction?: RedactionConfig;
|
|
119
|
+
/** Explicit executor for real tools; CLI scenarios remain fixture-only. */
|
|
120
|
+
toolExecutor?: ToolExecutor;
|
|
119
121
|
}
|
|
120
122
|
/**
|
|
121
123
|
* Options for red team testing
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,wCAAwC;IACxC,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
package/dist/utils/index.js
CHANGED
|
@@ -1,50 +1,4 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = import.meta.require;
|
|
47
|
-
|
|
48
2
|
// src/utils/index.ts
|
|
49
3
|
function isRunResult(result) {
|
|
50
4
|
return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
|
|
@@ -89,16 +43,16 @@ function calculateSuccessRate(result) {
|
|
|
89
43
|
return result.cases.filter((c) => c.ok).length / result.cases.length;
|
|
90
44
|
}
|
|
91
45
|
export {
|
|
92
|
-
|
|
93
|
-
isStressManifestType,
|
|
94
|
-
isRunResult,
|
|
95
|
-
isRunManifestType,
|
|
96
|
-
isRedTeamResult,
|
|
97
|
-
isRedTeamManifestType,
|
|
98
|
-
getPassedCases,
|
|
99
|
-
getFailedCases,
|
|
100
|
-
getCasesByTag,
|
|
101
|
-
calculateSuccessRate,
|
|
46
|
+
assert,
|
|
102
47
|
assertDefined,
|
|
103
|
-
|
|
48
|
+
calculateSuccessRate,
|
|
49
|
+
getCasesByTag,
|
|
50
|
+
getFailedCases,
|
|
51
|
+
getPassedCases,
|
|
52
|
+
isRedTeamManifestType,
|
|
53
|
+
isRedTeamResult,
|
|
54
|
+
isRunManifestType,
|
|
55
|
+
isRunResult,
|
|
56
|
+
isStressManifestType,
|
|
57
|
+
isStressResult
|
|
104
58
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.4.1",
|
|
75
|
+
"@artemiskit/redteam": "0.3.2"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
|
@@ -135,12 +135,15 @@ describe('Builders - TestCaseBuilder', () => {
|
|
|
135
135
|
.expectLLMGrade('Response should be helpful and accurate', {
|
|
136
136
|
threshold: 0.8,
|
|
137
137
|
model: 'gpt-4',
|
|
138
|
+
strict: true,
|
|
138
139
|
})
|
|
139
140
|
.build();
|
|
140
141
|
|
|
141
142
|
expect(tc.expected.type).toBe('llm_grader');
|
|
142
|
-
|
|
143
|
-
expect(
|
|
143
|
+
if (tc.expected.type !== 'llm_grader') throw new Error('Expected LLM grader expectation');
|
|
144
|
+
expect(tc.expected.rubric).toBe('Response should be helpful and accurate');
|
|
145
|
+
expect(tc.expected.threshold).toBe(0.8);
|
|
146
|
+
expect(tc.expected.strict).toBe(true);
|
|
144
147
|
});
|
|
145
148
|
|
|
146
149
|
it('should create similarity expectation', () => {
|
|
@@ -380,9 +383,11 @@ describe('Builders - Expectation Helpers', () => {
|
|
|
380
383
|
});
|
|
381
384
|
|
|
382
385
|
it('llmGrade() should create llm_grader expectation', () => {
|
|
383
|
-
const exp = llmGrade('Be accurate', { threshold: 0.8 });
|
|
386
|
+
const exp = llmGrade('Be accurate', { threshold: 0.8, strict: true });
|
|
384
387
|
expect(exp.type).toBe('llm_grader');
|
|
385
|
-
|
|
388
|
+
if (exp.type !== 'llm_grader') throw new Error('Expected LLM grader expectation');
|
|
389
|
+
expect(exp.rubric).toBe('Be accurate');
|
|
390
|
+
expect(exp.strict).toBe(true);
|
|
386
391
|
});
|
|
387
392
|
|
|
388
393
|
it('similarity() should create similarity expectation', () => {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
|
|
6
|
+
import { createStorageAdapter as createRealStorageAdapter } from '../../../core/src/storage/factory';
|
|
6
7
|
|
|
7
8
|
// Track parseScenarioFile calls and responses
|
|
8
9
|
let parseScenarioFileResponse: unknown = {};
|
|
@@ -34,7 +35,9 @@ mock.module('@artemiskit/core', () => ({
|
|
|
34
35
|
}),
|
|
35
36
|
runScenario: async () => ({}),
|
|
36
37
|
getGitInfo: async () => ({ commit: 'abc123', branch: 'main', dirty: false }),
|
|
37
|
-
|
|
38
|
+
// Bun's module mocks are shared by test files in a monorepo run. Preserve the
|
|
39
|
+
// real storage factory so unrelated CLI integration tests can create storage.
|
|
40
|
+
createStorageAdapter: createRealStorageAdapter,
|
|
38
41
|
}));
|
|
39
42
|
|
|
40
43
|
mock.module('@artemiskit/redteam', () => ({
|
package/src/artemiskit.ts
CHANGED
|
@@ -270,6 +270,7 @@ export class ArtemisKit {
|
|
|
270
270
|
timeout: options.timeout ?? this.config.timeout,
|
|
271
271
|
retries: options.retries ?? this.config.retries,
|
|
272
272
|
redaction: options.redaction ?? this.config.redaction,
|
|
273
|
+
toolExecutor: options.toolExecutor,
|
|
273
274
|
onCaseComplete: (caseResult, index, total) => {
|
|
274
275
|
this.emit('caseComplete', { result: caseResult, index, total });
|
|
275
276
|
this.emit('progress', {
|
package/src/builders/index.ts
CHANGED
|
@@ -138,7 +138,7 @@ export class TestCaseBuilder {
|
|
|
138
138
|
*/
|
|
139
139
|
expectLLMGrade(
|
|
140
140
|
rubric: string,
|
|
141
|
-
options?: { threshold?: number; model?: string; provider?: Provider }
|
|
141
|
+
options?: { threshold?: number; model?: string; provider?: Provider; strict?: boolean }
|
|
142
142
|
): this {
|
|
143
143
|
this._expected = {
|
|
144
144
|
type: 'llm_grader',
|
|
@@ -146,6 +146,7 @@ export class TestCaseBuilder {
|
|
|
146
146
|
threshold: options?.threshold ?? 0.7,
|
|
147
147
|
model: options?.model,
|
|
148
148
|
provider: options?.provider,
|
|
149
|
+
strict: options?.strict ?? false,
|
|
149
150
|
};
|
|
150
151
|
return this;
|
|
151
152
|
}
|
|
@@ -639,7 +640,7 @@ export function jsonSchema(schema: Record<string, unknown>): Expected {
|
|
|
639
640
|
*/
|
|
640
641
|
export function llmGrade(
|
|
641
642
|
rubric: string,
|
|
642
|
-
options?: { threshold?: number; model?: string; provider?: Provider }
|
|
643
|
+
options?: { threshold?: number; model?: string; provider?: Provider; strict?: boolean }
|
|
643
644
|
): Expected {
|
|
644
645
|
return {
|
|
645
646
|
type: 'llm_grader',
|
|
@@ -647,6 +648,7 @@ export function llmGrade(
|
|
|
647
648
|
threshold: options?.threshold ?? 0.7,
|
|
648
649
|
model: options?.model,
|
|
649
650
|
provider: options?.provider,
|
|
651
|
+
strict: options?.strict ?? false,
|
|
650
652
|
};
|
|
651
653
|
}
|
|
652
654
|
|
package/src/types.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
StorageConfig,
|
|
13
13
|
StressManifest,
|
|
14
14
|
StressRequestResult,
|
|
15
|
+
ToolExecutor,
|
|
15
16
|
} from '@artemiskit/core';
|
|
16
17
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
17
18
|
import type { Scenario } from '@artemiskit/core';
|
|
@@ -147,6 +148,8 @@ export interface RunOptions {
|
|
|
147
148
|
retries?: number;
|
|
148
149
|
/** Redaction configuration */
|
|
149
150
|
redaction?: RedactionConfig;
|
|
151
|
+
/** Explicit executor for real tools; CLI scenarios remain fixture-only. */
|
|
152
|
+
toolExecutor?: ToolExecutor;
|
|
150
153
|
}
|
|
151
154
|
|
|
152
155
|
/**
|