@artemiskit/sdk 0.4.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/artemiskit.d.ts +1 -1
- package/dist/builders/index.js +66 -20
- package/dist/contracts/index.js +50 -4
- package/dist/guardian/action-validator.d.ts +1 -1
- package/dist/guardian/circuit-breaker.d.ts +1 -1
- package/dist/guardian/guardian.d.ts +5 -5
- package/dist/guardian/guardrails.d.ts +1 -1
- package/dist/guardian/index.d.ts +9 -9
- package/dist/guardian/intent-classifier.d.ts +1 -1
- package/dist/guardian/interceptor.d.ts +1 -1
- package/dist/guardian/policy.d.ts +1 -1
- package/dist/guardian/semantic-validator.d.ts +1 -1
- package/dist/index.d.ts +12 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3152 -46920
- package/dist/matchers/core.d.ts +1 -1
- package/dist/matchers/index.d.ts +1 -1
- package/dist/matchers/index.js +60 -14
- package/dist/matchers/jest.d.ts +14 -14
- package/dist/matchers/jest.js +47 -1
- package/dist/matchers/vitest.d.ts +14 -14
- package/dist/matchers/vitest.js +47 -1
- package/dist/types-only.d.ts +6 -6
- package/dist/types-only.js +0 -1
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.js +58 -12
- package/package.json +18 -18
- package/src/index.ts +29 -0
package/dist/matchers/core.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Core matcher implementations for Jest/Vitest integration
|
|
4
4
|
*/
|
|
5
|
-
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types.js';
|
|
6
6
|
/**
|
|
7
7
|
* Matcher result interface compatible with Jest/Vitest
|
|
8
8
|
*/
|
package/dist/matchers/index.d.ts
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Test matchers - core implementations without framework bindings
|
|
4
4
|
*/
|
|
5
|
-
export { artemiskitMatchers, type ArtemisKitMatchers, type MatcherResult, toPassAllCases, toHaveSuccessRate, toPassCasesWithTag, toHaveMedianLatencyBelow, toHaveP95LatencyBelow, toHaveDefenseRate, toHaveNoCriticalVulnerabilities, toHaveNoHighSeverityVulnerabilities, toPassRedTeam, toHaveStressSuccessRate, toAchieveRPS, toHaveStressP95LatencyBelow, toPassStressTest, } from './core';
|
|
5
|
+
export { artemiskitMatchers, type ArtemisKitMatchers, type MatcherResult, toPassAllCases, toHaveSuccessRate, toPassCasesWithTag, toHaveMedianLatencyBelow, toHaveP95LatencyBelow, toHaveDefenseRate, toHaveNoCriticalVulnerabilities, toHaveNoHighSeverityVulnerabilities, toPassRedTeam, toHaveStressSuccessRate, toAchieveRPS, toHaveStressP95LatencyBelow, toPassStressTest, } from './core.js';
|
|
6
6
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/matchers/index.js
CHANGED
|
@@ -1,4 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
+
|
|
2
48
|
// src/matchers/core.ts
|
|
3
49
|
function formatManifestSummary(manifest) {
|
|
4
50
|
const { metrics } = manifest;
|
|
@@ -161,18 +207,18 @@ var artemiskitMatchers = {
|
|
|
161
207
|
toPassStressTest
|
|
162
208
|
};
|
|
163
209
|
export {
|
|
164
|
-
|
|
165
|
-
toAchieveRPS,
|
|
166
|
-
toHaveDefenseRate,
|
|
167
|
-
toHaveMedianLatencyBelow,
|
|
168
|
-
toHaveNoCriticalVulnerabilities,
|
|
169
|
-
toHaveNoHighSeverityVulnerabilities,
|
|
170
|
-
toHaveP95LatencyBelow,
|
|
171
|
-
toHaveStressP95LatencyBelow,
|
|
172
|
-
toHaveStressSuccessRate,
|
|
173
|
-
toHaveSuccessRate,
|
|
174
|
-
toPassAllCases,
|
|
175
|
-
toPassCasesWithTag,
|
|
210
|
+
toPassStressTest,
|
|
176
211
|
toPassRedTeam,
|
|
177
|
-
|
|
212
|
+
toPassCasesWithTag,
|
|
213
|
+
toPassAllCases,
|
|
214
|
+
toHaveSuccessRate,
|
|
215
|
+
toHaveStressSuccessRate,
|
|
216
|
+
toHaveStressP95LatencyBelow,
|
|
217
|
+
toHaveP95LatencyBelow,
|
|
218
|
+
toHaveNoHighSeverityVulnerabilities,
|
|
219
|
+
toHaveNoCriticalVulnerabilities,
|
|
220
|
+
toHaveMedianLatencyBelow,
|
|
221
|
+
toHaveDefenseRate,
|
|
222
|
+
toAchieveRPS,
|
|
223
|
+
artemiskitMatchers
|
|
178
224
|
};
|
package/dist/matchers/jest.d.ts
CHANGED
|
@@ -2,24 +2,24 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Jest integration - custom matchers for ArtemisKit
|
|
4
4
|
*/
|
|
5
|
-
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types.js';
|
|
6
6
|
/**
|
|
7
7
|
* Jest matchers implementation
|
|
8
8
|
*/
|
|
9
9
|
export declare const jestMatchers: {
|
|
10
|
-
toPassAllCases(received: RunResult): import("./core").MatcherResult;
|
|
11
|
-
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core").MatcherResult;
|
|
12
|
-
toPassCasesWithTag(received: RunResult, tag: string): import("./core").MatcherResult;
|
|
13
|
-
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
14
|
-
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
15
|
-
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core").MatcherResult;
|
|
16
|
-
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
17
|
-
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
18
|
-
toPassRedTeam(received: RedTeamResult): import("./core").MatcherResult;
|
|
19
|
-
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core").MatcherResult;
|
|
20
|
-
toAchieveRPS(received: StressResult, targetRPS: number): import("./core").MatcherResult;
|
|
21
|
-
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
22
|
-
toPassStressTest(received: StressResult): import("./core").MatcherResult;
|
|
10
|
+
toPassAllCases(received: RunResult): import("./core.js").MatcherResult;
|
|
11
|
+
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
12
|
+
toPassCasesWithTag(received: RunResult, tag: string): import("./core.js").MatcherResult;
|
|
13
|
+
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
14
|
+
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
15
|
+
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
16
|
+
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
17
|
+
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
18
|
+
toPassRedTeam(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
19
|
+
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
20
|
+
toAchieveRPS(received: StressResult, targetRPS: number): import("./core.js").MatcherResult;
|
|
21
|
+
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
22
|
+
toPassStressTest(received: StressResult): import("./core.js").MatcherResult;
|
|
23
23
|
};
|
|
24
24
|
/**
|
|
25
25
|
* TypeScript declarations for Jest matchers
|
package/dist/matchers/jest.js
CHANGED
|
@@ -1,4 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
+
|
|
2
48
|
// src/matchers/core.ts
|
|
3
49
|
function formatManifestSummary(manifest) {
|
|
4
50
|
const { metrics } = manifest;
|
|
@@ -2,23 +2,23 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Vitest integration - custom matchers for ArtemisKit
|
|
4
4
|
*/
|
|
5
|
-
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types.js';
|
|
6
6
|
/**
|
|
7
7
|
* Vitest matchers implementation
|
|
8
8
|
*/
|
|
9
9
|
export declare const vitestMatchers: {
|
|
10
|
-
toPassAllCases(received: RunResult): import("./core").MatcherResult;
|
|
11
|
-
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core").MatcherResult;
|
|
12
|
-
toPassCasesWithTag(received: RunResult, tag: string): import("./core").MatcherResult;
|
|
13
|
-
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
14
|
-
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
15
|
-
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core").MatcherResult;
|
|
16
|
-
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
17
|
-
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
18
|
-
toPassRedTeam(received: RedTeamResult): import("./core").MatcherResult;
|
|
19
|
-
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core").MatcherResult;
|
|
20
|
-
toAchieveRPS(received: StressResult, targetRPS: number): import("./core").MatcherResult;
|
|
21
|
-
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
22
|
-
toPassStressTest(received: StressResult): import("./core").MatcherResult;
|
|
10
|
+
toPassAllCases(received: RunResult): import("./core.js").MatcherResult;
|
|
11
|
+
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
12
|
+
toPassCasesWithTag(received: RunResult, tag: string): import("./core.js").MatcherResult;
|
|
13
|
+
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
14
|
+
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
15
|
+
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
16
|
+
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
17
|
+
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
18
|
+
toPassRedTeam(received: RedTeamResult): import("./core.js").MatcherResult;
|
|
19
|
+
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core.js").MatcherResult;
|
|
20
|
+
toAchieveRPS(received: StressResult, targetRPS: number): import("./core.js").MatcherResult;
|
|
21
|
+
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core.js").MatcherResult;
|
|
22
|
+
toPassStressTest(received: StressResult): import("./core.js").MatcherResult;
|
|
23
23
|
};
|
|
24
24
|
//# sourceMappingURL=vitest.d.ts.map
|
package/dist/matchers/vitest.js
CHANGED
|
@@ -1,4 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
+
|
|
2
48
|
// src/matchers/core.ts
|
|
3
49
|
function formatManifestSummary(manifest) {
|
|
4
50
|
const { metrics } = manifest;
|
package/dist/types-only.d.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* } from '@artemiskit/sdk/types'
|
|
16
16
|
* ```
|
|
17
17
|
*/
|
|
18
|
-
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types';
|
|
18
|
+
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types.js';
|
|
19
19
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
20
20
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
21
21
|
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
|
@@ -23,9 +23,9 @@ export type { Evaluator, EvaluatorContext, EvaluatorResult } from '@artemiskit/c
|
|
|
23
23
|
export type { StorageAdapter, StorageConfig, RunListItem, ListOptions, ComparisonResult, BaselineMetadata, BaselineStorageAdapter, CaseResultRecord, CaseResultStatus, CaseResultQueryOptions, MetricsSnapshot, MetricsTrendOptions, TrendDataPoint, AnalyticsStorageAdapter, } from '@artemiskit/core';
|
|
24
24
|
export type { RedactionConfig } from '@artemiskit/core';
|
|
25
25
|
export type { Severity, SeverityInfo, CvssScore } from '@artemiskit/redteam';
|
|
26
|
-
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, } from './guardian';
|
|
27
|
-
export type { ArtemisKitMatchers, MatcherResult } from './matchers';
|
|
28
|
-
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils';
|
|
29
|
-
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts';
|
|
30
|
-
export type { ScenarioBuilder, TestCaseBuilder } from './builders';
|
|
26
|
+
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, } from './guardian/index.js';
|
|
27
|
+
export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
28
|
+
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
|
+
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
|
+
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
31
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.js
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
// @bun
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Utility types and helper functions
|
|
4
4
|
*/
|
|
5
5
|
import type { AdapterConfig, AnyManifest, CaseResult, Expected, RedTeamManifest, RunManifest, Scenario, StressManifest } from '@artemiskit/core';
|
|
6
|
-
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
6
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types.js';
|
|
7
7
|
/**
|
|
8
8
|
* All supported provider names as a union type
|
|
9
9
|
*/
|
package/dist/utils/index.js
CHANGED
|
@@ -1,4 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
+
|
|
2
48
|
// src/utils/index.ts
|
|
3
49
|
function isRunResult(result) {
|
|
4
50
|
return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
|
|
@@ -43,16 +89,16 @@ function calculateSuccessRate(result) {
|
|
|
43
89
|
return result.cases.filter((c) => c.ok).length / result.cases.length;
|
|
44
90
|
}
|
|
45
91
|
export {
|
|
46
|
-
|
|
47
|
-
assertDefined,
|
|
48
|
-
calculateSuccessRate,
|
|
49
|
-
getCasesByTag,
|
|
50
|
-
getFailedCases,
|
|
51
|
-
getPassedCases,
|
|
52
|
-
isRedTeamManifestType,
|
|
53
|
-
isRedTeamResult,
|
|
54
|
-
isRunManifestType,
|
|
55
|
-
isRunResult,
|
|
92
|
+
isStressResult,
|
|
56
93
|
isStressManifestType,
|
|
57
|
-
|
|
94
|
+
isRunResult,
|
|
95
|
+
isRunManifestType,
|
|
96
|
+
isRedTeamResult,
|
|
97
|
+
isRedTeamManifestType,
|
|
98
|
+
getPassedCases,
|
|
99
|
+
getFailedCases,
|
|
100
|
+
getCasesByTag,
|
|
101
|
+
calculateSuccessRate,
|
|
102
|
+
assertDefined,
|
|
103
|
+
assert
|
|
58
104
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -32,47 +32,47 @@
|
|
|
32
32
|
"types": "./dist/index.d.ts",
|
|
33
33
|
"exports": {
|
|
34
34
|
".": {
|
|
35
|
-
"
|
|
36
|
-
"
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"import": "./dist/index.js"
|
|
37
37
|
},
|
|
38
38
|
"./matchers": {
|
|
39
|
-
"
|
|
40
|
-
"
|
|
39
|
+
"types": "./dist/matchers/index.d.ts",
|
|
40
|
+
"import": "./dist/matchers/index.js"
|
|
41
41
|
},
|
|
42
42
|
"./vitest": {
|
|
43
|
-
"
|
|
44
|
-
"
|
|
43
|
+
"types": "./dist/matchers/vitest.d.ts",
|
|
44
|
+
"import": "./dist/matchers/vitest.js"
|
|
45
45
|
},
|
|
46
46
|
"./jest": {
|
|
47
|
-
"
|
|
48
|
-
"
|
|
47
|
+
"types": "./dist/matchers/jest.d.ts",
|
|
48
|
+
"import": "./dist/matchers/jest.js"
|
|
49
49
|
},
|
|
50
50
|
"./builders": {
|
|
51
|
-
"
|
|
52
|
-
"
|
|
51
|
+
"types": "./dist/builders/index.d.ts",
|
|
52
|
+
"import": "./dist/builders/index.js"
|
|
53
53
|
},
|
|
54
54
|
"./contracts": {
|
|
55
|
-
"
|
|
56
|
-
"
|
|
55
|
+
"types": "./dist/contracts/index.d.ts",
|
|
56
|
+
"import": "./dist/contracts/index.js"
|
|
57
57
|
},
|
|
58
58
|
"./utils": {
|
|
59
|
-
"
|
|
60
|
-
"
|
|
59
|
+
"types": "./dist/utils/index.d.ts",
|
|
60
|
+
"import": "./dist/utils/index.js"
|
|
61
61
|
},
|
|
62
62
|
"./types": {
|
|
63
63
|
"types": "./dist/types-only.d.ts"
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
66
|
"scripts": {
|
|
67
|
-
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target
|
|
67
|
+
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target node --external '@artemiskit/*' && node ../../scripts/finalize-declarations.mjs ./dist",
|
|
68
68
|
"typecheck": "tsc --noEmit",
|
|
69
69
|
"clean": "rm -rf dist",
|
|
70
70
|
"test": "bun test",
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.0",
|
|
75
|
+
"@artemiskit/redteam": "0.3.8"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
package/src/index.ts
CHANGED
|
@@ -6,6 +6,35 @@
|
|
|
6
6
|
// Main class
|
|
7
7
|
export { ArtemisKit } from './artemiskit';
|
|
8
8
|
|
|
9
|
+
// Workflow authoring and single-step primitives (full workflow execution follows separately).
|
|
10
|
+
export {
|
|
11
|
+
AgentWorkflowSchema,
|
|
12
|
+
WorkflowPolicySchema,
|
|
13
|
+
WORKFLOW_TOOL_IDS,
|
|
14
|
+
validateAgentWorkflow,
|
|
15
|
+
parseAgentWorkflow,
|
|
16
|
+
loadAgentWorkflow,
|
|
17
|
+
listWorkflowTools,
|
|
18
|
+
getWorkflowTool,
|
|
19
|
+
executeSimulatedTool,
|
|
20
|
+
createModelClientTarget,
|
|
21
|
+
} from '@artemiskit/core';
|
|
22
|
+
export type {
|
|
23
|
+
AgentWorkflow,
|
|
24
|
+
WorkflowPolicy,
|
|
25
|
+
WorkflowJson,
|
|
26
|
+
WorkflowToolId,
|
|
27
|
+
WorkflowResource,
|
|
28
|
+
WorkflowToolDescriptor,
|
|
29
|
+
SimulatedToolEvidence,
|
|
30
|
+
SimulatedToolResult,
|
|
31
|
+
AgentTarget,
|
|
32
|
+
AgentTurnRequest,
|
|
33
|
+
AgentTurnResult,
|
|
34
|
+
AgentTargetCapabilities,
|
|
35
|
+
AgentTargetFailure,
|
|
36
|
+
} from '@artemiskit/core';
|
|
37
|
+
|
|
9
38
|
// Types
|
|
10
39
|
export type {
|
|
11
40
|
// Configuration
|