@artemiskit/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +213 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +17 -2
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Utility types and helper functions
|
|
4
|
+
*/
|
|
5
|
+
import type { AdapterConfig, AnyManifest, CaseResult, Expected, RedTeamManifest, RunManifest, Scenario, StressManifest } from '@artemiskit/core';
|
|
6
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
7
|
+
/**
|
|
8
|
+
* All supported provider names as a union type
|
|
9
|
+
*/
|
|
10
|
+
export type ProviderName = AdapterConfig['provider'];
|
|
11
|
+
/**
|
|
12
|
+
* All supported expectation types
|
|
13
|
+
*/
|
|
14
|
+
export type ExpectationType = Expected['type'];
|
|
15
|
+
/**
|
|
16
|
+
* Union of all SDK result types
|
|
17
|
+
*/
|
|
18
|
+
export type AnyResult = RunResult | RedTeamResult | StressResult;
|
|
19
|
+
/**
|
|
20
|
+
* Union of all manifest types (re-export from core for convenience)
|
|
21
|
+
*/
|
|
22
|
+
export type { AnyManifest };
|
|
23
|
+
/**
|
|
24
|
+
* Extract case results from a RunResult
|
|
25
|
+
*/
|
|
26
|
+
export type ExtractRunCases = RunResult['cases'];
|
|
27
|
+
/**
|
|
28
|
+
* Extract red team cases from a RedTeamResult
|
|
29
|
+
*/
|
|
30
|
+
export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
|
|
31
|
+
/**
|
|
32
|
+
* Extract stress results from a StressResult
|
|
33
|
+
*/
|
|
34
|
+
export type ExtractStressResults = StressResult['manifest']['sample_results'];
|
|
35
|
+
/**
|
|
36
|
+
* Extract the manifest type from any result type
|
|
37
|
+
*/
|
|
38
|
+
export type ExtractManifest<T extends AnyResult> = T extends RunResult ? RunManifest : T extends RedTeamResult ? RedTeamManifest : T extends StressResult ? StressManifest : never;
|
|
39
|
+
/**
|
|
40
|
+
* Deep partial utility type - makes all nested properties optional
|
|
41
|
+
*/
|
|
42
|
+
export type DeepPartial<T> = T extends object ? {
|
|
43
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
44
|
+
} : T;
|
|
45
|
+
/**
|
|
46
|
+
* Partial scenario for incremental construction
|
|
47
|
+
*/
|
|
48
|
+
export type PartialScenario = DeepPartial<Scenario>;
|
|
49
|
+
/**
|
|
50
|
+
* Make specific fields required
|
|
51
|
+
*/
|
|
52
|
+
export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
|
|
53
|
+
/**
|
|
54
|
+
* Make specific fields optional
|
|
55
|
+
*/
|
|
56
|
+
export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
|
|
57
|
+
/**
|
|
58
|
+
* Strict version of AdapterConfig with all common fields required
|
|
59
|
+
*/
|
|
60
|
+
export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
|
|
61
|
+
/**
|
|
62
|
+
* Type guard for RunResult
|
|
63
|
+
*/
|
|
64
|
+
export declare function isRunResult(result: AnyResult): result is RunResult;
|
|
65
|
+
/**
|
|
66
|
+
* Type guard for RedTeamResult
|
|
67
|
+
*/
|
|
68
|
+
export declare function isRedTeamResult(result: AnyResult): result is RedTeamResult;
|
|
69
|
+
/**
|
|
70
|
+
* Type guard for StressResult
|
|
71
|
+
*/
|
|
72
|
+
export declare function isStressResult(result: AnyResult): result is StressResult;
|
|
73
|
+
/**
|
|
74
|
+
* Type guard for RunManifest
|
|
75
|
+
*
|
|
76
|
+
* Uses structural field checks rather than absence-of-field logic for robustness.
|
|
77
|
+
* Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
|
|
78
|
+
*/
|
|
79
|
+
export declare function isRunManifestType(manifest: AnyManifest): manifest is RunManifest;
|
|
80
|
+
/**
|
|
81
|
+
* Type guard for RedTeamManifest
|
|
82
|
+
*/
|
|
83
|
+
export declare function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest;
|
|
84
|
+
/**
|
|
85
|
+
* Type guard for StressManifest
|
|
86
|
+
*/
|
|
87
|
+
export declare function isStressManifestType(manifest: AnyManifest): manifest is StressManifest;
|
|
88
|
+
/**
|
|
89
|
+
* Assert that a value is defined (non-null, non-undefined)
|
|
90
|
+
*/
|
|
91
|
+
export declare function assertDefined<T>(value: T | null | undefined, message?: string): asserts value is T;
|
|
92
|
+
/**
|
|
93
|
+
* Assert that a condition is true
|
|
94
|
+
*/
|
|
95
|
+
export declare function assert(condition: boolean, message?: string): asserts condition;
|
|
96
|
+
/**
|
|
97
|
+
* Get failed cases from a run result
|
|
98
|
+
*/
|
|
99
|
+
export declare function getFailedCases(result: RunResult): CaseResult[];
|
|
100
|
+
/**
|
|
101
|
+
* Get passed cases from a run result
|
|
102
|
+
*/
|
|
103
|
+
export declare function getPassedCases(result: RunResult): CaseResult[];
|
|
104
|
+
/**
|
|
105
|
+
* Get cases by tag from a run result
|
|
106
|
+
*/
|
|
107
|
+
export declare function getCasesByTag(result: RunResult, tag: string): CaseResult[];
|
|
108
|
+
/**
|
|
109
|
+
* Calculate success rate from a run result
|
|
110
|
+
*/
|
|
111
|
+
export declare function calculateSuccessRate(result: RunResult): number;
|
|
112
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,UAAU,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,QAAQ,EACR,cAAc,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAMvE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAErD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAM/C;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,aAAa,GAAG,YAAY,CAAC;AAEjE;;GAEG;AACH,YAAY,EAAE,WAAW,EAAE,CAAC;AAM5B;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;AAE9E;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,SAAS,IAAI,CAAC,SAAS,SAAS,GAClE,WAAW,GACX,CAAC,SAAS,aAAa,GACrB,eAAe,GACf,CAAC,SAAS,YAAY,GACpB,cAAc,GACd,KAAK,CAAC;AAMd;;GAEG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,GACzC;KACG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACnC,GACD,CAAC,CAAC;AAEN;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAEpD;;GAEG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAE3E;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAMpF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,aAAa,EAAE,UAAU,GAAG,QAAQ,CAAC,CAAC;AAMtF;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,SAAS,CAElE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,aAAa,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,YAAY,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAShF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAExF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAEtF;AAMD;;GAEG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAC7B,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,SAAS,EAC3B,OAAO,SAA+B,GACrC,OAAO,CAAC,KAAK,IAAI,CAAC,CAIpB;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,SAAqB,GAAG,OAAO,CAAC,SAAS,CAI1F;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,EAAE,CAE1E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAG9D"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
function __accessProp(key) {
|
|
8
|
+
return this[key];
|
|
9
|
+
}
|
|
10
|
+
var __toESMCache_node;
|
|
11
|
+
var __toESMCache_esm;
|
|
12
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
+
var canCache = mod != null && typeof mod === "object";
|
|
14
|
+
if (canCache) {
|
|
15
|
+
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
+
var cached = cache.get(mod);
|
|
17
|
+
if (cached)
|
|
18
|
+
return cached;
|
|
19
|
+
}
|
|
20
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
+
for (let key of __getOwnPropNames(mod))
|
|
23
|
+
if (!__hasOwnProp.call(to, key))
|
|
24
|
+
__defProp(to, key, {
|
|
25
|
+
get: __accessProp.bind(mod, key),
|
|
26
|
+
enumerable: true
|
|
27
|
+
});
|
|
28
|
+
if (canCache)
|
|
29
|
+
cache.set(mod, to);
|
|
30
|
+
return to;
|
|
31
|
+
};
|
|
32
|
+
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
+
var __returnValue = (v) => v;
|
|
34
|
+
function __exportSetter(name, newValue) {
|
|
35
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
+
}
|
|
37
|
+
var __export = (target, all) => {
|
|
38
|
+
for (var name in all)
|
|
39
|
+
__defProp(target, name, {
|
|
40
|
+
get: all[name],
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
set: __exportSetter.bind(all, name)
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var __require = import.meta.require;
|
|
47
|
+
|
|
48
|
+
// src/utils/index.ts
|
|
49
|
+
function isRunResult(result) {
|
|
50
|
+
return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
|
|
51
|
+
}
|
|
52
|
+
function isRedTeamResult(result) {
|
|
53
|
+
return "defenseRate" in result && "unsafeCount" in result;
|
|
54
|
+
}
|
|
55
|
+
function isStressResult(result) {
|
|
56
|
+
return "rps" in result && "p95LatencyMs" in result;
|
|
57
|
+
}
|
|
58
|
+
function isRunManifestType(manifest) {
|
|
59
|
+
return "metrics" in manifest && manifest.metrics !== null && typeof manifest.metrics === "object" && "success_rate" in manifest.metrics && "cases" in manifest && Array.isArray(manifest.cases);
|
|
60
|
+
}
|
|
61
|
+
function isRedTeamManifestType(manifest) {
|
|
62
|
+
return "type" in manifest && manifest.type === "redteam";
|
|
63
|
+
}
|
|
64
|
+
function isStressManifestType(manifest) {
|
|
65
|
+
return "type" in manifest && manifest.type === "stress";
|
|
66
|
+
}
|
|
67
|
+
function assertDefined(value, message = "Value is null or undefined") {
|
|
68
|
+
if (value === null || value === undefined) {
|
|
69
|
+
throw new Error(message);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function assert(condition, message = "Assertion failed") {
|
|
73
|
+
if (!condition) {
|
|
74
|
+
throw new Error(message);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function getFailedCases(result) {
|
|
78
|
+
return result.cases.filter((c) => !c.ok);
|
|
79
|
+
}
|
|
80
|
+
function getPassedCases(result) {
|
|
81
|
+
return result.cases.filter((c) => c.ok);
|
|
82
|
+
}
|
|
83
|
+
function getCasesByTag(result, tag) {
|
|
84
|
+
return result.cases.filter((c) => c.tags.includes(tag));
|
|
85
|
+
}
|
|
86
|
+
function calculateSuccessRate(result) {
|
|
87
|
+
if (result.cases.length === 0)
|
|
88
|
+
return 0;
|
|
89
|
+
return result.cases.filter((c) => c.ok).length / result.cases.length;
|
|
90
|
+
}
|
|
91
|
+
export {
|
|
92
|
+
isStressResult,
|
|
93
|
+
isStressManifestType,
|
|
94
|
+
isRunResult,
|
|
95
|
+
isRunManifestType,
|
|
96
|
+
isRedTeamResult,
|
|
97
|
+
isRedTeamManifestType,
|
|
98
|
+
getPassedCases,
|
|
99
|
+
getFailedCases,
|
|
100
|
+
getCasesByTag,
|
|
101
|
+
calculateSuccessRate,
|
|
102
|
+
assertDefined,
|
|
103
|
+
assert
|
|
104
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -46,10 +46,25 @@
|
|
|
46
46
|
"./jest": {
|
|
47
47
|
"import": "./dist/matchers/jest.js",
|
|
48
48
|
"types": "./dist/matchers/jest.d.ts"
|
|
49
|
+
},
|
|
50
|
+
"./builders": {
|
|
51
|
+
"import": "./dist/builders/index.js",
|
|
52
|
+
"types": "./dist/builders/index.d.ts"
|
|
53
|
+
},
|
|
54
|
+
"./contracts": {
|
|
55
|
+
"import": "./dist/contracts/index.js",
|
|
56
|
+
"types": "./dist/contracts/index.d.ts"
|
|
57
|
+
},
|
|
58
|
+
"./utils": {
|
|
59
|
+
"import": "./dist/utils/index.js",
|
|
60
|
+
"types": "./dist/utils/index.d.ts"
|
|
61
|
+
},
|
|
62
|
+
"./types": {
|
|
63
|
+
"types": "./dist/types-only.d.ts"
|
|
49
64
|
}
|
|
50
65
|
},
|
|
51
66
|
"scripts": {
|
|
52
|
-
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts --outdir ./dist --target bun",
|
|
67
|
+
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target bun",
|
|
53
68
|
"typecheck": "tsc --noEmit",
|
|
54
69
|
"clean": "rm -rf dist",
|
|
55
70
|
"test": "bun test",
|