@artemiskit/sdk 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +17 -0
  3. package/dist/__tests__/experiment-fixtures.d.ts +44 -0
  4. package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
  5. package/dist/artemiskit.d.ts +6 -1
  6. package/dist/artemiskit.d.ts.map +1 -1
  7. package/dist/builders/index.js +19 -66
  8. package/dist/contracts/index.js +3 -50
  9. package/dist/experiment-files.d.ts +54 -0
  10. package/dist/experiment-files.d.ts.map +1 -0
  11. package/dist/experiment-fixture.d.ts +20 -0
  12. package/dist/experiment-fixture.d.ts.map +1 -0
  13. package/dist/experiment-summary.d.ts +44 -0
  14. package/dist/experiment-summary.d.ts.map +1 -0
  15. package/dist/experiment-validation.d.ts +48 -0
  16. package/dist/experiment-validation.d.ts.map +1 -0
  17. package/dist/experiment.d.ts +162 -0
  18. package/dist/experiment.d.ts.map +1 -0
  19. package/dist/guardian/circuit-breaker.d.ts +6 -1
  20. package/dist/guardian/circuit-breaker.d.ts.map +1 -1
  21. package/dist/index.d.ts +7 -0
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +2516 -1801
  24. package/dist/matchers/index.js +13 -60
  25. package/dist/matchers/jest.js +0 -47
  26. package/dist/matchers/vitest.js +0 -47
  27. package/dist/types-only.d.ts +1 -0
  28. package/dist/types-only.d.ts.map +1 -1
  29. package/dist/utils/index.js +11 -58
  30. package/package.json +6 -4
  31. package/src/__tests__/artemiskit.test.ts +1 -2
  32. package/src/__tests__/builders.test.ts +0 -2
  33. package/src/__tests__/circuit-breaker.test.ts +1 -1
  34. package/src/__tests__/compare.test.ts +1 -1
  35. package/src/__tests__/contracts.test.ts +1 -1
  36. package/src/__tests__/experiment-case-level.test.ts +212 -0
  37. package/src/__tests__/experiment-fixtures.ts +194 -0
  38. package/src/__tests__/experiment.test.ts +1262 -0
  39. package/src/__tests__/guardian-modes.test.ts +2 -2
  40. package/src/__tests__/guardrails.test.ts +1 -1
  41. package/src/__tests__/matchers.test.ts +1 -1
  42. package/src/__tests__/semantic-validator.test.ts +15 -10
  43. package/src/__tests__/validate.test.ts +0 -4
  44. package/src/__tests__/workflow-recovery.test.ts +7 -3
  45. package/src/__tests__/workflow-report.test.ts +112 -0
  46. package/src/__tests__/workflow.test.ts +11 -5
  47. package/src/artemiskit.ts +16 -1
  48. package/src/experiment-files.ts +190 -0
  49. package/src/experiment-fixture.ts +70 -0
  50. package/src/experiment-summary.ts +172 -0
  51. package/src/experiment-validation.ts +426 -0
  52. package/src/experiment.ts +630 -0
  53. package/src/guardian/circuit-breaker.ts +6 -1
  54. package/src/index.ts +71 -0
  55. package/src/types-only.ts +11 -0
  56. package/tsconfig.typecheck.json +12 -0
@@ -1,50 +1,3 @@
1
- import { createRequire } from "node:module";
2
- var __create = Object.create;
3
- var __getProtoOf = Object.getPrototypeOf;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __hasOwnProp = Object.prototype.hasOwnProperty;
7
- function __accessProp(key) {
8
- return this[key];
9
- }
10
- var __toESMCache_node;
11
- var __toESMCache_esm;
12
- var __toESM = (mod, isNodeMode, target) => {
13
- var canCache = mod != null && typeof mod === "object";
14
- if (canCache) {
15
- var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
16
- var cached = cache.get(mod);
17
- if (cached)
18
- return cached;
19
- }
20
- target = mod != null ? __create(__getProtoOf(mod)) : {};
21
- const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
22
- for (let key of __getOwnPropNames(mod))
23
- if (!__hasOwnProp.call(to, key))
24
- __defProp(to, key, {
25
- get: __accessProp.bind(mod, key),
26
- enumerable: true
27
- });
28
- if (canCache)
29
- cache.set(mod, to);
30
- return to;
31
- };
32
- var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
33
- var __returnValue = (v) => v;
34
- function __exportSetter(name, newValue) {
35
- this[name] = __returnValue.bind(null, newValue);
36
- }
37
- var __export = (target, all) => {
38
- for (var name in all)
39
- __defProp(target, name, {
40
- get: all[name],
41
- enumerable: true,
42
- configurable: true,
43
- set: __exportSetter.bind(all, name)
44
- });
45
- };
46
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
47
-
48
1
  // src/matchers/core.ts
49
2
  function formatManifestSummary(manifest) {
50
3
  const { metrics } = manifest;
@@ -207,18 +160,18 @@ var artemiskitMatchers = {
207
160
  toPassStressTest
208
161
  };
209
162
  export {
210
- toPassStressTest,
211
- toPassRedTeam,
212
- toPassCasesWithTag,
213
- toPassAllCases,
214
- toHaveSuccessRate,
215
- toHaveStressSuccessRate,
216
- toHaveStressP95LatencyBelow,
217
- toHaveP95LatencyBelow,
218
- toHaveNoHighSeverityVulnerabilities,
219
- toHaveNoCriticalVulnerabilities,
220
- toHaveMedianLatencyBelow,
221
- toHaveDefenseRate,
163
+ artemiskitMatchers,
222
164
  toAchieveRPS,
223
- artemiskitMatchers
165
+ toHaveDefenseRate,
166
+ toHaveMedianLatencyBelow,
167
+ toHaveNoCriticalVulnerabilities,
168
+ toHaveNoHighSeverityVulnerabilities,
169
+ toHaveP95LatencyBelow,
170
+ toHaveStressP95LatencyBelow,
171
+ toHaveStressSuccessRate,
172
+ toHaveSuccessRate,
173
+ toPassAllCases,
174
+ toPassCasesWithTag,
175
+ toPassRedTeam,
176
+ toPassStressTest
224
177
  };
@@ -1,50 +1,3 @@
1
- import { createRequire } from "node:module";
2
- var __create = Object.create;
3
- var __getProtoOf = Object.getPrototypeOf;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __hasOwnProp = Object.prototype.hasOwnProperty;
7
- function __accessProp(key) {
8
- return this[key];
9
- }
10
- var __toESMCache_node;
11
- var __toESMCache_esm;
12
- var __toESM = (mod, isNodeMode, target) => {
13
- var canCache = mod != null && typeof mod === "object";
14
- if (canCache) {
15
- var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
16
- var cached = cache.get(mod);
17
- if (cached)
18
- return cached;
19
- }
20
- target = mod != null ? __create(__getProtoOf(mod)) : {};
21
- const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
22
- for (let key of __getOwnPropNames(mod))
23
- if (!__hasOwnProp.call(to, key))
24
- __defProp(to, key, {
25
- get: __accessProp.bind(mod, key),
26
- enumerable: true
27
- });
28
- if (canCache)
29
- cache.set(mod, to);
30
- return to;
31
- };
32
- var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
33
- var __returnValue = (v) => v;
34
- function __exportSetter(name, newValue) {
35
- this[name] = __returnValue.bind(null, newValue);
36
- }
37
- var __export = (target, all) => {
38
- for (var name in all)
39
- __defProp(target, name, {
40
- get: all[name],
41
- enumerable: true,
42
- configurable: true,
43
- set: __exportSetter.bind(all, name)
44
- });
45
- };
46
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
47
-
48
1
  // src/matchers/core.ts
49
2
  function formatManifestSummary(manifest) {
50
3
  const { metrics } = manifest;
@@ -1,50 +1,3 @@
1
- import { createRequire } from "node:module";
2
- var __create = Object.create;
3
- var __getProtoOf = Object.getPrototypeOf;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __hasOwnProp = Object.prototype.hasOwnProperty;
7
- function __accessProp(key) {
8
- return this[key];
9
- }
10
- var __toESMCache_node;
11
- var __toESMCache_esm;
12
- var __toESM = (mod, isNodeMode, target) => {
13
- var canCache = mod != null && typeof mod === "object";
14
- if (canCache) {
15
- var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
16
- var cached = cache.get(mod);
17
- if (cached)
18
- return cached;
19
- }
20
- target = mod != null ? __create(__getProtoOf(mod)) : {};
21
- const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
22
- for (let key of __getOwnPropNames(mod))
23
- if (!__hasOwnProp.call(to, key))
24
- __defProp(to, key, {
25
- get: __accessProp.bind(mod, key),
26
- enumerable: true
27
- });
28
- if (canCache)
29
- cache.set(mod, to);
30
- return to;
31
- };
32
- var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
33
- var __returnValue = (v) => v;
34
- function __exportSetter(name, newValue) {
35
- this[name] = __returnValue.bind(null, newValue);
36
- }
37
- var __export = (target, all) => {
38
- for (var name in all)
39
- __defProp(target, name, {
40
- get: all[name],
41
- enumerable: true,
42
- configurable: true,
43
- set: __exportSetter.bind(all, name)
44
- });
45
- };
46
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
47
-
48
1
  // src/matchers/core.ts
49
2
  function formatManifestSummary(manifest) {
50
3
  const { metrics } = manifest;
@@ -29,4 +29,5 @@ export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, Extract
29
29
  export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
30
30
  export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
31
31
  export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
32
+ export type { WorkflowReport, WorkflowReportView, WorkflowReportFormat, WorkflowReportOptions, GenerateWorkflowReportOptions, WorkflowReportFinding, WorkflowReportEvidence, WorkflowReportRow, WorkflowReportSection, } from '@artemiskit/reports';
32
33
  //# sourceMappingURL=types-only.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,6BAA6B,EAC7B,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC"}
@@ -1,50 +1,3 @@
1
- import { createRequire } from "node:module";
2
- var __create = Object.create;
3
- var __getProtoOf = Object.getPrototypeOf;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __hasOwnProp = Object.prototype.hasOwnProperty;
7
- function __accessProp(key) {
8
- return this[key];
9
- }
10
- var __toESMCache_node;
11
- var __toESMCache_esm;
12
- var __toESM = (mod, isNodeMode, target) => {
13
- var canCache = mod != null && typeof mod === "object";
14
- if (canCache) {
15
- var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
16
- var cached = cache.get(mod);
17
- if (cached)
18
- return cached;
19
- }
20
- target = mod != null ? __create(__getProtoOf(mod)) : {};
21
- const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
22
- for (let key of __getOwnPropNames(mod))
23
- if (!__hasOwnProp.call(to, key))
24
- __defProp(to, key, {
25
- get: __accessProp.bind(mod, key),
26
- enumerable: true
27
- });
28
- if (canCache)
29
- cache.set(mod, to);
30
- return to;
31
- };
32
- var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
33
- var __returnValue = (v) => v;
34
- function __exportSetter(name, newValue) {
35
- this[name] = __returnValue.bind(null, newValue);
36
- }
37
- var __export = (target, all) => {
38
- for (var name in all)
39
- __defProp(target, name, {
40
- get: all[name],
41
- enumerable: true,
42
- configurable: true,
43
- set: __exportSetter.bind(all, name)
44
- });
45
- };
46
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
47
-
48
1
  // src/utils/index.ts
49
2
  function isRunResult(result) {
50
3
  return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
@@ -89,16 +42,16 @@ function calculateSuccessRate(result) {
89
42
  return result.cases.filter((c) => c.ok).length / result.cases.length;
90
43
  }
91
44
  export {
92
- isStressResult,
93
- isStressManifestType,
94
- isRunResult,
95
- isRunManifestType,
96
- isRedTeamResult,
97
- isRedTeamManifestType,
98
- getPassedCases,
99
- getFailedCases,
100
- getCasesByTag,
101
- calculateSuccessRate,
45
+ assert,
102
46
  assertDefined,
103
- assert
47
+ calculateSuccessRate,
48
+ getCasesByTag,
49
+ getFailedCases,
50
+ getPassedCases,
51
+ isRedTeamManifestType,
52
+ isRedTeamResult,
53
+ isRunManifestType,
54
+ isRunResult,
55
+ isStressManifestType,
56
+ isStressResult
104
57
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.6.2",
3
+ "version": "0.7.0",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -65,14 +65,16 @@
65
65
  },
66
66
  "scripts": {
67
67
  "build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target node --external '@artemiskit/*' && node ../../scripts/finalize-declarations.mjs ./dist",
68
- "typecheck": "tsc --noEmit",
68
+ "typecheck": "tsc -p tsconfig.typecheck.json",
69
69
  "clean": "rm -rf dist",
70
70
  "test": "bun test",
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.6.3",
75
- "@artemiskit/redteam": "0.3.11"
74
+ "@artemiskit/core": "0.7.0",
75
+ "@artemiskit/redteam": "0.3.13",
76
+ "@artemiskit/reports": "0.6.0",
77
+ "yaml": "^2.3.0"
76
78
  },
77
79
  "devDependencies": {
78
80
  "@types/bun": "^1.1.0",
@@ -3,7 +3,7 @@
3
3
  * Tests for ArtemisKit class
4
4
  */
5
5
 
6
- import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
6
+ import { afterEach, beforeEach, describe, expect, it, vi } from 'bun:test';
7
7
 
8
8
  // Mock modules before importing ArtemisKit
9
9
  vi.mock('@artemiskit/core', () => ({
@@ -416,7 +416,6 @@ describe('ArtemisKit with custom client', () => {
416
416
  const kit = new ArtemisKit();
417
417
  const result = await kit.run({
418
418
  scenario: './test.yaml',
419
- // @ts-expect-error - Testing with mock client
420
419
  client: mockClient,
421
420
  });
422
421
 
@@ -4,8 +4,6 @@
4
4
 
5
5
  import { describe, expect, it } from 'bun:test';
6
6
  import {
7
- ScenarioBuilder,
8
- TestCaseBuilder,
9
7
  allOf,
10
8
  anyOf,
11
9
  contains,
@@ -5,7 +5,7 @@
5
5
  * - RateLimiter token bucket
6
6
  */
7
7
 
8
- import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
8
+ import { describe, expect, test } from 'bun:test';
9
9
  import { CircuitBreaker, MetricsCollector, RateLimiter } from '../guardian/circuit-breaker';
10
10
  import type { Violation } from '../guardian/types';
11
11
 
@@ -2,7 +2,7 @@
2
2
  * Tests for ArtemisKit.compare() method (v0.3.2+)
3
3
  */
4
4
 
5
- import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
5
+ import { afterEach, beforeEach, describe, expect, it, vi } from 'bun:test';
6
6
 
7
7
  // Mock the core module
8
8
  const mockStorageAdapter = {
@@ -150,7 +150,7 @@ describe('Contracts - defineEvaluator', () => {
150
150
  },
151
151
  });
152
152
 
153
- const result = await evaluator.evaluate('This is a test response with many words', {} as any);
153
+ const result = await evaluator.evaluate('This is a test response with many words');
154
154
  expect(result.passed).toBe(true);
155
155
  expect(result.details).toBeDefined();
156
156
  expect(result.details?.wordCount).toBe(8); // "This is a test response with many words" = 8 words
@@ -0,0 +1,212 @@
1
+ import { afterEach, describe, expect, mock, setDefaultTimeout, test } from 'bun:test';
2
+ import { appendFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import type { ExperimentRunResult } from '@artemiskit/core';
5
+ import * as realCore from '../../../core/src/index';
6
+
7
+ // Earlier SDK test files stub `@artemiskit/core` without restoring it; these tests exercise the
8
+ // real experiment pipeline, so rebind the real implementations (see experiment.test.ts).
9
+ mock.module('@artemiskit/core', () => ({ ...realCore }));
10
+
11
+ import {
12
+ ExperimentSurfaceError,
13
+ formatExperimentCaseSummary,
14
+ formatExperimentRunSummary,
15
+ reportExperimentResult,
16
+ runExperimentFile,
17
+ summarizeExperimentCases,
18
+ } from '../index';
19
+ import type { ExperimentCaseLevelSummary, ExperimentFileRunOutcome } from '../index';
20
+ import { sha256, writeExperimentFixture } from './experiment-fixtures';
21
+
22
+ setDefaultTimeout(30_000);
23
+
24
+ /** Two cases: the fixture client's reply passes the first and fails the second. */
25
+ const MIXED_SCENARIO = `name: fixture-mixed
26
+ cases:
27
+ - id: echo
28
+ prompt: "Reply with anything."
29
+ expected:
30
+ type: contains
31
+ values: ["Fixture response"]
32
+ - id: capital
33
+ prompt: "What is the capital of Nigeria?"
34
+ expected:
35
+ type: contains
36
+ values: ["Abuja"]
37
+ `;
38
+
39
+ const originalFetch = globalThis.fetch;
40
+ afterEach(() => {
41
+ globalThis.fetch = originalFetch;
42
+ });
43
+
44
+ function forbidNetwork(): { calls: number } {
45
+ const counter = { calls: 0 };
46
+ globalThis.fetch = (() => {
47
+ counter.calls += 1;
48
+ throw new Error('NETWORK-FORBIDDEN');
49
+ }) as unknown as typeof fetch;
50
+ return counter;
51
+ }
52
+
53
+ /** Run the two-target fixture with its second task replaced by a two-case scenario. */
54
+ async function runMixedCaseFixture(): Promise<ExperimentFileRunOutcome & { outputDir: string }> {
55
+ const fixture = await writeExperimentFixture();
56
+ await writeFile(path.join(fixture.directory, 'tasks/fail.yaml'), MIXED_SCENARIO);
57
+ const tasks = fixture.manifest.tasks as Array<{ source: { artifact: { digest: string } } }>;
58
+ tasks[1].source.artifact.digest = sha256(MIXED_SCENARIO);
59
+ await writeFile(fixture.file, JSON.stringify(fixture.manifest, null, 2));
60
+ const outcome = await runExperimentFile({
61
+ experiment: fixture.file,
62
+ outputDir: path.join(fixture.directory, 'out'),
63
+ });
64
+ return outcome as ExperimentFileRunOutcome & { outputDir: string };
65
+ }
66
+
67
+ function availableCounts(summary: ExperimentCaseLevelSummary | undefined) {
68
+ if (summary?.availability !== 'available') throw new Error('case level unavailable');
69
+ return summary;
70
+ }
71
+
72
+ describe('case-level summary surfaces', () => {
73
+ test('a run reports cases beside coordinates from its saved, verified evidence', async () => {
74
+ const network = forbidNetwork();
75
+ const outcome = await runMixedCaseFixture();
76
+ expect(network.calls).toBe(0);
77
+
78
+ const result = outcome.result as ExperimentRunResult;
79
+ expect(result.summaries.overall).toMatchObject({ planned: 4, passed: 2, task_failed: 2 });
80
+ const caseLevel = availableCounts(outcome.caseLevel);
81
+ expect(caseLevel.verifiedEvidenceFiles).toBe(4);
82
+ expect(caseLevel.overall).toMatchObject({
83
+ planned: 6,
84
+ recorded: 6,
85
+ valid: 6,
86
+ passed: 4,
87
+ failed: 2,
88
+ invalid: 0,
89
+ unmeasured: 0,
90
+ });
91
+ expect(caseLevel.targets.map((group) => [group.key, group.counts.passed])).toEqual([
92
+ ['qwen-small', 2],
93
+ ['qwen-large', 2],
94
+ ]);
95
+
96
+ const summary = formatExperimentRunSummary(result, {
97
+ resultPath: outcome.resultPath,
98
+ caseLevel: outcome.caseLevel,
99
+ });
100
+ expect(summary).toContain('Coordinate level: counts below are task × target × repetition');
101
+ expect(summary).toContain('valid 2/2 planned; passed 1 and task-failed 1 of 2 valid');
102
+ expect(summary).toContain('Case level: counts below are individual scenario cases from 4');
103
+ expect(summary).toContain(
104
+ ' - qwen-small: passed 2 and failed 1 of 3 valid case(s) (passed/valid 2/3 (66.7%))'
105
+ );
106
+ expect(summary).toContain('Overall cases: passed 4 and failed 2 of 6 valid case(s)');
107
+ expect(summary).toContain('No ranking or winner is implied');
108
+ expect(summary.indexOf('qwen-small: passed')).toBeLessThan(
109
+ summary.indexOf('qwen-large: passed')
110
+ );
111
+ expect(formatExperimentRunSummary(result)).not.toContain('Case level');
112
+ });
113
+
114
+ test('reports from a saved result read evidence beside it in every view', async () => {
115
+ forbidNetwork();
116
+ const outcome = await runMixedCaseFixture();
117
+ const reports = await reportExperimentResult({ result: outcome.resultPath as string });
118
+ expect(reports).toHaveLength(6);
119
+ for (const report of reports) {
120
+ expect(report.content).toContain('Case-level summary');
121
+ expect(report.content).toContain('Coordinate level');
122
+ expect(report.content).not.toContain('Unavailable.');
123
+ }
124
+ const html = reports.find((report) => report.format === 'html')?.content ?? '';
125
+ expect(html).toContain('<dt>Passed cases</dt><dd>4</dd>');
126
+ expect(html).toContain('<dt>Planned coordinates</dt><dd>4</dd>');
127
+ });
128
+
129
+ test('a parsed result needs an explicit evidence directory', async () => {
130
+ forbidNetwork();
131
+ const outcome = await runMixedCaseFixture();
132
+ const result = outcome.result as ExperimentRunResult;
133
+ const [without] = await reportExperimentResult({
134
+ result,
135
+ views: ['executive'],
136
+ formats: ['markdown'],
137
+ });
138
+ expect(without.content).toContain('**Unavailable.**');
139
+ const [withEvidence] = await reportExperimentResult({
140
+ result,
141
+ views: ['executive'],
142
+ formats: ['markdown'],
143
+ evidenceDir: path.join(outcome.outputDir, 'evidence'),
144
+ });
145
+ expect(withEvidence.content).toContain('| Passed cases | 4 |');
146
+ expect(
147
+ await summarizeExperimentCases({
148
+ result,
149
+ evidenceDir: path.join(outcome.outputDir, 'evidence'),
150
+ })
151
+ ).toEqual(outcome.caseLevel as ExperimentCaseLevelSummary);
152
+ });
153
+
154
+ test('altered, missing or absent evidence makes the case level unavailable', async () => {
155
+ forbidNetwork();
156
+ const outcome = await runMixedCaseFixture();
157
+ const resultPath = outcome.resultPath as string;
158
+ const evidenceDir = path.join(outcome.outputDir, 'evidence');
159
+ const [first, second] = (await readdir(evidenceDir)).sort();
160
+
161
+ await appendFile(path.join(evidenceDir, first), ' ');
162
+ const altered = await summarizeExperimentCases({ result: resultPath });
163
+ expect(altered).toMatchObject({
164
+ availability: 'unavailable',
165
+ reason: 'evidence_digest_mismatch',
166
+ affectedEvidence: [first.replace('.json', '')],
167
+ });
168
+ expect(formatExperimentCaseSummary(altered)).toStartWith(
169
+ 'Case level: unavailable. Case-level summary withheld: 1 referenced evidence file(s) failed SHA-256 verification'
170
+ );
171
+
172
+ await rm(path.join(evidenceDir, second));
173
+ expect(await summarizeExperimentCases({ result: resultPath })).toMatchObject({
174
+ reason: 'evidence_digest_mismatch',
175
+ affectedEvidenceCount: 2,
176
+ });
177
+
178
+ await rename(evidenceDir, `${evidenceDir}-moved`);
179
+ const absent = await summarizeExperimentCases({ result: resultPath });
180
+ expect(absent).toMatchObject({ reason: 'evidence_not_provided' });
181
+ // Old results without evidence still produce every report.
182
+ const reports = await reportExperimentResult({ result: resultPath });
183
+ expect(reports).toHaveLength(6);
184
+ for (const report of reports) expect(report.content).toContain('Planned coordinates');
185
+ });
186
+
187
+ test('a deleted evidence file is reported as missing', async () => {
188
+ forbidNetwork();
189
+ const outcome = await runMixedCaseFixture();
190
+ const evidenceDir = path.join(outcome.outputDir, 'evidence');
191
+ const [first] = (await readdir(evidenceDir)).sort();
192
+ await rm(path.join(evidenceDir, first));
193
+ expect(await summarizeExperimentCases({ result: outcome.resultPath as string })).toMatchObject({
194
+ reason: 'evidence_missing',
195
+ affectedEvidenceCount: 1,
196
+ });
197
+ });
198
+
199
+ test('maps unreadable and invalid results to exit codes 1 and 2', async () => {
200
+ const fixture = await writeExperimentFixture();
201
+ const missing = await summarizeExperimentCases({
202
+ result: path.join(fixture.directory, 'missing.json'),
203
+ }).catch((error) => error);
204
+ const invalid = await summarizeExperimentCases({ result: fixture.file }).catch(
205
+ (error) => error
206
+ );
207
+ expect(missing).toBeInstanceOf(ExperimentSurfaceError);
208
+ expect(missing.exitCode).toBe(1);
209
+ expect(invalid).toBeInstanceOf(ExperimentSurfaceError);
210
+ expect(invalid.exitCode).toBe(2);
211
+ });
212
+ });