@artemiskit/core 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +4 -0
  3. package/dist/adapters/registry.d.ts.map +1 -1
  4. package/dist/adapters/types.d.ts +20 -2
  5. package/dist/adapters/types.d.ts.map +1 -1
  6. package/dist/agent-evaluation/index.d.ts +3 -0
  7. package/dist/agent-evaluation/index.d.ts.map +1 -0
  8. package/dist/agent-evaluation/scorer.d.ts +35 -0
  9. package/dist/agent-evaluation/scorer.d.ts.map +1 -0
  10. package/dist/agent-evaluation/types.d.ts +37 -0
  11. package/dist/agent-evaluation/types.d.ts.map +1 -0
  12. package/dist/artifacts/manifest.d.ts.map +1 -1
  13. package/dist/artifacts/types.d.ts +42 -0
  14. package/dist/artifacts/types.d.ts.map +1 -1
  15. package/dist/evaluators/index.d.ts +1 -0
  16. package/dist/evaluators/index.d.ts.map +1 -1
  17. package/dist/evaluators/json-schema.d.ts +0 -1
  18. package/dist/evaluators/json-schema.d.ts.map +1 -1
  19. package/dist/evaluators/llm-grader.d.ts +2 -0
  20. package/dist/evaluators/llm-grader.d.ts.map +1 -1
  21. package/dist/evaluators/tool-trace.d.ts +7 -0
  22. package/dist/evaluators/tool-trace.d.ts.map +1 -0
  23. package/dist/evaluators/types.d.ts +20 -0
  24. package/dist/evaluators/types.d.ts.map +1 -1
  25. package/dist/index.d.ts +2 -0
  26. package/dist/index.d.ts.map +1 -1
  27. package/dist/index.js +20169 -13138
  28. package/dist/runner/executor.d.ts.map +1 -1
  29. package/dist/runner/runner.d.ts.map +1 -1
  30. package/dist/runner/types.d.ts +4 -0
  31. package/dist/runner/types.d.ts.map +1 -1
  32. package/dist/scenario/schema.d.ts +721 -63
  33. package/dist/scenario/schema.d.ts.map +1 -1
  34. package/dist/storage/supabase.d.ts +1 -1
  35. package/dist/storage/supabase.d.ts.map +1 -1
  36. package/dist/storage/types.d.ts +6 -2
  37. package/dist/storage/types.d.ts.map +1 -1
  38. package/dist/tools/fixture-executor.d.ts +10 -0
  39. package/dist/tools/fixture-executor.d.ts.map +1 -0
  40. package/dist/tools/index.d.ts +4 -0
  41. package/dist/tools/index.d.ts.map +1 -0
  42. package/dist/tools/schema-validator.d.ts +10 -0
  43. package/dist/tools/schema-validator.d.ts.map +1 -0
  44. package/dist/tools/types.d.ts +50 -0
  45. package/dist/tools/types.d.ts.map +1 -0
  46. package/package.json +2 -1
  47. package/src/adapters/registry.ts +7 -0
  48. package/src/adapters/types.test.ts +21 -0
  49. package/src/adapters/types.ts +18 -0
  50. package/src/agent-evaluation/index.ts +2 -0
  51. package/src/agent-evaluation/scorer.test.ts +1194 -0
  52. package/src/agent-evaluation/scorer.ts +640 -0
  53. package/src/agent-evaluation/types.test.ts +27 -0
  54. package/src/agent-evaluation/types.ts +43 -0
  55. package/src/artifacts/manifest.test.ts +66 -19
  56. package/src/artifacts/manifest.ts +18 -5
  57. package/src/artifacts/types.ts +49 -0
  58. package/src/evaluators/index.ts +3 -0
  59. package/src/evaluators/json-schema.test.ts +130 -0
  60. package/src/evaluators/json-schema.ts +38 -63
  61. package/src/evaluators/llm-grader.test.ts +80 -0
  62. package/src/evaluators/llm-grader.ts +44 -6
  63. package/src/evaluators/tool-trace.test.ts +46 -0
  64. package/src/evaluators/tool-trace.ts +50 -0
  65. package/src/evaluators/types.ts +20 -0
  66. package/src/index.ts +6 -0
  67. package/src/runner/executor.test.ts +340 -0
  68. package/src/runner/executor.ts +289 -20
  69. package/src/runner/release-validation.test.ts +169 -0
  70. package/src/runner/runner.ts +5 -1
  71. package/src/runner/types.ts +4 -0
  72. package/src/scenario/schema.ts +56 -1
  73. package/src/storage/supabase.test.ts +65 -1
  74. package/src/storage/supabase.ts +17 -2
  75. package/src/storage/types.ts +12 -2
  76. package/src/tools/fixture-executor.test.ts +88 -0
  77. package/src/tools/fixture-executor.ts +112 -0
  78. package/src/tools/index.ts +3 -0
  79. package/src/tools/schema-validator.test.ts +32 -0
  80. package/src/tools/schema-validator.ts +56 -0
  81. package/src/tools/types.ts +80 -0
  82. package/adapters/openai/dist/index.js +0 -5626
package/dist/runner/executor.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAqB,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAGxE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAoCrB"}
1
+ {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAuCrB"}
package/dist/runner/runner.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAqHzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAyHzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
package/dist/runner/types.d.ts CHANGED
@@ -5,6 +5,7 @@ import type { ModelClient } from '../adapters/types';
5
5
  import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
6
6
  import type { RedactionConfig } from '../redaction/types';
7
7
  import type { Scenario } from '../scenario/schema';
8
+ import type { ToolExecutor } from '../tools';
8
9
  /**
9
10
  * Options for running a scenario
10
11
  */
@@ -27,6 +28,8 @@ export interface RunOptions {
27
28
  retries?: number;
28
29
  /** Redaction configuration (CLI overrides scenario) */
29
30
  redaction?: RedactionConfig;
31
+ /** SDK-only executor for explicitly supplied real tools. */
32
+ toolExecutor?: ToolExecutor;
30
33
  /** Callback for each case result */
31
34
  onCaseComplete?: (result: CaseResult, index: number, total: number) => void;
32
35
  /** Callback for progress updates */
@@ -53,5 +56,6 @@ export interface ExecutorContext {
53
56
  retries?: number;
54
57
  /** Redaction configuration for this execution */
55
58
  redaction?: RedactionConfig;
59
+ toolExecutor?: ToolExecutor;
56
60
  }
57
61
  //# sourceMappingURL=types.d.ts.map
package/dist/runner/types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}