@artemiskit/core 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/adapters/factory.d.ts +1 -1
  3. package/dist/adapters/index.d.ts +3 -3
  4. package/dist/adapters/registry.d.ts +1 -1
  5. package/dist/agent-evaluation/index.d.ts +2 -2
  6. package/dist/agent-evaluation/scorer.d.ts +1 -1
  7. package/dist/agent-workflow/catalog.d.ts +27 -0
  8. package/dist/agent-workflow/catalog.d.ts.map +1 -0
  9. package/dist/agent-workflow/index.d.ts +7 -0
  10. package/dist/agent-workflow/index.d.ts.map +1 -0
  11. package/dist/agent-workflow/parser.d.ts +7 -0
  12. package/dist/agent-workflow/parser.d.ts.map +1 -0
  13. package/dist/agent-workflow/schema.d.ts +664 -0
  14. package/dist/agent-workflow/schema.d.ts.map +1 -0
  15. package/dist/agent-workflow/simulated-tools.d.ts +35 -0
  16. package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
  17. package/dist/agent-workflow/target.d.ts +230 -0
  18. package/dist/agent-workflow/target.d.ts.map +1 -0
  19. package/dist/artifacts/index.d.ts +2 -2
  20. package/dist/artifacts/manifest.d.ts +1 -1
  21. package/dist/artifacts/types.d.ts +2 -2
  22. package/dist/comparison/eligibility.d.ts +26 -0
  23. package/dist/comparison/eligibility.d.ts.map +1 -0
  24. package/dist/comparison/index.d.ts +2 -0
  25. package/dist/comparison/index.d.ts.map +1 -0
  26. package/dist/evaluators/combined.d.ts +2 -2
  27. package/dist/evaluators/contains.d.ts +2 -2
  28. package/dist/evaluators/exact.d.ts +2 -2
  29. package/dist/evaluators/fuzzy.d.ts +2 -2
  30. package/dist/evaluators/index.d.ts +13 -13
  31. package/dist/evaluators/inline.d.ts +2 -2
  32. package/dist/evaluators/json-schema.d.ts +2 -2
  33. package/dist/evaluators/llm-grader.d.ts +2 -2
  34. package/dist/evaluators/not-contains.d.ts +2 -2
  35. package/dist/evaluators/regex.d.ts +2 -2
  36. package/dist/evaluators/similarity.d.ts +2 -2
  37. package/dist/evaluators/tool-trace.d.ts +2 -2
  38. package/dist/evaluators/types.d.ts +3 -3
  39. package/dist/index.d.ts +15 -13
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +16346 -14967
  42. package/dist/provenance/execution-provenance.d.ts +1 -1
  43. package/dist/provenance/git.d.ts +1 -1
  44. package/dist/provenance/index.d.ts +4 -4
  45. package/dist/provenance/workload-identity.d.ts +2 -2
  46. package/dist/redaction/index.d.ts +2 -2
  47. package/dist/redaction/redactor.d.ts +1 -1
  48. package/dist/runner/executor.d.ts +3 -3
  49. package/dist/runner/index.d.ts +3 -3
  50. package/dist/runner/runner.d.ts +1 -1
  51. package/dist/runner/types.d.ts +5 -5
  52. package/dist/scenario/index.d.ts +4 -4
  53. package/dist/scenario/parser.d.ts +1 -1
  54. package/dist/scenario/variables.d.ts +1 -1
  55. package/dist/storage/factory.d.ts +1 -1
  56. package/dist/storage/index.d.ts +4 -4
  57. package/dist/storage/local.d.ts +2 -2
  58. package/dist/storage/local.d.ts.map +1 -1
  59. package/dist/storage/supabase.d.ts +2 -2
  60. package/dist/storage/supabase.d.ts.map +1 -1
  61. package/dist/storage/types.d.ts +6 -2
  62. package/dist/storage/types.d.ts.map +1 -1
  63. package/dist/tools/fixture-executor.d.ts +2 -2
  64. package/dist/tools/index.d.ts +3 -3
  65. package/dist/tools/types.d.ts +1 -1
  66. package/dist/utils/index.d.ts +2 -2
  67. package/dist/validator/index.d.ts +2 -2
  68. package/dist/validator/validator.d.ts +1 -1
  69. package/package.json +4 -4
  70. package/src/agent-workflow/catalog.ts +219 -0
  71. package/src/agent-workflow/index.ts +6 -0
  72. package/src/agent-workflow/parser.ts +43 -0
  73. package/src/agent-workflow/schema.test.ts +218 -0
  74. package/src/agent-workflow/schema.ts +254 -0
  75. package/src/agent-workflow/simulated-tools.test.ts +177 -0
  76. package/src/agent-workflow/simulated-tools.ts +230 -0
  77. package/src/agent-workflow/target.test.ts +276 -0
  78. package/src/agent-workflow/target.ts +292 -0
  79. package/src/comparison/eligibility.test.ts +101 -0
  80. package/src/comparison/eligibility.ts +116 -0
  81. package/src/comparison/index.ts +8 -0
  82. package/src/index.ts +6 -0
  83. package/src/storage/local.test.ts +39 -0
  84. package/src/storage/local.ts +9 -1
  85. package/src/storage/supabase.ts +9 -1
  86. package/src/storage/types.ts +5 -1
@@ -0,0 +1,35 @@
1
+ import { type WorkflowJson, type WorkflowPolicy } from './schema.js';
2
+ type JsonObject = {
3
+ [key: string]: WorkflowJson;
4
+ };
5
+ type ToolFailureCode = 'undeclared_tool' | 'permission_denied' | 'invalid_input' | 'invalid_state' | 'not_found' | 'output_limit' | 'invalid_policy' | 'tool_error';
6
+ export interface SimulatedToolEvidence {
7
+ tool: string;
8
+ version: '1';
9
+ status: 'succeeded' | 'denied' | 'invalid' | 'failed';
10
+ code?: ToolFailureCode;
11
+ }
12
+ export type SimulatedToolResult = {
13
+ status: 'succeeded';
14
+ output: WorkflowJson;
15
+ state: JsonObject;
16
+ evidence: SimulatedToolEvidence;
17
+ } | {
18
+ status: 'denied' | 'invalid' | 'failed';
19
+ code: ToolFailureCode;
20
+ evidence: SimulatedToolEvidence;
21
+ };
22
+ /**
23
+ * One deterministic simulated call, with isolated input/output state and metadata-only evidence.
24
+ * Budgets are declarative here: a future environment runner owns action counts and deadlines.
25
+ * `side_effects` governs external effects; declared local state writes never authorize real effects.
26
+ */
27
+ export declare function executeSimulatedTool(request: {
28
+ tool: string;
29
+ input: unknown;
30
+ state: unknown;
31
+ policy: WorkflowPolicy;
32
+ declaredTools: readonly string[];
33
+ }): SimulatedToolResult;
34
+ export {};
35
+ //# sourceMappingURL=simulated-tools.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"simulated-tools.d.ts","sourceRoot":"","sources":["../../src/agent-workflow/simulated-tools.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,cAAc,EAIpB,MAAM,UAAU,CAAC;AAElB,KAAK,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,CAAA;CAAE,CAAC;AAClD,KAAK,eAAe,GAChB,iBAAiB,GACjB,mBAAmB,GACnB,eAAe,GACf,eAAe,GACf,WAAW,GACX,cAAc,GACd,gBAAgB,GAChB,YAAY,CAAC;AACjB,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,GAAG,CAAC;IACb,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;IACtD,IAAI,CAAC,EAAE,eAAe,CAAC;CACxB;AACD,MAAM,MAAM,mBAAmB,GAC3B;IACE,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACjC,GACD;IACE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;IACxC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,qBAAqB,CAAC;CACjC,CAAC;AAqCN;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,cAAc,CAAC;IACvB,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC,GAAG,mBAAmB,CAgJtB"}
@@ -0,0 +1,230 @@
1
+ import { z } from 'zod';
2
+ import type { ModelClient, TokenUsage, ToolCall } from '../adapters/types.js';
3
+ declare const requestSchema: z.ZodObject<{
4
+ messages: z.ZodArray<z.ZodObject<{
5
+ role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
6
+ content: z.ZodString;
7
+ toolCallId: z.ZodOptional<z.ZodString>;
8
+ tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
9
+ id: z.ZodString;
10
+ type: z.ZodLiteral<"function">;
11
+ function: z.ZodObject<{
12
+ name: z.ZodString;
13
+ arguments: z.ZodString;
14
+ }, "strict", z.ZodTypeAny, {
15
+ name: string;
16
+ arguments: string;
17
+ }, {
18
+ name: string;
19
+ arguments: string;
20
+ }>;
21
+ }, "strict", z.ZodTypeAny, {
22
+ function: {
23
+ name: string;
24
+ arguments: string;
25
+ };
26
+ type: "function";
27
+ id: string;
28
+ }, {
29
+ function: {
30
+ name: string;
31
+ arguments: string;
32
+ };
33
+ type: "function";
34
+ id: string;
35
+ }>, "many">>;
36
+ }, "strict", z.ZodTypeAny, {
37
+ role: "system" | "user" | "assistant" | "tool";
38
+ content: string;
39
+ tool_calls?: {
40
+ function: {
41
+ name: string;
42
+ arguments: string;
43
+ };
44
+ type: "function";
45
+ id: string;
46
+ }[] | undefined;
47
+ toolCallId?: string | undefined;
48
+ }, {
49
+ role: "system" | "user" | "assistant" | "tool";
50
+ content: string;
51
+ tool_calls?: {
52
+ function: {
53
+ name: string;
54
+ arguments: string;
55
+ };
56
+ type: "function";
57
+ id: string;
58
+ }[] | undefined;
59
+ toolCallId?: string | undefined;
60
+ }>, "many">;
61
+ tools: z.ZodArray<z.ZodObject<{
62
+ type: z.ZodLiteral<"function">;
63
+ function: z.ZodObject<{
64
+ name: z.ZodString;
65
+ description: z.ZodOptional<z.ZodString>;
66
+ parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
67
+ }, "strict", z.ZodTypeAny, {
68
+ name: string;
69
+ parameters: Record<string, unknown>;
70
+ description?: string | undefined;
71
+ }, {
72
+ name: string;
73
+ parameters: Record<string, unknown>;
74
+ description?: string | undefined;
75
+ }>;
76
+ }, "strict", z.ZodTypeAny, {
77
+ function: {
78
+ name: string;
79
+ parameters: Record<string, unknown>;
80
+ description?: string | undefined;
81
+ };
82
+ type: "function";
83
+ }, {
84
+ function: {
85
+ name: string;
86
+ parameters: Record<string, unknown>;
87
+ description?: string | undefined;
88
+ };
89
+ type: "function";
90
+ }>, "many">;
91
+ model: z.ZodOptional<z.ZodString>;
92
+ generation: z.ZodObject<{
93
+ maxTokens: z.ZodNumber;
94
+ temperature: z.ZodOptional<z.ZodNumber>;
95
+ topP: z.ZodOptional<z.ZodNumber>;
96
+ seed: z.ZodOptional<z.ZodNumber>;
97
+ stop: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
98
+ }, "strict", z.ZodTypeAny, {
99
+ maxTokens: number;
100
+ stop?: string[] | undefined;
101
+ seed?: number | undefined;
102
+ temperature?: number | undefined;
103
+ topP?: number | undefined;
104
+ }, {
105
+ maxTokens: number;
106
+ stop?: string[] | undefined;
107
+ seed?: number | undefined;
108
+ temperature?: number | undefined;
109
+ topP?: number | undefined;
110
+ }>;
111
+ budgets: z.ZodObject<{
112
+ timeoutMs: z.ZodNumber;
113
+ maxToolCalls: z.ZodNumber;
114
+ }, "strict", z.ZodTypeAny, {
115
+ timeoutMs: number;
116
+ maxToolCalls: number;
117
+ }, {
118
+ timeoutMs: number;
119
+ maxToolCalls: number;
120
+ }>;
121
+ }, "strict", z.ZodTypeAny, {
122
+ tools: {
123
+ function: {
124
+ name: string;
125
+ parameters: Record<string, unknown>;
126
+ description?: string | undefined;
127
+ };
128
+ type: "function";
129
+ }[];
130
+ generation: {
131
+ maxTokens: number;
132
+ stop?: string[] | undefined;
133
+ seed?: number | undefined;
134
+ temperature?: number | undefined;
135
+ topP?: number | undefined;
136
+ };
137
+ budgets: {
138
+ timeoutMs: number;
139
+ maxToolCalls: number;
140
+ };
141
+ messages: {
142
+ role: "system" | "user" | "assistant" | "tool";
143
+ content: string;
144
+ tool_calls?: {
145
+ function: {
146
+ name: string;
147
+ arguments: string;
148
+ };
149
+ type: "function";
150
+ id: string;
151
+ }[] | undefined;
152
+ toolCallId?: string | undefined;
153
+ }[];
154
+ model?: string | undefined;
155
+ }, {
156
+ tools: {
157
+ function: {
158
+ name: string;
159
+ parameters: Record<string, unknown>;
160
+ description?: string | undefined;
161
+ };
162
+ type: "function";
163
+ }[];
164
+ generation: {
165
+ maxTokens: number;
166
+ stop?: string[] | undefined;
167
+ seed?: number | undefined;
168
+ temperature?: number | undefined;
169
+ topP?: number | undefined;
170
+ };
171
+ budgets: {
172
+ timeoutMs: number;
173
+ maxToolCalls: number;
174
+ };
175
+ messages: {
176
+ role: "system" | "user" | "assistant" | "tool";
177
+ content: string;
178
+ tool_calls?: {
179
+ function: {
180
+ name: string;
181
+ arguments: string;
182
+ };
183
+ type: "function";
184
+ id: string;
185
+ }[] | undefined;
186
+ toolCallId?: string | undefined;
187
+ }[];
188
+ model?: string | undefined;
189
+ }>;
190
+ export type AgentTurnRequest = z.infer<typeof requestSchema>;
191
+ export type AgentTargetFailure = {
192
+ status: 'unsupported' | 'invalid' | 'error';
193
+ code: 'invalid_request' | 'tool_use_unsupported' | 'invalid_response' | 'target_error' | 'timeout' | 'aborted';
194
+ };
195
+ export type AgentTargetCapabilities = {
196
+ status: 'available';
197
+ toolUse: boolean;
198
+ /** ModelClient has no AbortSignal contract. Timeouts bound waiting, not transport work. */
199
+ transportCancellation: false;
200
+ };
201
+ export type AgentTurnResult = AgentTargetFailure | {
202
+ status: 'completed';
203
+ id: string;
204
+ model: string;
205
+ message: {
206
+ role: 'assistant';
207
+ content: string;
208
+ tool_calls?: ToolCall[];
209
+ };
210
+ /** Adapter-reported counts only; zero can mean unavailable in existing adapters. */
211
+ tokens: TokenUsage;
212
+ latencyMs: number;
213
+ finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter';
214
+ };
215
+ /** One bounded model turn. Tool execution, policy enforcement, and scoring belong to the harness. */
216
+ export interface AgentTarget {
217
+ readonly provider: string;
218
+ capabilities(options: {
219
+ timeoutMs: number;
220
+ }, signal?: AbortSignal): Promise<AgentTargetCapabilities | AgentTargetFailure>;
221
+ turn(request: AgentTurnRequest, signal?: AbortSignal): Promise<AgentTurnResult>;
222
+ }
223
+ /**
224
+ * Bridge existing adapters without provider-specific dispatch. Callers must configure adapter
225
+ * transport timeouts/retries separately: cancellation here cannot stop an in-flight provider call.
226
+ * Returned text/tool arguments are working conversation data, not sanitized retained evidence.
227
+ */
228
+ export declare function createModelClientTarget(client: ModelClient): AgentTarget;
229
+ export {};
230
+ //# sourceMappingURL=target.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"target.d.ts","sourceRoot":"","sources":["../../src/agent-workflow/target.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAmB,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAqB5F,QAAA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoCR,CAAC;AAkBZ,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,aAAa,GAAG,SAAS,GAAG,OAAO,CAAC;IAC5C,IAAI,EACA,iBAAiB,GACjB,sBAAsB,GACtB,kBAAkB,GAClB,cAAc,GACd,SAAS,GACT,SAAS,CAAC;CACf,CAAC;AACF,MAAM,MAAM,uBAAuB,GAAG;IACpC,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,2FAA2F;IAC3F,qBAAqB,EAAE,KAAK,CAAC;CAC9B,CAAC;AACF,MAAM,MAAM,eAAe,GACvB,kBAAkB,GAClB;IACE,MAAM,EAAE,WAAW,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,IAAI,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAC;IACzE,oFAAoF;IACpF,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,GAAG,gBAAgB,CAAC;CACpE,CAAC;AAEN,qGAAqG;AACrG,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,YAAY,CACV,OAAO,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,EAC9B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,uBAAuB,GAAG,kBAAkB,CAAC,CAAC;IACzD,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjF;AA0DD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CAiHxE"}
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Artifacts module exports
3
3
  */
4
- export * from './types';
5
- export { createRunManifest } from './manifest';
4
+ export * from './types.js';
5
+ export { createRunManifest } from './manifest.js';
6
6
  //# sourceMappingURL=index.d.ts.map
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Run manifest generation utilities
3
3
  */
4
- import type { CaseResult, CostProvenance, ExecutionProvenance, ManifestRedactionInfo, ResolvedConfig, RunAttemptEvidence, RunConfig, RunManifest, WorkloadIdentity } from './types';
4
+ import type { CaseResult, CostProvenance, ExecutionProvenance, ManifestRedactionInfo, ResolvedConfig, RunAttemptEvidence, RunConfig, RunManifest, WorkloadIdentity } from './types.js';
5
5
  /**
6
6
  * Create a new run manifest
7
7
  */
@@ -162,9 +162,9 @@ export interface CaseResult {
162
162
  /** Redaction information for this case */
163
163
  redaction?: CaseRedactionInfo;
164
164
  /** Ordered tool activity captured for an enabled tool loop. */
165
- toolTrace?: import('../tools').ToolTraceEntry[];
165
+ toolTrace?: import('../tools/index.js').ToolTraceEntry[];
166
166
  /** Terminal status for an enabled tool loop. */
167
- toolLoop?: import('../tools').ToolLoopSummary;
167
+ toolLoop?: import('../tools/index.js').ToolLoopSummary;
168
168
  }
169
169
  /**
170
170
  * Cost estimation details
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Compatibility decisions for comparisons of saved scenario-evaluation runs.
3
+ *
4
+ * This contract compares declared evidence only. A matching digest proves the
5
+ * same canonical workload/rubric was declared; it does not attest to provider
6
+ * behaviour or replace a future assessment-profile compatibility decision.
7
+ */
8
+ import type { RunManifest } from '../artifacts/types.js';
9
+ export type ComparisonEligibilityStatus = 'compatible' | 'qualified' | 'incomparable';
10
+ export type ComparisonEligibilityReasonCode = 'scenario_mismatch' | 'workload_identity_missing' | 'rubric_identity_missing' | 'workload_mismatch' | 'rubric_mismatch' | 'execution_provenance_missing' | 'target_provider_changed' | 'target_model_changed' | 'generation_settings_changed';
11
+ /** A bounded, machine-readable reason for a comparison decision. */
12
+ export interface ComparisonEligibilityReason {
13
+ code: ComparisonEligibilityReasonCode;
14
+ }
15
+ /**
16
+ * Versioned decision that accompanies every comparison. `qualified` permits
17
+ * a visibly-qualified delta; `incomparable` prohibits a delta entirely.
18
+ */
19
+ export interface ComparisonEligibility {
20
+ schema_version: '1';
21
+ status: ComparisonEligibilityStatus;
22
+ reasons: ComparisonEligibilityReason[];
23
+ }
24
+ export declare function assessComparisonEligibility(baseline: RunManifest, current: RunManifest): ComparisonEligibility;
25
+ export declare function isComparisonAvailable(eligibility: ComparisonEligibility): boolean;
26
+ //# sourceMappingURL=eligibility.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eligibility.d.ts","sourceRoot":"","sources":["../../src/comparison/eligibility.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,MAAM,2BAA2B,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,CAAC;AAEtF,MAAM,MAAM,+BAA+B,GACvC,mBAAmB,GACnB,2BAA2B,GAC3B,yBAAyB,GACzB,mBAAmB,GACnB,iBAAiB,GACjB,8BAA8B,GAC9B,yBAAyB,GACzB,sBAAsB,GACtB,6BAA6B,CAAC;AAElC,oEAAoE;AACpE,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,+BAA+B,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,cAAc,EAAE,GAAG,CAAC;IACpB,MAAM,EAAE,2BAA2B,CAAC;IACpC,OAAO,EAAE,2BAA2B,EAAE,CAAC;CACxC;AAED,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,WAAW,GACnB,qBAAqB,CAmDvB;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,qBAAqB,GAAG,OAAO,CAEjF"}
@@ -0,0 +1,2 @@
1
+ export { assessComparisonEligibility, isComparisonAvailable, type ComparisonEligibility, type ComparisonEligibilityReason, type ComparisonEligibilityReasonCode, type ComparisonEligibilityStatus, } from './eligibility.js';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/comparison/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,EAChC,KAAK,+BAA+B,EACpC,KAAK,2BAA2B,GACjC,MAAM,eAAe,CAAC"}
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Combined evaluator - evaluates multiple expectations with and/or logic
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types.js';
6
6
  export declare class CombinedEvaluator implements Evaluator {
7
7
  readonly type = "combined";
8
8
  evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Contains evaluator - checks if response contains specific values
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class ContainsEvaluator implements Evaluator {
7
7
  readonly type = "contains";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Exact match evaluator
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class ExactEvaluator implements Evaluator {
7
7
  readonly type = "exact";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Fuzzy match evaluator using Levenshtein distance
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class FuzzyEvaluator implements Evaluator {
7
7
  readonly type = "fuzzy";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Evaluators module - exports all evaluator types and utilities
3
3
  */
4
- import type { Evaluator } from './types';
4
+ import type { Evaluator } from './types.js';
5
5
  /**
6
6
  * Get an evaluator by type
7
7
  */
@@ -14,16 +14,16 @@ export declare function registerEvaluator(type: string, evaluator: Evaluator): v
14
14
  * List available evaluator types
15
15
  */
16
16
  export declare function listEvaluators(): string[];
17
- export * from './types';
18
- export { ExactEvaluator } from './exact';
19
- export { RegexEvaluator } from './regex';
20
- export { FuzzyEvaluator } from './fuzzy';
21
- export { ContainsEvaluator } from './contains';
22
- export { NotContainsEvaluator } from './not-contains';
23
- export { CombinedEvaluator } from './combined';
24
- export { JsonSchemaEvaluator } from './json-schema';
25
- export { LLMGraderEvaluator } from './llm-grader';
26
- export { SimilarityEvaluator } from './similarity';
27
- export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';
28
- export { ToolTraceEvaluator } from './tool-trace';
17
+ export * from './types.js';
18
+ export { ExactEvaluator } from './exact.js';
19
+ export { RegexEvaluator } from './regex.js';
20
+ export { FuzzyEvaluator } from './fuzzy.js';
21
+ export { ContainsEvaluator } from './contains.js';
22
+ export { NotContainsEvaluator } from './not-contains.js';
23
+ export { CombinedEvaluator } from './combined.js';
24
+ export { JsonSchemaEvaluator } from './json-schema.js';
25
+ export { LLMGraderEvaluator } from './llm-grader.js';
26
+ export { SimilarityEvaluator } from './similarity.js';
27
+ export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline.js';
28
+ export { ToolTraceEvaluator } from './tool-trace.js';
29
29
  //# sourceMappingURL=index.d.ts.map
@@ -9,8 +9,8 @@
9
9
  * - Regex matching: response.match(/pattern/)
10
10
  * - JSON parsing: JSON.parse(response)
11
11
  */
12
- import type { Expected } from '../scenario/schema';
13
- import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
12
+ import type { Expected } from '../scenario/schema.js';
13
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types.js';
14
14
  export declare class InlineEvaluator implements Evaluator {
15
15
  readonly type = "inline";
16
16
  evaluate(response: string, expected: Expected, _context?: EvaluatorContext): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * JSON Schema evaluator - validates response against a JSON schema
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class JsonSchemaEvaluator implements Evaluator {
7
7
  readonly type = "json_schema";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * LLM-based grader evaluator
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types.js';
6
6
  export declare class LLMGraderEvaluator implements Evaluator {
7
7
  readonly type = "llm_grader";
8
8
  evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Not Contains evaluator - checks if response does NOT contain specific values
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class NotContainsEvaluator implements Evaluator {
7
7
  readonly type = "not_contains";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Regex pattern evaluator
3
3
  */
4
- import type { Expected } from '../scenario/schema';
5
- import type { Evaluator, EvaluatorResult } from './types';
4
+ import type { Expected } from '../scenario/schema.js';
5
+ import type { Evaluator, EvaluatorResult } from './types.js';
6
6
  export declare class RegexEvaluator implements Evaluator {
7
7
  readonly type = "regex";
8
8
  evaluate(response: string, expected: Expected): Promise<EvaluatorResult>;
@@ -3,8 +3,8 @@
3
3
  * Uses vector embeddings for semantic similarity matching when available,
4
4
  * falls back to LLM-based semantic comparison otherwise.
5
5
  */
6
- import type { Expected } from '../scenario/schema';
7
- import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
6
+ import type { Expected } from '../scenario/schema.js';
7
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types.js';
8
8
  export declare class SimilarityEvaluator implements Evaluator {
9
9
  readonly type = "similarity";
10
10
  evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
@@ -1,5 +1,5 @@
1
- import type { Expected } from '../scenario/schema';
2
- import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
1
+ import type { Expected } from '../scenario/schema.js';
2
+ import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types.js';
3
3
  export declare class ToolTraceEvaluator implements Evaluator {
4
4
  readonly type = "tool_trace";
5
5
  evaluate(_response: string, expected: Expected, context?: EvaluatorContext): Promise<EvaluatorResult>;
@@ -1,9 +1,9 @@
1
1
  /**
2
2
  * Evaluator types and interfaces
3
3
  */
4
- import type { ModelClient } from '../adapters/types';
5
- import type { Expected, TestCase } from '../scenario/schema';
6
- import type { ToolTraceEntry } from '../tools';
4
+ import type { ModelClient } from '../adapters/types.js';
5
+ import type { Expected, TestCase } from '../scenario/schema.js';
6
+ import type { ToolTraceEntry } from '../tools/index.js';
7
7
  /**
8
8
  * Context provided to evaluators
9
9
  */
package/dist/index.d.ts CHANGED
@@ -2,17 +2,19 @@
2
2
  * @artemiskit/core
3
3
  * Core library for Artemis Agent Reliability Toolkit
4
4
  */
5
- export * from './adapters';
6
- export * from './scenario';
7
- export * from './evaluators';
8
- export * from './runner';
9
- export * from './storage';
10
- export * from './artifacts';
11
- export * from './provenance';
12
- export * from './utils';
13
- export * from './redaction';
14
- export * from './cost';
15
- export * from './tools';
16
- export * from './agent-evaluation';
17
- export * from './validator';
5
+ export * from './adapters/index.js';
6
+ export * from './scenario/index.js';
7
+ export * from './evaluators/index.js';
8
+ export * from './runner/index.js';
9
+ export * from './storage/index.js';
10
+ export * from './artifacts/index.js';
11
+ export * from './provenance/index.js';
12
+ export * from './comparison/index.js';
13
+ export * from './utils/index.js';
14
+ export * from './redaction/index.js';
15
+ export * from './cost/index.js';
16
+ export * from './tools/index.js';
17
+ export * from './agent-evaluation/index.js';
18
+ export * from './agent-workflow/index.js';
19
+ export * from './validator/index.js';
18
20
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC;AAGvB,cAAc,SAAS,CAAC;AAGxB,cAAc,oBAAoB,CAAC;AAGnC,cAAc,aAAa,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC;AAGvB,cAAc,SAAS,CAAC;AAGxB,cAAc,oBAAoB,CAAC;AAGnC,cAAc,kBAAkB,CAAC;AAGjC,cAAc,aAAa,CAAC"}