@artemiskit/core 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -0
- package/dist/adapters/registry.d.ts.map +1 -1
- package/dist/adapters/types.d.ts +20 -2
- package/dist/adapters/types.d.ts.map +1 -1
- package/dist/agent-evaluation/index.d.ts +3 -0
- package/dist/agent-evaluation/index.d.ts.map +1 -0
- package/dist/agent-evaluation/scorer.d.ts +35 -0
- package/dist/agent-evaluation/scorer.d.ts.map +1 -0
- package/dist/agent-evaluation/types.d.ts +37 -0
- package/dist/agent-evaluation/types.d.ts.map +1 -0
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +52 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/evaluators/index.d.ts +1 -0
- package/dist/evaluators/index.d.ts.map +1 -1
- package/dist/evaluators/json-schema.d.ts +0 -1
- package/dist/evaluators/json-schema.d.ts.map +1 -1
- package/dist/evaluators/llm-grader.d.ts +2 -0
- package/dist/evaluators/llm-grader.d.ts.map +1 -1
- package/dist/evaluators/tool-trace.d.ts +7 -0
- package/dist/evaluators/tool-trace.d.ts.map +1 -0
- package/dist/evaluators/types.d.ts +20 -0
- package/dist/evaluators/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20281 -13144
- package/dist/runner/executor.d.ts.map +1 -1
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/types.d.ts +4 -0
- package/dist/runner/types.d.ts.map +1 -1
- package/dist/scenario/schema.d.ts +721 -63
- package/dist/scenario/schema.d.ts.map +1 -1
- package/dist/storage/local.d.ts +1 -1
- package/dist/storage/local.d.ts.map +1 -1
- package/dist/storage/supabase.d.ts +1 -1
- package/dist/storage/supabase.d.ts.map +1 -1
- package/dist/storage/types.d.ts +6 -2
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/tools/fixture-executor.d.ts +10 -0
- package/dist/tools/fixture-executor.d.ts.map +1 -0
- package/dist/tools/index.d.ts +4 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/schema-validator.d.ts +10 -0
- package/dist/tools/schema-validator.d.ts.map +1 -0
- package/dist/tools/types.d.ts +50 -0
- package/dist/tools/types.d.ts.map +1 -0
- package/package.json +2 -1
- package/src/adapters/registry.ts +7 -0
- package/src/adapters/types.test.ts +21 -0
- package/src/adapters/types.ts +18 -0
- package/src/agent-evaluation/index.ts +2 -0
- package/src/agent-evaluation/scorer.test.ts +1194 -0
- package/src/agent-evaluation/scorer.ts +640 -0
- package/src/agent-evaluation/types.test.ts +27 -0
- package/src/agent-evaluation/types.ts +43 -0
- package/src/artifacts/manifest.test.ts +90 -19
- package/src/artifacts/manifest.ts +18 -5
- package/src/artifacts/types.ts +133 -0
- package/src/evaluators/index.ts +3 -0
- package/src/evaluators/json-schema.test.ts +130 -0
- package/src/evaluators/json-schema.ts +38 -63
- package/src/evaluators/llm-grader.test.ts +80 -0
- package/src/evaluators/llm-grader.ts +44 -6
- package/src/evaluators/tool-trace.test.ts +46 -0
- package/src/evaluators/tool-trace.ts +50 -0
- package/src/evaluators/types.ts +20 -0
- package/src/index.ts +6 -0
- package/src/runner/executor.test.ts +374 -0
- package/src/runner/executor.ts +349 -22
- package/src/runner/release-validation.test.ts +169 -0
- package/src/runner/runner.ts +7 -1
- package/src/runner/types.ts +4 -0
- package/src/scenario/schema.ts +56 -1
- package/src/storage/local.test.ts +24 -0
- package/src/storage/local.ts +13 -2
- package/src/storage/supabase.test.ts +111 -1
- package/src/storage/supabase.ts +26 -3
- package/src/storage/types.ts +12 -2
- package/src/tools/fixture-executor.test.ts +88 -0
- package/src/tools/fixture-executor.ts +112 -0
- package/src/tools/index.ts +3 -0
- package/src/tools/schema-validator.test.ts +32 -0
- package/src/tools/schema-validator.ts +56 -0
- package/src/tools/types.ts +80 -0
- package/adapters/openai/dist/index.js +0 -5626
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import { validateToolArguments } from './schema-validator';
|
|
3
|
+
|
|
4
|
+
const weatherSchema = {
|
|
5
|
+
type: 'object',
|
|
6
|
+
additionalProperties: false,
|
|
7
|
+
required: ['city'],
|
|
8
|
+
properties: { city: { type: 'string', minLength: 1 } },
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
describe('validateToolArguments', () => {
|
|
12
|
+
it('returns parsed arguments that satisfy the JSON schema', () => {
|
|
13
|
+
expect(validateToolArguments('{"city":"Lagos"}', weatherSchema)).toEqual({
|
|
14
|
+
valid: true,
|
|
15
|
+
arguments: { city: 'Lagos' },
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('rejects malformed JSON without echoing the arguments', () => {
|
|
20
|
+
expect(validateToolArguments('{city:Lagos}', weatherSchema)).toEqual({
|
|
21
|
+
valid: false,
|
|
22
|
+
error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('rejects arguments that fail the declared schema', () => {
|
|
27
|
+
const result = validateToolArguments('{"city":42}', weatherSchema);
|
|
28
|
+
|
|
29
|
+
expect(result.valid).toBe(false);
|
|
30
|
+
expect(result.error?.code).toBe('TOOL_ARGUMENTS_SCHEMA_INVALID');
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import Ajv, { type ValidateFunction } from 'ajv';
|
|
2
|
+
|
|
3
|
+
export interface ToolArgumentValidation {
|
|
4
|
+
valid: boolean;
|
|
5
|
+
arguments?: Record<string, unknown>;
|
|
6
|
+
error?: { code: 'TOOL_ARGUMENTS_INVALID' | 'TOOL_ARGUMENTS_SCHEMA_INVALID'; message: string };
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
10
|
+
const validators = new WeakMap<object, ValidateFunction>();
|
|
11
|
+
|
|
12
|
+
function getValidator(schema: Record<string, unknown>): ValidateFunction {
|
|
13
|
+
const cached = validators.get(schema);
|
|
14
|
+
if (cached) return cached;
|
|
15
|
+
const validator = ajv.compile(schema);
|
|
16
|
+
validators.set(schema, validator);
|
|
17
|
+
return validator;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function validateToolArguments(
|
|
21
|
+
rawArguments: string,
|
|
22
|
+
schema: Record<string, unknown>
|
|
23
|
+
): ToolArgumentValidation {
|
|
24
|
+
let args: unknown;
|
|
25
|
+
try {
|
|
26
|
+
args = JSON.parse(rawArguments);
|
|
27
|
+
} catch {
|
|
28
|
+
return {
|
|
29
|
+
valid: false,
|
|
30
|
+
error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (!args || typeof args !== 'object' || Array.isArray(args)) {
|
|
35
|
+
return {
|
|
36
|
+
valid: false,
|
|
37
|
+
error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be a JSON object.' },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const validator = getValidator(schema);
|
|
42
|
+
if (!validator(args)) {
|
|
43
|
+
const details = (validator.errors ?? [])
|
|
44
|
+
.map((error) => `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`)
|
|
45
|
+
.join('; ');
|
|
46
|
+
return {
|
|
47
|
+
valid: false,
|
|
48
|
+
error: {
|
|
49
|
+
code: 'TOOL_ARGUMENTS_SCHEMA_INVALID',
|
|
50
|
+
message: `Tool arguments do not match the schema: ${details}`,
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return { valid: true, arguments: args as Record<string, unknown> };
|
|
56
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { ToolCall, ToolDefinition } from '../adapters/types';
|
|
2
|
+
|
|
3
|
+
export type ToolLoopTerminationReason =
|
|
4
|
+
| 'completed'
|
|
5
|
+
| 'max_steps'
|
|
6
|
+
| 'timeout'
|
|
7
|
+
| 'duplicate_call'
|
|
8
|
+
| 'invalid_arguments'
|
|
9
|
+
| 'unknown_tool'
|
|
10
|
+
| 'tool_error';
|
|
11
|
+
|
|
12
|
+
export interface ToolLoopPolicy {
|
|
13
|
+
enabled: boolean;
|
|
14
|
+
maxSteps: number;
|
|
15
|
+
timeoutMs: number;
|
|
16
|
+
maxToolResultBytes: number;
|
|
17
|
+
rejectDuplicateCalls: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const DEFAULT_TOOL_LOOP_POLICY: ToolLoopPolicy = {
|
|
21
|
+
enabled: false,
|
|
22
|
+
maxSteps: 5,
|
|
23
|
+
timeoutMs: 60_000,
|
|
24
|
+
maxToolResultBytes: 32_768,
|
|
25
|
+
rejectDuplicateCalls: true,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export interface ToolExecutionContext {
|
|
29
|
+
caseId: string;
|
|
30
|
+
step: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface ToolExecutionError {
|
|
34
|
+
code:
|
|
35
|
+
| 'TOOL_ARGUMENTS_INVALID'
|
|
36
|
+
| 'TOOL_ARGUMENTS_SCHEMA_INVALID'
|
|
37
|
+
| 'TOOL_UNKNOWN'
|
|
38
|
+
| 'TOOL_FIXTURE_NOT_FOUND'
|
|
39
|
+
| 'TOOL_EXECUTION_FAILED'
|
|
40
|
+
| 'TOOL_RESULT_TOO_LARGE';
|
|
41
|
+
message: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface ToolExecutionResult {
|
|
45
|
+
status: 'success' | 'error';
|
|
46
|
+
result?: unknown;
|
|
47
|
+
error?: ToolExecutionError;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface ToolExecutor {
|
|
51
|
+
execute(call: ToolCall, context: ToolExecutionContext): Promise<ToolExecutionResult>;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface ToolTraceEntry {
|
|
55
|
+
step: number;
|
|
56
|
+
toolCall: ToolCall;
|
|
57
|
+
result?: unknown;
|
|
58
|
+
error?: ToolExecutionError;
|
|
59
|
+
latencyMs: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface ToolLoopSummary {
|
|
63
|
+
status: 'completed' | 'error';
|
|
64
|
+
steps: number;
|
|
65
|
+
terminationReason: ToolLoopTerminationReason;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ToolFixture {
|
|
69
|
+
when?: Record<string, unknown>;
|
|
70
|
+
result?: unknown;
|
|
71
|
+
error?: string;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export type ToolFixtures = Record<string, ToolFixture[]>;
|
|
75
|
+
|
|
76
|
+
export interface FixtureExecutorOptions {
|
|
77
|
+
tools: ToolDefinition[];
|
|
78
|
+
fixtures: ToolFixtures;
|
|
79
|
+
maxToolResultBytes?: number;
|
|
80
|
+
}
|