@artemiskit/core 0.2.4 → 0.4.0

package/src/scenario/schema.ts

@@ -18,6 +18,9 @@ export const ProviderSchema = z.enum([
   'cohere',
   'huggingface',
   'ollama',
+  'langchain',
+  'deepagents',
+  'ling',
   'custom',
 ]);
 
@@ -50,6 +53,17 @@ export const ProviderConfigSchema = z
 
     // Vercel AI specific
     underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
+
+    // LangChain specific
+    name: z.string().optional(),
+    runnableType: z.enum(['chain', 'agent', 'llm', 'runnable']).optional(),
+
+    // DeepAgents specific
+    captureTraces: z.boolean().optional(),
+    captureMessages: z.boolean().optional(),
+    thinking: z.object({ type: z.enum(['enabled', 'disabled']) }).optional(),
+    enableSearch: z.boolean().optional(),
+    searchOptions: z.record(z.unknown()).optional(),
   })
   .optional();
 
@@ -81,6 +95,8 @@ const BaseExpectedSchema = z.discriminatedUnion('type', [
     model: z.string().optional(),
     provider: ProviderSchema.optional(),
     threshold: z.number().min(0).max(1).default(0.7),
+    /** Require exact, validated JSON from the judge for assurance assessments. */
+    strict: z.boolean().optional().default(false),
   }),
 
   z.object({
@@ -106,6 +122,14 @@ const BaseExpectedSchema = z.discriminatedUnion('type', [
     config: z.record(z.unknown()).optional(),
   }),
 
+  z.object({
+    type: z.literal('tool_trace'),
+    requiredTools: z.array(z.string()).optional(),
+    forbiddenTools: z.array(z.string()).optional(),
+    ordered: z.boolean().optional().default(false),
+    maxCalls: z.number().int().min(0).optional(),
+  }),
+
   z.object({
     type: z.literal('similarity'),
     value: z.string(),
@@ -145,10 +169,39 @@ export const ExpectedSchema = z.union([BaseExpectedSchema, CombinedExpectedSchem
  * Chat message schema
  */
 export const ChatMessageSchema = z.object({
-  role: z.enum(['system', 'user', 'assistant']),
+  role: z.enum(['system', 'user', 'assistant', 'tool']),
   content: z.string(),
+  name: z.string().optional(),
+  toolCallId: z.string().optional(),
+  tool_calls: z
+    .array(
+      z.object({
+        id: z.string(),
+        type: z.literal('function'),
+        function: z.object({ name: z.string(), arguments: z.string() }),
+      })
+    )
+    .optional(),
 });
 
+const ToolSchema = z
+  .object({
+    type: z.literal('function'),
+    function: z.object({
+      name: z.string(),
+      description: z.string().optional(),
+      parameters: z.record(z.unknown()),
+    }),
+  })
+  .strict();
+const ToolFixtureSchema = z
+  .object({
+    when: z.record(z.unknown()).optional(),
+    result: z.unknown().optional(),
+    error: z.string().optional(),
+  })
+  .strict();
+
 /**
  * Variables schema - key-value pairs for template substitution
  */
@@ -201,6 +254,18 @@ export const ScenarioSchema = z.object({
     .object({
       systemPrompt: z.string().optional(),
       functions: z.array(z.unknown()).optional(),
+      tools: z.array(ToolSchema).optional(),
+      fixtures: z.record(z.array(ToolFixtureSchema)).optional(),
+      toolLoop: z
+        .object({
+          enabled: z.boolean().default(false),
+          maxSteps: z.number().int().min(1).max(10).default(5),
+          timeoutMs: z.number().int().min(1).max(300_000).default(60_000),
+          maxToolResultBytes: z.number().int().min(1).max(1_048_576).default(32_768),
+          rejectDuplicateCalls: z.boolean().default(true),
+        })
+        .strict()
+        .optional(),
     })
     .optional(),
   cases: z.array(TestCaseSchema).min(1),