@artemiskit/core 0.5.2 → 0.6.0

package/src/agent-workflow/target.ts

@@ -0,0 +1,292 @@
+import Ajv from 'ajv';
+import { z } from 'zod';
+import type { GenerateOptions, ModelClient, TokenUsage, ToolCall } from '../adapters/types';
+
+const identifier = z.string().min(1).max(256);
+const toolName = z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_-]{0,63}$/);
+const text = z.string().max(1_000_000);
+const toolCallSchema = z
+  .object({
+    id: identifier,
+    type: z.literal('function'),
+    function: z.object({ name: toolName, arguments: text }).strict(),
+  })
+  .strict();
+const messageSchema = z
+  .object({
+    role: z.enum(['system', 'user', 'assistant', 'tool']),
+    content: text,
+    toolCallId: identifier.optional(),
+    tool_calls: z.array(toolCallSchema).min(1).max(100).optional(),
+  })
+  .strict();
+const timeoutSchema = z.number().int().min(1).max(2_147_483_647);
+const requestSchema = z
+  .object({
+    messages: z.array(messageSchema).min(1).max(1000),
+    tools: z
+      .array(
+        z
+          .object({
+            type: z.literal('function'),
+            function: z
+              .object({
+                name: toolName,
+                description: z.string().max(10_000).optional(),
+                parameters: z.record(z.unknown()),
+              })
+              .strict(),
+          })
+          .strict()
+      )
+      .max(100),
+    model: identifier.optional(),
+    generation: z
+      .object({
+        maxTokens: z.number().int().positive().max(1_000_000),
+        temperature: z.number().finite().min(0).max(2).optional(),
+        topP: z.number().finite().min(0).max(1).optional(),
+        seed: z.number().int().safe().optional(),
+        stop: z.array(z.string().min(1).max(1000)).max(16).optional(),
+      })
+      .strict(),
+    budgets: z
+      .object({
+        timeoutMs: timeoutSchema,
+        maxToolCalls: z.number().int().min(0).max(100),
+      })
+      .strict(),
+  })
+  .strict();
+const resultSchema = z.object({
+  id: identifier,
+  model: identifier,
+  text,
+  tokens: z.object({
+    prompt: z.number().int().nonnegative().safe(),
+    completion: z.number().int().nonnegative().safe(),
+    total: z.number().int().nonnegative().safe(),
+  }),
+  latencyMs: z.number().finite().nonnegative(),
+  finishReason: z
+    .enum(['stop', 'length', 'function_call', 'tool_calls', 'content_filter'])
+    .optional(),
+  toolCalls: z.array(toolCallSchema).max(100).optional(),
+  functionCall: z.unknown().optional(),
+});
+
+export type AgentTurnRequest = z.infer<typeof requestSchema>;
+export type AgentTargetFailure = {
+  status: 'unsupported' | 'invalid' | 'error';
+  code:
+    | 'invalid_request'
+    | 'tool_use_unsupported'
+    | 'invalid_response'
+    | 'target_error'
+    | 'timeout'
+    | 'aborted';
+};
+export type AgentTargetCapabilities = {
+  status: 'available';
+  toolUse: boolean;
+  /** ModelClient has no AbortSignal contract. Timeouts bound waiting, not transport work. */
+  transportCancellation: false;
+};
+export type AgentTurnResult =
+  | AgentTargetFailure
+  | {
+      status: 'completed';
+      id: string;
+      model: string;
+      message: { role: 'assistant'; content: string; tool_calls?: ToolCall[] };
+      /** Adapter-reported counts only; zero can mean unavailable in existing adapters. */
+      tokens: TokenUsage;
+      latencyMs: number;
+      finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter';
+    };
+
+/** One bounded model turn. Tool execution, policy enforcement, and scoring belong to the harness. */
+export interface AgentTarget {
+  readonly provider: string;
+  capabilities(
+    options: { timeoutMs: number },
+    signal?: AbortSignal
+  ): Promise<AgentTargetCapabilities | AgentTargetFailure>;
+  turn(request: AgentTurnRequest, signal?: AbortSignal): Promise<AgentTurnResult>;
+}
+
+const failure = (
+  status: AgentTargetFailure['status'],
+  code: AgentTargetFailure['code']
+): AgentTargetFailure => ({ status, code });
+
+/** Do not expose provider exception text, which can contain credentials or customer content. */
+function bounded<T>(
+  run: () => Promise<T>,
+  timeoutMs: number,
+  signal?: AbortSignal
+): Promise<T | AgentTargetFailure> {
+  if (signal?.aborted) return Promise.resolve(failure('error', 'aborted'));
+  return new Promise((resolve) => {
+    let finished = false;
+    const finish = (value: T | AgentTargetFailure) => {
+      if (finished) return;
+      finished = true;
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', abort);
+      resolve(value);
+    };
+    const abort = () => finish(failure('error', 'aborted'));
+    const timer = setTimeout(() => finish(failure('error', 'timeout')), timeoutMs);
+    signal?.addEventListener('abort', abort, { once: true });
+    Promise.resolve()
+      .then<T | AgentTargetFailure>(() => (signal?.aborted ? failure('error', 'aborted') : run()))
+      .then(finish, () => finish(failure('error', 'target_error')));
+  });
+}
+
+function validTranscript(messages: AgentTurnRequest['messages']): boolean {
+  const seen = new Set<string>();
+  const pending = new Set<string>();
+  for (const message of messages) {
+    if (message.role === 'tool') {
+      if (!message.toolCallId || message.tool_calls || !pending.delete(message.toolCallId))
+        return false;
+      continue;
+    }
+    if (message.toolCallId || pending.size || (message.tool_calls && message.role !== 'assistant'))
+      return false;
+    for (const call of message.tool_calls ?? []) {
+      if (seen.has(call.id)) return false;
+      try {
+        const args: unknown = JSON.parse(call.function.arguments);
+        if (!args || typeof args !== 'object' || Array.isArray(args)) return false;
+      } catch {
+        return false;
+      }
+      seen.add(call.id);
+      pending.add(call.id);
+    }
+  }
+  return pending.size === 0;
+}
+
+/**
+ * Bridge existing adapters without provider-specific dispatch. Callers must configure adapter
+ * transport timeouts/retries separately: cancellation here cannot stop an in-flight provider call.
+ * Returned text/tool arguments are working conversation data, not sanitized retained evidence.
+ */
+export function createModelClientTarget(client: ModelClient): AgentTarget {
+  if (
+    !client ||
+    !identifier.safeParse(client.provider).success ||
+    typeof client.generate !== 'function' ||
+    typeof client.capabilities !== 'function'
+  ) {
+    throw new TypeError('Invalid ModelClient');
+  }
+  const readCapabilities = async (): Promise<AgentTargetCapabilities | AgentTargetFailure> => {
+    const value = await client.capabilities();
+    if (!value || typeof value.toolUse !== 'boolean') return failure('invalid', 'invalid_response');
+    return { status: 'available', toolUse: value.toolUse, transportCancellation: false };
+  };
+  return {
+    provider: client.provider,
+    async capabilities(options, signal) {
+      if (!timeoutSchema.safeParse(options?.timeoutMs).success)
+        return failure('invalid', 'invalid_request');
+      return bounded(readCapabilities, options.timeoutMs, signal);
+    },
+    async turn(request, signal) {
+      let parsed: ReturnType<typeof requestSchema.safeParse>;
+      try {
+        parsed = requestSchema.safeParse(request);
+      } catch {
+        return failure('invalid', 'invalid_request');
+      }
+      if (!parsed.success || !validTranscript(parsed.data.messages))
+        return failure('invalid', 'invalid_request');
+      const input = parsed.data;
+      const validators = new Map<string, ReturnType<Ajv['compile']>>();
+      try {
+        const ajv = new Ajv({ strict: false, allErrors: false, validateFormats: false });
+        for (const tool of input.tools) {
+          if (validators.has(tool.function.name)) return failure('invalid', 'invalid_request');
+          // Compile a detached JSON schema without remote loading or async validation.
+          const schema = JSON.parse(JSON.stringify(tool.function.parameters));
+          if (schema.$async) return failure('invalid', 'invalid_request');
+          const validate = ajv.compile(schema);
+          if ('$async' in validate && validate.$async) return failure('invalid', 'invalid_request');
+          validators.set(tool.function.name, validate);
+        }
+      } catch {
+        return failure('invalid', 'invalid_request');
+      }
+      const started = Date.now();
+      return bounded(
+        async (): Promise<AgentTurnResult> => {
+          const capabilities = await readCapabilities();
+          if (capabilities.status !== 'available') return capabilities;
+          if (input.tools.length && !capabilities.toolUse)
+            return failure('unsupported', 'tool_use_unsupported');
+          if (signal?.aborted) return failure('error', 'aborted');
+          if (Date.now() - started >= input.budgets.timeoutMs) return failure('error', 'timeout');
+          const options: GenerateOptions = {
+            prompt: input.messages,
+            tools: input.tools,
+            model: input.model,
+            ...input.generation,
+          };
+          const generated = resultSchema.safeParse(await client.generate(options));
+          if (!generated.success) return failure('invalid', 'invalid_response');
+          const result = generated.data;
+          const calls = result.toolCalls ?? [];
+          if (
+            result.functionCall !== undefined ||
+            result.finishReason === 'function_call' ||
+            (result.finishReason === 'tool_calls' && calls.length === 0) ||
+            calls.length > input.budgets.maxToolCalls ||
+            result.tokens.total !== result.tokens.prompt + result.tokens.completion
+          )
+            return failure('invalid', 'invalid_response');
+          const ids = new Set(
+            input.messages.flatMap((message) => message.tool_calls?.map((call) => call.id) ?? [])
+          );
+          for (const call of calls) {
+            if (ids.has(call.id)) return failure('invalid', 'invalid_response');
+            ids.add(call.id);
+            const validate = validators.get(call.function.name);
+            try {
+              const args: unknown = JSON.parse(call.function.arguments);
+              if (
+                !args ||
+                typeof args !== 'object' ||
+                Array.isArray(args) ||
+                !validate ||
+                validate(args) !== true
+              )
+                return failure('invalid', 'invalid_response');
+            } catch {
+              return failure('invalid', 'invalid_response');
+            }
+          }
+          return {
+            status: 'completed',
+            id: result.id,
+            model: result.model,
+            message: {
+              role: 'assistant',
+              content: result.text,
+              ...(calls.length ? { tool_calls: calls } : {}),
+            },
+            tokens: result.tokens,
+            latencyMs: result.latencyMs,
+            finishReason: result.finishReason,
+          };
+        },
+        input.budgets.timeoutMs,
+        signal
+      );
+    },
+  };
+}

package/src/comparison/eligibility.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from 'bun:test';
+import type { RunManifest } from '../artifacts';
+import { assessComparisonEligibility, isComparisonAvailable } from './eligibility';
+
+function manifest(overrides: Partial<RunManifest> = {}): RunManifest {
+  return {
+    version: '1.3',
+    run_id: 'run-id',
+    project: 'project',
+    start_time: '2026-09-10T00:00:00.000Z',
+    end_time: '2026-09-10T00:00:01.000Z',
+    duration_ms: 1000,
+    config: { scenario: 'customer-service', provider: 'openai', model: 'model-a' },
+    workload_identity: {
+      schema_version: '1',
+      workload: { schema_version: '1', algorithm: 'sha256', digest: 'a'.repeat(64) },
+      rubric: { schema_version: '1', algorithm: 'sha256', digest: 'b'.repeat(64) },
+    },
+    execution_provenance: {
+      schema_version: '1',
+      target: { provider: 'openai', requested_models: ['model-a'], generation: { temperature: 0 } },
+    },
+    metrics: {
+      success_rate: 1,
+      total_cases: 1,
+      passed_cases: 1,
+      failed_cases: 0,
+      median_latency_ms: 1,
+      p95_latency_ms: 1,
+      total_tokens: 1,
+      total_prompt_tokens: 1,
+      total_completion_tokens: 0,
+    },
+    git: { commit: 'commit', branch: 'main', dirty: false },
+    provenance: { run_by: 'test' },
+    cases: [],
+    environment: { node_version: 'test', platform: 'test', arch: 'test' },
+    ...overrides,
+  };
+}
+
+describe('assessComparisonEligibility', () => {
+  test('accepts matching declared workload, rubric, and execution configuration', () => {
+    const eligibility = assessComparisonEligibility(manifest(), manifest({ run_id: 'current' }));
+
+    expect(eligibility).toEqual({ schema_version: '1', status: 'compatible', reasons: [] });
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies legacy artifacts instead of presenting them as fully compatible', () => {
+    const legacy = manifest({ workload_identity: undefined, execution_provenance: undefined });
+    const eligibility = assessComparisonEligibility(legacy, manifest({ run_id: 'current' }));
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_identity_missing',
+      'rubric_identity_missing',
+      'execution_provenance_missing',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies a deliberate target-model change without treating it as the same execution', () => {
+    const current = manifest({
+      run_id: 'current',
+      execution_provenance: {
+        schema_version: '1',
+        target: {
+          provider: 'openai',
+          requested_models: ['model-b'],
+          generation: { temperature: 0 },
+        },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons).toEqual([{ code: 'target_model_changed' }]);
+  });
+
+  test('refuses deltas when workload or rubric evidence differs', () => {
+    const current = manifest({
+      run_id: 'current',
+      workload_identity: {
+        schema_version: '1',
+        workload: { schema_version: '1', algorithm: 'sha256', digest: 'c'.repeat(64) },
+        rubric: { schema_version: '1', algorithm: 'sha256', digest: 'd'.repeat(64) },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('incomparable');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_mismatch',
+      'rubric_mismatch',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(false);
+  });
+});

package/src/comparison/eligibility.ts

@@ -0,0 +1,116 @@
+/**
+ * Compatibility decisions for comparisons of saved scenario-evaluation runs.
+ *
+ * This contract compares declared evidence only. A matching digest proves the
+ * same canonical workload/rubric was declared; it does not attest to provider
+ * behaviour or replace a future assessment-profile compatibility decision.
+ */
+
+import type { RunManifest } from '../artifacts/types';
+
+export type ComparisonEligibilityStatus = 'compatible' | 'qualified' | 'incomparable';
+
+export type ComparisonEligibilityReasonCode =
+  | 'scenario_mismatch'
+  | 'workload_identity_missing'
+  | 'rubric_identity_missing'
+  | 'workload_mismatch'
+  | 'rubric_mismatch'
+  | 'execution_provenance_missing'
+  | 'target_provider_changed'
+  | 'target_model_changed'
+  | 'generation_settings_changed';
+
+/** A bounded, machine-readable reason for a comparison decision. */
+export interface ComparisonEligibilityReason {
+  code: ComparisonEligibilityReasonCode;
+}
+
+/**
+ * Versioned decision that accompanies every comparison. `qualified` permits
+ * a visibly-qualified delta; `incomparable` prohibits a delta entirely.
+ */
+export interface ComparisonEligibility {
+  schema_version: '1';
+  status: ComparisonEligibilityStatus;
+  reasons: ComparisonEligibilityReason[];
+}
+
+export function assessComparisonEligibility(
+  baseline: RunManifest,
+  current: RunManifest
+): ComparisonEligibility {
+  const reasons: ComparisonEligibilityReason[] = [];
+
+  if (baseline.config.scenario !== current.config.scenario) {
+    reasons.push({ code: 'scenario_mismatch' });
+  }
+
+  const baselineIdentity = baseline.workload_identity;
+  const currentIdentity = current.workload_identity;
+  if (!baselineIdentity || !currentIdentity) {
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'workload_identity_missing' });
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'rubric_identity_missing' });
+  } else {
+    if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
+      reasons.push({ code: 'workload_mismatch' });
+    }
+    if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
+      reasons.push({ code: 'rubric_mismatch' });
+    }
+  }
+
+  if (reasons.some((reason) => isIncomparableReason(reason.code))) {
+    return { schema_version: '1', status: 'incomparable', reasons };
+  }
+
+  const baselineExecution = baseline.execution_provenance;
+  const currentExecution = current.execution_provenance;
+  if (!baselineExecution || !currentExecution) {
+    reasons.push({ code: 'execution_provenance_missing' });
+  } else {
+    if (baselineExecution.target.provider !== currentExecution.target.provider) {
+      reasons.push({ code: 'target_provider_changed' });
+    }
+    if (
+      !sameStringSet(
+        baselineExecution.target.requested_models,
+        currentExecution.target.requested_models
+      )
+    ) {
+      reasons.push({ code: 'target_model_changed' });
+    }
+    if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
+      reasons.push({ code: 'generation_settings_changed' });
+    }
+  }
+
+  return {
+    schema_version: '1',
+    status: reasons.length === 0 ? 'compatible' : 'qualified',
+    reasons,
+  };
+}
+
+export function isComparisonAvailable(eligibility: ComparisonEligibility): boolean {
+  return eligibility.status !== 'incomparable';
+}
+
+function isIncomparableReason(code: ComparisonEligibilityReasonCode): boolean {
+  return code === 'scenario_mismatch' || code === 'workload_mismatch' || code === 'rubric_mismatch';
+}
+
+function sameStringSet(left?: string[], right?: string[]): boolean {
+  return JSON.stringify([...(left ?? [])].sort()) === JSON.stringify([...(right ?? [])].sort());
+}
+
+function sameGeneration(
+  left?: { temperature?: number; max_tokens?: number; seed?: number },
+  right?: { temperature?: number; max_tokens?: number; seed?: number }
+): boolean {
+  return (
+    left?.temperature === right?.temperature &&
+    left?.max_tokens === right?.max_tokens &&
+    left?.seed === right?.seed
+  );
+}

package/src/comparison/index.ts

@@ -0,0 +1,8 @@
+export {
+  assessComparisonEligibility,
+  isComparisonAvailable,
+  type ComparisonEligibility,
+  type ComparisonEligibilityReason,
+  type ComparisonEligibilityReasonCode,
+  type ComparisonEligibilityStatus,
+} from './eligibility';

package/src/index.ts

@@ -24,6 +24,9 @@ export * from './artifacts';
 // Provenance
 export * from './provenance';
 
+// Comparison eligibility
+export * from './comparison';
+
 // Utilities
 export * from './utils';
 
@@ -39,5 +42,8 @@ export * from './tools';
 // Real-agent evaluation contracts
 export * from './agent-evaluation';
 
+// Versioned workflow contracts and single-step primitives
+export * from './agent-workflow';
+
 // Validator
 export * from './validator';

package/src/storage/local.test.ts

@@ -179,6 +179,45 @@ describe('LocalStorageAdapter', () => {
     expect(comparison.delta.successRate).toBeCloseTo(0.1, 2);
     expect(comparison.delta.latency).toBe(-30);
     expect(comparison.delta.tokens).toBe(100);
+    expect(comparison.eligibility.status).toBe('qualified');
+  });
+
+  test('withholds deltas for incompatible workload evidence', async () => {
+    const baseline = {
+      ...mockManifest,
+      run_id: 'incompatible-baseline',
+      workload_identity: {
+        schema_version: '1' as const,
+        workload: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'a'.repeat(64),
+        },
+        rubric: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'b'.repeat(64),
+        },
+      },
+      execution_provenance: { schema_version: '1' as const, target: { provider: 'openai' } },
+    };
+    const current = {
+      ...baseline,
+      run_id: 'incompatible-current',
+      workload_identity: {
+        ...baseline.workload_identity,
+        workload: { ...baseline.workload_identity.workload, digest: 'c'.repeat(64) },
+      },
+    };
+
+    await storage.save(baseline);
+    await storage.save(current);
+
+    const comparison = await storage.compare('incompatible-baseline', 'incompatible-current');
+
+    expect(comparison.eligibility.status).toBe('incomparable');
+    expect(comparison.eligibility.reasons).toEqual([{ code: 'workload_mismatch' }]);
+    expect(comparison.delta).toBeUndefined();
   });
 
   test('handles empty storage gracefully', async () => {

package/src/storage/local.ts

@@ -12,6 +12,7 @@ import {
   assertRunManifestIntegrity,
   isRunManifest,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   BaselineMetadata,
   BaselineStorageAdapter,
@@ -212,9 +213,15 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
       this.loadRun(currentId),
     ]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -375,7 +382,8 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (negative delta in success rate)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/supabase.ts

@@ -9,6 +9,7 @@ import {
   assertRunManifestIntegrity,
   getCaseEvaluationStatus,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   AnalyticsStorageAdapter,
   BaselineMetadata,
@@ -199,9 +200,15 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
   async compare(baselineId: string, currentId: string): Promise<ComparisonResult> {
     const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -400,7 +407,8 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (success rate dropped by more than threshold)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/types.ts

@@ -9,6 +9,7 @@ import type {
   RunManifest,
   StressManifest,
 } from '../artifacts/types';
+import type { ComparisonEligibility } from '../comparison';
 
 /**
  * Run listing item
@@ -30,7 +31,10 @@ export interface RunListItem {
 export interface ComparisonResult {
   baseline: RunManifest;
   current: RunManifest;
-  delta: {
+  /** Compatibility decision made before any metric delta is calculated. */
+  eligibility: ComparisonEligibility;
+  /** Absent when workloads or rubrics are incomparable. */
+  delta?: {
     successRate: number;
     latency: number;
     tokens: number;