@abassey/aid 0.1.0

package/dist/eval/index.cjs

@@ -0,0 +1,308 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; } async function _asyncOptionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = await fn(value); } else if (op === 'call' || op === 'optionalCall') { value = await fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }// src/eval/suite.ts
+function evalSuite(name) {
+  const cases = [];
+  const scorers = [];
+  let models = ["sonnet"];
+  let threshold = 0.5;
+  let concurrencyLimit = 1;
+  let aiFn = null;
+  const builder = {
+    case(caseName, testCase) {
+      cases.push({ name: caseName, ...testCase });
+      return builder;
+    },
+    scorer(scorer) {
+      scorers.push(scorer);
+      return builder;
+    },
+    models(m) {
+      models = m;
+      return builder;
+    },
+    threshold(value) {
+      threshold = value;
+      return builder;
+    },
+    concurrency(limit) {
+      concurrencyLimit = limit;
+      return builder;
+    },
+    ai(fn) {
+      aiFn = fn;
+      return builder;
+    },
+    async run() {
+      const ai = await _asyncNullishCoalesce(aiFn, async () => ( (await Promise.resolve().then(() => _interopRequireWildcard(require("../ai-DOAYJKKI.cjs")))).ai));
+      const jobs = [];
+      for (const evalCase of cases) {
+        for (const model of models) {
+          jobs.push({ evalCase, model });
+        }
+      }
+      const results = [];
+      if (concurrencyLimit <= 1) {
+        for (const job of jobs) {
+          results.push(await runSingleCase(ai, job.evalCase, job.model, scorers, threshold));
+        }
+      } else {
+        let index = 0;
+        const workers = Array.from({ length: Math.min(concurrencyLimit, jobs.length) }, async () => {
+          while (index < jobs.length) {
+            const jobIndex = index++;
+            const job = jobs[jobIndex];
+            const result = await runSingleCase(ai, job.evalCase, job.model, scorers, threshold);
+            results.push(result);
+          }
+        });
+        await Promise.all(workers);
+      }
+      return buildResults(name, results, scorers);
+    }
+  };
+  return builder;
+}
+async function runSingleCase(ai, evalCase, model, scorers, threshold) {
+  let output = "";
+  let tokens = { input: 0, output: 0, total: 0 };
+  let cost = 0;
+  let latencyMs = 0;
+  const scores = {};
+  try {
+    const startTime = Date.now();
+    const response = await ai(evalCase.input, {
+      model,
+      system: evalCase.system
+    });
+    latencyMs = Date.now() - startTime;
+    output = response.text;
+    tokens = response.tokens;
+    cost = response.cost;
+    for (const scorer of scorers) {
+      scores[scorer.name] = await scorer.score(output, evalCase.expected, evalCase.input);
+    }
+  } catch (error) {
+    for (const scorer of scorers) {
+      scores[scorer.name] = 0;
+    }
+  }
+  const pass = Object.values(scores).every((s) => s >= threshold);
+  return {
+    case: evalCase.name,
+    model,
+    output,
+    scores,
+    tokens,
+    cost,
+    latencyMs,
+    pass
+  };
+}
+function buildResults(suiteName, cases, scorers) {
+  const byModel = {};
+  const modelGroups = /* @__PURE__ */ new Map();
+  for (const c of cases) {
+    if (!modelGroups.has(c.model)) modelGroups.set(c.model, []);
+    modelGroups.get(c.model).push(c);
+  }
+  for (const [model, modelCases] of modelGroups) {
+    const allScores2 = modelCases.flatMap((c) => Object.values(c.scores));
+    byModel[model] = {
+      avgScore: allScores2.length > 0 ? allScores2.reduce((a, b) => a + b, 0) / allScores2.length : 0,
+      passRate: modelCases.filter((c) => c.pass).length / modelCases.length,
+      totalCost: modelCases.reduce((sum, c) => sum + c.cost, 0),
+      totalTokens: {
+        input: modelCases.reduce((sum, c) => sum + c.tokens.input, 0),
+        output: modelCases.reduce((sum, c) => sum + c.tokens.output, 0),
+        total: modelCases.reduce((sum, c) => sum + c.tokens.total, 0)
+      },
+      avgLatencyMs: modelCases.reduce((sum, c) => sum + c.latencyMs, 0) / modelCases.length
+    };
+  }
+  const byScorer = {};
+  for (const scorer of scorers) {
+    const scores = cases.map((c) => _nullishCoalesce(c.scores[scorer.name], () => ( 0)));
+    byScorer[scorer.name] = {
+      avgScore: scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0
+    };
+  }
+  const allScores = cases.flatMap((c) => Object.values(c.scores));
+  const overall = {
+    avgScore: allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : 0,
+    passRate: cases.length > 0 ? cases.filter((c) => c.pass).length / cases.length : 0,
+    totalCost: cases.reduce((sum, c) => sum + c.cost, 0)
+  };
+  return {
+    suite: suiteName,
+    timestamp: /* @__PURE__ */ new Date(),
+    cases,
+    summary: { byModel, byScorer, overall }
+  };
+}
+
+// src/eval/scorers.ts
+function exactMatch() {
+  return {
+    name: "exactMatch",
+    score(output, expected) {
+      if (expected === void 0) return 0;
+      return output.trim() === expected.trim() ? 1 : 0;
+    }
+  };
+}
+function contains(substring) {
+  return {
+    name: `contains(${substring})`,
+    score(output) {
+      return output.includes(substring) ? 1 : 0;
+    }
+  };
+}
+function regex(pattern) {
+  return {
+    name: `regex(${pattern})`,
+    score(output) {
+      return pattern.test(output) ? 1 : 0;
+    }
+  };
+}
+function custom(fn, name) {
+  return {
+    name: _nullishCoalesce(name, () => ( "custom")),
+    score(output, expected, input) {
+      const result = fn(output, expected, input);
+      if (result instanceof Promise) {
+        return result.then((raw) => Math.max(0, Math.min(1, raw)));
+      }
+      return Math.max(0, Math.min(1, result));
+    }
+  };
+}
+
+// src/eval/llm-judge.ts
+var BUILT_IN_CRITERIA = {
+  accuracy: "Does the output correctly and factually answer the input?",
+  relevance: "Is the output relevant and on-topic for the input?",
+  coherence: "Is the output well-structured, logical, and coherent?",
+  helpfulness: "Is the output helpful, actionable, and complete?"
+};
+function llmJudge(name, options) {
+  const model = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _ => _.model]), () => ( "sonnet"));
+  const criteria = _nullishCoalesce(_nullishCoalesce(_optionalChain([options, 'optionalAccess', _2 => _2.criteria]), () => ( BUILT_IN_CRITERIA[name])), () => ( name));
+  return {
+    name: `llmJudge(${name})`,
+    async score(output, expected, input) {
+      const ai = await _asyncNullishCoalesce(await _asyncOptionalChain([options, 'optionalAccess', async _3 => _3.ai]), async () => ( (await Promise.resolve().then(() => _interopRequireWildcard(require("../ai-DOAYJKKI.cjs")))).ai));
+      const prompt = [
+        "You are evaluating an AI output. Score from 0.0 to 1.0.",
+        "",
+        `Criteria: ${criteria}`,
+        `Input: ${input}`,
+        `Expected: ${_nullishCoalesce(expected, () => ( "Not provided"))}`,
+        `Actual Output: ${output}`,
+        "",
+        'Respond with JSON only: { "score": <number 0.0-1.0>, "reasoning": "<brief explanation>" }'
+      ].join("\n");
+      try {
+        const response = await ai(prompt, {
+          model,
+          system: "You are a precise evaluator. Respond with valid JSON only."
+        });
+        const parsed = JSON.parse(response.text);
+        if (typeof parsed.score !== "number") return 0;
+        return Math.max(0, Math.min(1, parsed.score));
+      } catch (e) {
+        return 0;
+      }
+    }
+  };
+}
+
+// src/eval/agent-test.ts
+async function testAgent(agent, cases) {
+  const results = [];
+  for (const testCase of cases) {
+    const caseResult = await runAgentTestCase(agent, testCase);
+    results.push(caseResult);
+  }
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.filter((r) => !r.passed).length;
+  return {
+    agent: agent.name,
+    cases: results,
+    summary: { passed, failed, total: results.length }
+  };
+}
+async function runAgentTestCase(agent, testCase) {
+  const failures = [];
+  let text = "";
+  let toolsCalled = [];
+  let steps = 0;
+  let cost = 0;
+  let latencyMs = 0;
+  try {
+    const result = await agent.run(testCase.input);
+    text = result.text;
+    toolsCalled = result.toolCalls.map((tc) => tc.name);
+    steps = result.steps.length;
+    cost = result.cost;
+    latencyMs = result.latencyMs;
+    if (testCase.expectContains) {
+      for (const expected of testCase.expectContains) {
+        if (!text.includes(expected)) {
+          failures.push(`Expected output to contain "${expected}"`);
+        }
+      }
+    }
+    if (testCase.expectNotContains) {
+      for (const notExpected of testCase.expectNotContains) {
+        if (text.includes(notExpected)) {
+          failures.push(`Expected output to NOT contain "${notExpected}"`);
+        }
+      }
+    }
+    if (testCase.expectToolCalled) {
+      if (!toolsCalled.includes(testCase.expectToolCalled)) {
+        failures.push(`Expected tool "${testCase.expectToolCalled}" to be called but it was not`);
+      }
+    }
+    if (testCase.expectToolsCalled) {
+      for (const tool of testCase.expectToolsCalled) {
+        if (!toolsCalled.includes(tool)) {
+          failures.push(`Expected tool "${tool}" to be called but it was not`);
+        }
+      }
+    }
+    if (testCase.expectMaxSteps !== void 0) {
+      if (steps > testCase.expectMaxSteps) {
+        failures.push(`Expected at most ${testCase.expectMaxSteps} steps but agent took ${steps}`);
+      }
+    }
+    if (testCase.expectMatch) {
+      if (!testCase.expectMatch.test(text)) {
+        failures.push(`Expected output to match ${testCase.expectMatch} but it did not`);
+      }
+    }
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    failures.push(`Agent execution failed: ${message}`);
+  }
+  return {
+    name: testCase.name,
+    passed: failures.length === 0,
+    text,
+    toolsCalled,
+    steps,
+    cost,
+    latencyMs,
+    failures
+  };
+}
+
+
+
+
+
+
+
+
+exports.contains = contains; exports.custom = custom; exports.evalSuite = evalSuite; exports.exactMatch = exactMatch; exports.llmJudge = llmJudge; exports.regex = regex; exports.testAgent = testAgent;

package/dist/eval/index.d.cts

@@ -0,0 +1,106 @@
+import { g as TokenUsage, a as AiFunction } from '../types-BJReASS-.cjs';
+import { a as Agent } from '../types-CguX3F16.cjs';
+import '../errors-zAPbTlpe.cjs';
+
+interface Scorer {
+    name: string;
+    score(output: string, expected: string | undefined, input: string): number | Promise<number>;
+}
+interface EvalCaseInput {
+    input: string;
+    expected?: string;
+    system?: string;
+    metadata?: Record<string, unknown>;
+}
+interface EvalCase extends EvalCaseInput {
+    name: string;
+}
+interface EvalCaseResult {
+    case: string;
+    model: string;
+    output: string;
+    scores: Record<string, number>;
+    tokens: TokenUsage;
+    cost: number;
+    latencyMs: number;
+    pass: boolean;
+}
+interface ModelEvalSummary {
+    avgScore: number;
+    passRate: number;
+    totalCost: number;
+    totalTokens: TokenUsage;
+    avgLatencyMs: number;
+}
+interface EvalResults {
+    suite: string;
+    timestamp: Date;
+    cases: EvalCaseResult[];
+    summary: {
+        byModel: Record<string, ModelEvalSummary>;
+        byScorer: Record<string, {
+            avgScore: number;
+        }>;
+        overall: {
+            avgScore: number;
+            passRate: number;
+            totalCost: number;
+        };
+    };
+}
+interface EvalSuiteBuilder {
+    case(name: string, testCase: EvalCaseInput): EvalSuiteBuilder;
+    scorer(scorer: Scorer): EvalSuiteBuilder;
+    models(models: string[]): EvalSuiteBuilder;
+    threshold(value: number): EvalSuiteBuilder;
+    concurrency(limit: number): EvalSuiteBuilder;
+    ai(aiFn: AiFunction): EvalSuiteBuilder;
+    run(): Promise<EvalResults>;
+}
+interface LlmJudgeOptions {
+    model?: string;
+    criteria?: string;
+    ai?: AiFunction;
+}
+interface AgentTestCase {
+    name: string;
+    input: string;
+    expectContains?: string[];
+    expectNotContains?: string[];
+    expectToolCalled?: string;
+    expectToolsCalled?: string[];
+    expectMaxSteps?: number;
+    expectMatch?: RegExp;
+}
+interface AgentTestCaseResult {
+    name: string;
+    passed: boolean;
+    text: string;
+    toolsCalled: string[];
+    steps: number;
+    cost: number;
+    latencyMs: number;
+    failures: string[];
+}
+interface AgentTestResults {
+    agent: string;
+    cases: AgentTestCaseResult[];
+    summary: {
+        passed: number;
+        failed: number;
+        total: number;
+    };
+}
+
+declare function evalSuite(name: string): EvalSuiteBuilder;
+
+declare function exactMatch(): Scorer;
+declare function contains(substring: string): Scorer;
+declare function regex(pattern: RegExp): Scorer;
+declare function custom(fn: (output: string, expected: string | undefined, input: string) => number | Promise<number>, name?: string): Scorer;
+
+declare function llmJudge(name: string, options?: LlmJudgeOptions): Scorer;
+
+declare function testAgent(agent: Agent, cases: AgentTestCase[]): Promise<AgentTestResults>;
+
+export { type AgentTestCase, type AgentTestCaseResult, type AgentTestResults, type EvalCase, type EvalCaseInput, type EvalCaseResult, type EvalResults, type EvalSuiteBuilder, type LlmJudgeOptions, type ModelEvalSummary, type Scorer, contains, custom, evalSuite, exactMatch, llmJudge, regex, testAgent };

package/dist/eval/index.d.ts

@@ -0,0 +1,106 @@
+import { g as TokenUsage, a as AiFunction } from '../types-BJReASS-.js';
+import { a as Agent } from '../types-DvdzPmW0.js';
+import '../errors-CUVTnseb.js';
+
+interface Scorer {
+    name: string;
+    score(output: string, expected: string | undefined, input: string): number | Promise<number>;
+}
+interface EvalCaseInput {
+    input: string;
+    expected?: string;
+    system?: string;
+    metadata?: Record<string, unknown>;
+}
+interface EvalCase extends EvalCaseInput {
+    name: string;
+}
+interface EvalCaseResult {
+    case: string;
+    model: string;
+    output: string;
+    scores: Record<string, number>;
+    tokens: TokenUsage;
+    cost: number;
+    latencyMs: number;
+    pass: boolean;
+}
+interface ModelEvalSummary {
+    avgScore: number;
+    passRate: number;
+    totalCost: number;
+    totalTokens: TokenUsage;
+    avgLatencyMs: number;
+}
+interface EvalResults {
+    suite: string;
+    timestamp: Date;
+    cases: EvalCaseResult[];
+    summary: {
+        byModel: Record<string, ModelEvalSummary>;
+        byScorer: Record<string, {
+            avgScore: number;
+        }>;
+        overall: {
+            avgScore: number;
+            passRate: number;
+            totalCost: number;
+        };
+    };
+}
+interface EvalSuiteBuilder {
+    case(name: string, testCase: EvalCaseInput): EvalSuiteBuilder;
+    scorer(scorer: Scorer): EvalSuiteBuilder;
+    models(models: string[]): EvalSuiteBuilder;
+    threshold(value: number): EvalSuiteBuilder;
+    concurrency(limit: number): EvalSuiteBuilder;
+    ai(aiFn: AiFunction): EvalSuiteBuilder;
+    run(): Promise<EvalResults>;
+}
+interface LlmJudgeOptions {
+    model?: string;
+    criteria?: string;
+    ai?: AiFunction;
+}
+interface AgentTestCase {
+    name: string;
+    input: string;
+    expectContains?: string[];
+    expectNotContains?: string[];
+    expectToolCalled?: string;
+    expectToolsCalled?: string[];
+    expectMaxSteps?: number;
+    expectMatch?: RegExp;
+}
+interface AgentTestCaseResult {
+    name: string;
+    passed: boolean;
+    text: string;
+    toolsCalled: string[];
+    steps: number;
+    cost: number;
+    latencyMs: number;
+    failures: string[];
+}
+interface AgentTestResults {
+    agent: string;
+    cases: AgentTestCaseResult[];
+    summary: {
+        passed: number;
+        failed: number;
+        total: number;
+    };
+}
+
+declare function evalSuite(name: string): EvalSuiteBuilder;
+
+declare function exactMatch(): Scorer;
+declare function contains(substring: string): Scorer;
+declare function regex(pattern: RegExp): Scorer;
+declare function custom(fn: (output: string, expected: string | undefined, input: string) => number | Promise<number>, name?: string): Scorer;
+
+declare function llmJudge(name: string, options?: LlmJudgeOptions): Scorer;
+
+declare function testAgent(agent: Agent, cases: AgentTestCase[]): Promise<AgentTestResults>;
+
+export { type AgentTestCase, type AgentTestCaseResult, type AgentTestResults, type EvalCase, type EvalCaseInput, type EvalCaseResult, type EvalResults, type EvalSuiteBuilder, type LlmJudgeOptions, type ModelEvalSummary, type Scorer, contains, custom, evalSuite, exactMatch, llmJudge, regex, testAgent };