@heilgar/pest-core 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pest contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/chunk-T7HWWH52.js

@@ -0,0 +1,49 @@
+// src/send.ts
+var sendHooks = [];
+function onSend(hook) {
+  sendHooks.push(hook);
+  return () => {
+    const idx = sendHooks.indexOf(hook);
+    if (idx >= 0) sendHooks.splice(idx, 1);
+  };
+}
+async function send(provider, message, options) {
+  const start = performance.now();
+  const response = await provider.call({
+    systemPrompt: options?.systemPrompt,
+    messages: [{ role: "user", content: message }],
+    tools: options?.tools,
+    temperature: options?.temperature,
+    maxTokens: options?.maxTokens,
+    responseFormat: options?.responseFormat
+  });
+  const latencyMs = performance.now() - start;
+  const result = {
+    ...response,
+    latencyMs,
+    provider: provider.name,
+    model: provider.model
+  };
+  if (sendHooks.length > 0) {
+    const entry = {
+      input: message,
+      output: response.text,
+      systemPrompt: options?.systemPrompt,
+      provider: provider.name,
+      model: provider.model,
+      latencyMs,
+      usage: { ...response.usage },
+      toolCalls: response.toolCalls,
+      timestamp: Date.now()
+    };
+    for (const hook of sendHooks) {
+      hook(entry);
+    }
+  }
+  return result;
+}
+
+export {
+  onSend,
+  send
+};

package/dist/index.d.ts

@@ -0,0 +1,467 @@
+import * as v from 'valibot';
+import { z } from 'zod';
+
+declare const ProviderConfigSchema: v.ObjectSchema<{
+    readonly name: v.StringSchema<undefined>;
+    readonly type: v.PicklistSchema<["openai", "anthropic", "gemini", "xai", "ollama"], undefined>;
+    readonly model: v.StringSchema<undefined>;
+    readonly apiKey: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+    readonly baseUrl: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+    readonly temperature: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
+}, undefined>;
+declare const PestConfigSchema: v.ObjectSchema<{
+    readonly providers: v.SchemaWithPipe<readonly [v.ArraySchema<v.ObjectSchema<{
+        readonly name: v.StringSchema<undefined>;
+        readonly type: v.PicklistSchema<["openai", "anthropic", "gemini", "xai", "ollama"], undefined>;
+        readonly model: v.StringSchema<undefined>;
+        readonly apiKey: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+        readonly baseUrl: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+        readonly temperature: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
+    }, undefined>, undefined>, v.MinLengthAction<{
+        name: string;
+        type: "openai" | "anthropic" | "gemini" | "xai" | "ollama";
+        model: string;
+        apiKey?: string | undefined;
+        baseUrl?: string | undefined;
+        temperature?: number | undefined;
+    }[], 1, undefined>]>;
+    readonly judge: v.OptionalSchema<v.ObjectSchema<{
+        readonly provider: v.StringSchema<undefined>;
+    }, undefined>, undefined>;
+    readonly pricing: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.ObjectSchema<{
+        readonly inputCentsPer1M: v.NumberSchema<undefined>;
+        readonly outputCentsPer1M: v.NumberSchema<undefined>;
+    }, undefined>, undefined>, undefined>;
+}, undefined>;
+type PestConfig = v.InferOutput<typeof PestConfigSchema>;
+type ProviderConfig = v.InferOutput<typeof ProviderConfigSchema>;
+
+/** Reset env-loaded flag so loadEnv() can be called again. For testing only. */
+declare function resetEnv(): void;
+/**
+ * Load environment variables from .env files into process.env.
+ *
+ * Files loaded (lowest to highest priority):
+ * - `.env` — shared defaults, may be committed
+ * - `.env.local` — local overrides, should be gitignored
+ *
+ * Real environment variables are never overwritten.
+ * Files are loaded from the project root (where pest.config.ts or package.json is).
+ *
+ * This is called automatically by `loadConfig()` and `createProvider()`.
+ * Safe to call multiple times — only loads once.
+ */
+declare function loadEnv(cwd?: string): void;
+declare function loadConfig(cwd?: string): Promise<PestConfig>;
+declare function defineConfig(config: PestConfig): PestConfig;
+
+interface JudgeResult {
+    pass: boolean;
+    score: number;
+    reasoning: string;
+}
+
+interface ToolDefinition {
+    type: 'function';
+    function: {
+        name: string;
+        description?: string;
+        parameters?: Record<string, unknown>;
+    };
+}
+interface ToolCall {
+    name: string;
+    args: Record<string, unknown>;
+    /** Provider-assigned ID — required for multi-turn tool result routing */
+    id?: string;
+}
+type AgenticMessage = {
+    role: 'user';
+    content: string;
+} | {
+    role: 'assistant';
+    content: string;
+    toolCalls?: ToolCall[];
+} | {
+    role: 'tool';
+    toolCallName: string;
+    toolCallId: string;
+    content: string;
+};
+type ToolExecutor = (name: string, args: Record<string, unknown>) => Promise<unknown> | unknown;
+interface SendAgenticOptions extends SendOptions {
+    /** Called for each tool invocation. Defaults to returning '[]' (no-op mock). */
+    executor?: ToolExecutor;
+    /** Max tool-calling iterations before stopping. Default: 10 */
+    maxSteps?: number;
+}
+interface ProviderUsage {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+}
+interface ProviderResponse {
+    text: string;
+    toolCalls: ToolCall[];
+    usage: ProviderUsage;
+    raw: unknown;
+}
+interface ProviderRequestOptions {
+    systemPrompt?: string;
+    messages: AgenticMessage[];
+    tools?: ToolDefinition[];
+    temperature?: number;
+    maxTokens?: number;
+    responseFormat?: 'text' | 'json';
+}
+interface Provider {
+    name: string;
+    model: string;
+    call(options: ProviderRequestOptions): Promise<ProviderResponse>;
+}
+interface PestResponse extends ProviderResponse {
+    latencyMs: number;
+    provider: string;
+    model: string;
+}
+interface SendOptions {
+    systemPrompt?: string;
+    tools?: ToolDefinition[];
+    temperature?: number;
+    maxTokens?: number;
+    responseFormat?: 'text' | 'json';
+}
+
+/**
+ * Resolve the active provider from PEST_PROVIDER env + pest.config.ts.
+ *
+ * Used in test files so the CLI can switch providers via env:
+ * - CLI sets PEST_PROVIDER=gpt4o, then shells out to vitest
+ * - Test file calls useProvider() → gets the gpt4o provider
+ * - When running vitest directly (no CLI), falls back to first provider in config
+ */
+declare function useProvider(fallbackName?: string): Promise<Provider>;
+/**
+ * Resolve system prompt with PEST_SYSTEM_PROMPT env override.
+ *
+ * Used in test files so the CLI tune command can swap prompts via env:
+ * - CLI sets PEST_SYSTEM_PROMPT=<variant>, then shells out to vitest
+ * - Test file calls useSystemPrompt("default...") → gets the variant
+ * - When running vitest directly, returns the default
+ */
+declare function useSystemPrompt(defaultPrompt: string): string;
+
+declare function setJudge(provider: Provider): void;
+declare function getJudge(): Provider | null;
+declare function resolveJudge(options?: {
+    judge?: Provider;
+}): Provider;
+
+interface MatcherResult {
+    pass: boolean;
+    message: string;
+    score?: number;
+    reasoning?: string;
+    metadata?: Record<string, unknown>;
+}
+interface SemanticOptions {
+    threshold?: number;
+    judge?: Provider;
+}
+interface ClassificationOptions {
+    categories?: string[];
+    judge?: Provider;
+}
+interface RubricConfig {
+    criteria: string;
+    scoreRange?: [number, number];
+    passThreshold?: number;
+}
+declare function containsToolCall(response: PestResponse, name: string, args?: Record<string, unknown>): MatcherResult;
+declare function callsToolsInOrder(response: PestResponse, names: string[]): MatcherResult;
+declare function matchesResponseSchema(response: PestResponse, schema: {
+    _run?: unknown;
+    '~standard'?: {
+        validate: (input: unknown) => {
+            issues?: unknown[];
+        };
+    };
+}): MatcherResult;
+declare function respondsWithinTokens(response: PestResponse, maxTokens: number): MatcherResult;
+declare function containsText(response: PestResponse, text: string): MatcherResult;
+declare function hasToolCallCount(response: PestResponse, count: number): MatcherResult;
+declare function matchesSemanticMeaning(response: PestResponse, expected: string, judge: Provider, options?: SemanticOptions): Promise<MatcherResult>;
+declare function satisfiesCriteria(response: PestResponse, rubric: string | RubricConfig, judge: Provider): Promise<MatcherResult>;
+declare function classifiedAs(response: PestResponse, label: string, judge: Provider, options?: ClassificationOptions): Promise<MatcherResult>;
+declare function doesNotDisclose(response: PestResponse, topic: string, judge: Provider): Promise<MatcherResult>;
+declare function assertConsistent(provider: Provider, message: string, runs: number, options?: {
+    threshold?: number;
+    judge?: Provider;
+    systemPrompt?: string;
+    tools?: ToolDefinition[];
+}): Promise<{
+    pass: boolean;
+    scores: number[];
+    reasoning: string[];
+}>;
+
+interface ModelPricing {
+    inputCentsPer1M: number;
+    outputCentsPer1M: number;
+}
+declare function setPricing(pricing: Record<string, ModelPricing>): void;
+declare function resetPricing(): void;
+declare function getPricing(model: string): ModelPricing;
+declare function estimateCostCents(model: string, inputTokens: number, outputTokens: number): number;
+
+declare function createProvider(config: ProviderConfig): Provider;
+declare function createProviders(configs: ProviderConfig[]): Map<string, Provider>;
+
+interface SendEntry {
+    input: string;
+    output: string;
+    systemPrompt?: string;
+    provider: string;
+    model: string;
+    latencyMs: number;
+    usage: ProviderUsage;
+    toolCalls: ToolCall[];
+    timestamp: number;
+}
+interface MatcherEntry {
+    matcher: string;
+    pass: boolean;
+    score?: number;
+    reasoning?: string;
+    response?: {
+        provider: string;
+        model: string;
+        latencyMs: number;
+        usage: ProviderUsage;
+        toolCalls: ToolCall[];
+        text: string;
+    };
+    judgeModel?: string;
+}
+interface TestData {
+    testId: string;
+    testName?: string;
+    startTime: number;
+    endTime?: number;
+    sends: SendEntry[];
+    entries: MatcherEntry[];
+}
+declare function startTest(testId: string, testName?: string): void;
+declare function endTest(testId: string): void;
+declare function recordSend(testId: string, entry: SendEntry): void;
+declare function record(testId: string, entry: MatcherEntry): void;
+declare function getTestData(testId: string): TestData | undefined;
+declare function getAllTestData(): Map<string, TestData>;
+declare function clearAll(): void;
+declare function buildMatcherEntry(matcher: string, result: MatcherResult, response?: {
+    provider: string;
+    model: string;
+    latencyMs: number;
+    usage: ProviderUsage;
+    toolCalls: ToolCall[];
+    text: string;
+}, judgeModel?: string): MatcherEntry;
+
+type SendHook = (entry: SendEntry) => void;
+/**
+ * Register a hook that gets called after every send() call.
+ * Used by extensions to capture LLM input/output for reporters.
+ * Returns a dispose function to remove the hook.
+ */
+declare function onSend(hook: SendHook): () => void;
+declare function send(provider: Provider, message: string, options?: SendOptions): Promise<PestResponse>;
+
+/**
+ * Run a full agentic tool-call loop and return a single PestResponse with all
+ * accumulated tool calls across all steps.
+ *
+ * This lets you use toContainToolCall, toCallToolsInOrder, toHaveToolCallCount
+ * on multi-turn tool-calling conversations.
+ *
+ * @example
+ * const res = await sendAgentic(provider, 'List then create a segment', {
+ *   systemPrompt,
+ *   tools,
+ *   executor: async (name, args) => myApp.handleTool(name, args),
+ * });
+ * expect(res).toCallToolsInOrder(['list_segments', 'create_segment']);
+ */
+declare function sendAgentic(provider: Provider, message: string, options?: SendAgenticOptions): Promise<PestResponse>;
+
+/**
+ * Convert a Zod schema to a pest ToolDefinition.
+ *
+ * Eliminates the need to duplicate tool definitions in JSON Schema by hand.
+ * Requires zod >= 3.0.0 as a peer dependency.
+ *
+ * @example
+ * const tools = Object.values(TOOL_DEFINITIONS).map(def =>
+ *   zodTool(def.name, def.description, def.parameters)
+ * );
+ */
+declare function zodTool(name: string, description: string, schema: z.ZodTypeAny): ToolDefinition;
+
+declare const ansi: {
+    reset: string;
+    bold: string;
+    dim: string;
+    green: string;
+    red: string;
+    yellow: string;
+    cyan: string;
+    magenta: string;
+};
+declare function formatTokens(n: number): string;
+declare function formatDuration(ms: number): string;
+declare function formatCost(cents: number): string;
+declare function escapeHtml(str: string): string;
+
+type TestIdResolver = () => string | undefined;
+/**
+ * Create pest matchers bound to a test-id resolver.
+ * Shared by vitest and jest extensions to avoid code duplication.
+ */
+declare function createPestMatchers(getTestId: TestIdResolver): {
+    toContainToolCall(received: PestResponse, name: string, args?: Record<string, unknown>): {
+        pass: boolean;
+        message: () => string;
+    };
+    toCallToolsInOrder(received: PestResponse, names: string[]): {
+        pass: boolean;
+        message: () => string;
+    };
+    toMatchResponseSchema(received: PestResponse, schema: unknown): {
+        pass: boolean;
+        message: () => string;
+    };
+    toRespondWithinTokens(received: PestResponse, maxTokens: number): {
+        pass: boolean;
+        message: () => string;
+    };
+    toContainText(received: PestResponse, text: string): {
+        pass: boolean;
+        message: () => string;
+    };
+    toNotContainText(received: PestResponse, text: string): {
+        pass: boolean;
+        message: () => string;
+    };
+    toHaveToolCallCount(received: PestResponse, count: number): {
+        pass: boolean;
+        message: () => string;
+    };
+    toMatchSemanticMeaning(received: PestResponse, expected: string, options?: SemanticOptions): Promise<{
+        pass: boolean;
+        message: () => string;
+    }>;
+    toSatisfyCriteria(received: PestResponse, rubric: string | RubricConfig, options?: {
+        judge?: Provider;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+    }>;
+    toBeClassifiedAs(received: PestResponse, label: string, options?: ClassificationOptions): Promise<{
+        pass: boolean;
+        message: () => string;
+    }>;
+    toNotDisclose(received: PestResponse, topic: string, options?: {
+        judge?: Provider;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+    }>;
+};
+
+interface ReporterOptions {
+    verbose?: boolean;
+    showCost?: boolean;
+    logFile?: string | false;
+    htmlFile?: string | false;
+}
+interface RunStats {
+    tests: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalLatencyMs: number;
+    totalCostCents: number;
+    judgeCount: number;
+    toolCallCount: number;
+}
+interface LogEntry {
+    test: string;
+    status: string;
+    sends: SendEntry[];
+    matchers: Array<{
+        matcher: string;
+        pass: boolean;
+        score?: number;
+        reasoning?: string;
+        judgeModel?: string;
+    }>;
+}
+interface TestResult {
+    name: string;
+    status: 'passed' | 'failed' | string;
+    sends: SendEntry[];
+    entries: Array<{
+        matcher: string;
+        pass: boolean;
+        score?: number;
+        reasoning?: string;
+        judgeModel?: string;
+        response?: {
+            provider: string;
+            model: string;
+            latencyMs: number;
+            usage: {
+                inputTokens: number;
+                outputTokens: number;
+            };
+            toolCalls: Array<{
+                name: string;
+            }>;
+            text: string;
+        };
+    }>;
+}
+declare function emptyStats(): RunStats;
+/** Process a single test result and update stats. Returns a LogEntry. */
+declare function processTestResult(testResult: TestResult, stats: RunStats, options: ReporterOptions): LogEntry;
+/** Print summary and write log/HTML files. */
+declare function finishReport(stats: RunStats, logEntries: LogEntry[], options: ReporterOptions): void;
+
+interface ReportMatcher {
+    matcher: string;
+    pass: boolean;
+    score?: number;
+    reasoning?: string;
+    judgeModel?: string;
+}
+interface ReportTest {
+    test: string;
+    status: string;
+    sends: SendEntry[];
+    matchers: ReportMatcher[];
+}
+interface ReportSummary {
+    tests: number;
+    totalTokens: number;
+    inputTokens: number;
+    outputTokens: number;
+    estimatedCost: string;
+    judgeCount: number;
+    toolCallCount: number;
+}
+interface ReportData {
+    timestamp: string;
+    summary: ReportSummary;
+    tests: ReportTest[];
+}
+declare function buildHtmlReport(data: ReportData): string;
+
+export { type AgenticMessage, type ClassificationOptions, type JudgeResult, type LogEntry, type MatcherEntry, type MatcherResult, type ModelPricing, type PestConfig, type PestResponse, type Provider, type ProviderConfig, type ProviderRequestOptions, type ProviderResponse, type ProviderUsage, type ReporterOptions, type RubricConfig, type RunStats, type SemanticOptions, type SendAgenticOptions, type SendEntry, type SendOptions, type TestData, type TestResult, type ToolCall, type ToolDefinition, type ToolExecutor, ansi, assertConsistent, buildHtmlReport, buildMatcherEntry, callsToolsInOrder, classifiedAs, clearAll, containsText, containsToolCall, createPestMatchers, createProvider, createProviders, defineConfig, doesNotDisclose, emptyStats, endTest, escapeHtml, estimateCostCents, finishReport, formatCost, formatDuration, formatTokens, getAllTestData, getJudge, getPricing, getTestData, hasToolCallCount, loadConfig, loadEnv, matchesResponseSchema, matchesSemanticMeaning, onSend, processTestResult, record, recordSend, resetEnv, resetPricing, resolveJudge, respondsWithinTokens, satisfiesCriteria, send, sendAgentic, setJudge, setPricing, startTest, useProvider, useSystemPrompt, zodTool };