@taodigital/agents 0.0.0-stage → 1.2.0

package/dist/llm-mock.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * Mock / record / replay layer for the LLM choke points in llm.ts.
+ *
+ * Purpose: let agent flows (routing, dedup, persistence, token accounting) run
+ * deterministically in CI without LLM spend. See docs/120-AGENT-E2E-HARNESS-PLAN.md.
+ *
+ * Env flags
+ *   ZETA_LLM_MOCK        '1' | 'replay'  → replay fixtures (fallback if no fixture)
+ *                        'record'        → replay when a fixture exists, otherwise call
+ *                                          the real provider and save the response
+ *   ZETA_LLM_MOCK_DIR    fixture directory (default: <cwd>/eval/fixtures/llm)
+ *   ZETA_LLM_MOCK_FAIL   '429' | '503' | 'malformed-json' | 'empty'  → failure injection
+ *
+ * This module intentionally does NOT import llm.ts (avoids a circular import);
+ * llm.ts turns a `{ kind: 'error' }` outcome into a sanitized LLMProviderError.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resetLlmStats = resetLlmStats;
+exports.snapshotLlmStats = snapshotLlmStats;
+exports.recordLlmCall = recordLlmCall;
+exports.getMockMode = getMockMode;
+exports.fixtureDir = fixtureDir;
+exports.agentId = agentId;
+exports.mockKey = mockKey;
+exports.loadFixture = loadFixture;
+exports.saveFixture = saveFixture;
+exports.resolveMock = resolveMock;
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+let stats = { calls: 0, promptTokens: 0, completionTokens: 0, estimatedCostUsd: 0, mockKeys: {} };
+function resetLlmStats() {
+    stats = { calls: 0, promptTokens: 0, completionTokens: 0, estimatedCostUsd: 0, mockKeys: {} };
+}
+function snapshotLlmStats() {
+    return { ...stats, mockKeys: { ...stats.mockKeys } };
+}
+function recordLlmCall(usage, key, source) {
+    stats.calls += 1;
+    stats.promptTokens += usage?.promptTokens ?? 0;
+    stats.completionTokens += usage?.completionTokens ?? 0;
+    stats.estimatedCostUsd += usage?.estimatedCostUsd ?? 0;
+    if (key)
+        stats.mockKeys[key] = source ?? 'real';
+}
+let bannerShown = false;
+function getMockMode() {
+    const raw = (process.env.ZETA_LLM_MOCK ?? '').trim().toLowerCase();
+    if (!raw || raw === '0' || raw === 'false' || raw === 'off')
+        return 'off';
+    if (process.env.NODE_ENV === 'production') {
+        throw new Error('[llm-mock] ZETA_LLM_MOCK is not allowed when NODE_ENV=production.');
+    }
+    if (!bannerShown) {
+        bannerShown = true;
+        console.warn(`[llm-mock] ⚠ LLM MOCK ACTIVE (mode=${raw === 'record' ? 'record' : 'replay'}) — no real provider calls in replay mode.`);
+    }
+    return raw === 'record' ? 'record' : 'replay';
+}
+function fixtureDir() {
+    return process.env.ZETA_LLM_MOCK_DIR ?? (0, node_path_1.join)(process.cwd(), 'eval', 'fixtures', 'llm');
+}
+function lastSystemText(system) {
+    if (typeof system === 'string')
+        return system;
+    return system.length ? system[system.length - 1].text : '';
+}
+function slug(s, max = 40) {
+    return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, max) || 'x';
+}
+/**
+ * Stable agent/pass identifier derived from the pass-specific (last) system block.
+ * e.g. "…ZeTa Test Standard v2.0 Pass 3: Security…" → "pass-3-security"
+ * Falls back to a short hash of the block's first line so unknown agents still get a key.
+ */
+function agentId(system) {
+    const text = lastSystemText(system);
+    const m = text.match(/\bPass\s+(\d+)\s*[:—-]?\s*([A-Za-z0-9/ ()&-]{0,30})/);
+    if (m)
+        return `pass-${m[1]}-${slug(m[2], 24)}`.replace(/-$/, '');
+    const firstLine = text.split('\n').find((l) => l.trim()) ?? '';
+    return `agent-${slug(firstLine, 24)}-${shortHash(firstLine)}`;
+}
+function shortHash(s) {
+    return (0, node_crypto_1.createHash)('sha1').update(s).digest('hex').slice(0, 8);
+}
+/**
+ * Key = <agent/pass id>__<screen slug or head-of-prompt hash>.
+ * Keys deliberately avoid hashing the full prompt so wording tweaks to prompts
+ * don't invalidate every fixture; only a different agent or screen changes the key.
+ */
+function mockKey(system, user, kind = 'text') {
+    const screenLine = user.match(/^Screen:\s*(.+?)\s*(?:\(|$)/m)?.[1];
+    const scope = screenLine ? slug(screenLine, 40) : `h-${shortHash(user.slice(0, 200))}`;
+    return `${kind === 'vision' ? 'vision-' : ''}${agentId(system)}__${scope}`;
+}
+function estimateTokens(s) {
+    return Math.max(1, Math.ceil(s.length / 4));
+}
+function buildUsage(system, user, text, partial) {
+    const sysText = typeof system === 'string' ? system : system.map((b) => b.text).join('\n');
+    const promptTokens = partial?.promptTokens ?? estimateTokens(sysText + user);
+    const completionTokens = partial?.completionTokens ?? estimateTokens(text);
+    return {
+        provider: partial?.provider ?? 'mock',
+        model: partial?.model ?? 'mock-llm',
+        promptTokens,
+        completionTokens,
+        cacheReadTokens: partial?.cacheReadTokens ?? 0,
+        cacheWriteTokens: partial?.cacheWriteTokens ?? 0,
+        totalTokens: promptTokens + completionTokens,
+        estimatedCostUsd: 0,
+    };
+}
+/** Minimal schema-valid output so unknown prompts never crash a flow. */
+function fallbackText(system, user, schema) {
+    if (schema)
+        return '{}';
+    const sys = typeof system === 'string' ? system : system.map((b) => b.text).join('\n');
+    if (/JSON array/i.test(sys)) {
+        const id = agentId(system);
+        return JSON.stringify([
+            {
+                title: `Mock fallback — ${id} — produces a valid case`,
+                kind: 'FUNCTIONAL',
+                priority: 'MEDIUM',
+                techniqueOfRecord: 'equivalence-partition',
+                preconditions: 'Application is reachable.',
+                expectedResult: 'The screen responds with the documented result.',
+                steps: ['Navigate to the screen under test', 'Observe the rendered page'],
+                groundedOn: [],
+                gherkin: 'Given the screen is open\nWhen the user views it\nThen it renders',
+                tags: ['mock-fallback'],
+            },
+        ]);
+    }
+    if (/JSON/i.test(sys))
+        return '{}';
+    return 'MOCK RESPONSE';
+}
+function injectedFailure(system, user) {
+    const mode = (process.env.ZETA_LLM_MOCK_FAIL ?? '').trim().toLowerCase();
+    if (!mode)
+        return null;
+    if (mode === '429')
+        return { kind: 'error', status: 429 };
+    if (mode === '503')
+        return { kind: 'error', status: 503 };
+    if (mode === 'malformed-json') {
+        const text = '[{"title": "truncated case", "kind": "FUNCTIONAL", "steps": ["a", ';
+        return { kind: 'result', source: 'injected', res: { text, usage: buildUsage(system, user, text) } };
+    }
+    if (mode === 'empty') {
+        return { kind: 'result', source: 'injected', res: { text: '', usage: buildUsage(system, user, '') } };
+    }
+    return null;
+}
+function fixturePath(key) {
+    return (0, node_path_1.join)(fixtureDir(), `${key}.json`);
+}
+function loadFixture(key) {
+    const p = fixturePath(key);
+    if (!(0, node_fs_1.existsSync)(p))
+        return null;
+    try {
+        return JSON.parse((0, node_fs_1.readFileSync)(p, 'utf8'));
+    }
+    catch (e) {
+        throw new Error(`[llm-mock] fixture ${p} is not valid JSON: ${e instanceof Error ? e.message : e}`);
+    }
+}
+function saveFixture(key, res) {
+    (0, node_fs_1.mkdirSync)(fixtureDir(), { recursive: true });
+    const body = { key, text: res.text, usage: res.usage, recordedAt: new Date().toISOString() };
+    (0, node_fs_1.writeFileSync)(fixturePath(key), JSON.stringify(body, null, 2) + '\n');
+}
+/** Decide how to answer a call while mocking is on. Never performs a network call itself. */
+function resolveMock(mode, system, user, opts = {}) {
+    const injected = injectedFailure(system, user);
+    if (injected)
+        return injected;
+    const key = mockKey(system, user, opts.kind);
+    const fixture = loadFixture(key);
+    if (fixture) {
+        return { kind: 'result', source: 'fixture', res: { text: fixture.text, usage: buildUsage(system, user, fixture.text, fixture.usage) } };
+    }
+    if (mode === 'record')
+        return { kind: 'miss' };
+    const text = fallbackText(system, user, opts.schema);
+    return { kind: 'result', source: 'fallback', res: { text, usage: buildUsage(system, user, text) } };
+}

package/dist/llm.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Multi-provider LLM module.
+ *
+ * Providers using their own SDK: Anthropic.
+ * Providers via OpenAI-compatible API (single openai package): OpenAI, Gemini,
+ * Grok (xAI), OpenRouter, Ollama (local), Nvidia NIM.
+ *
+ * Config is resolved per-tenant only (set in Workspace → Settings → AI Provider).
+ * No platform-level API key fallback — every workspace must configure its own provider.
+ */
+import type { LLMUsage } from '@taodigital/core';
+export type LLMProvider = 'anthropic' | 'openai' | 'gemini' | 'grok' | 'openrouter' | 'ollama' | 'nvidia' | 'groq' | 'mistral' | 'deepseek' | 'perplexity' | 'togetherai' | 'bedrock' | 'vertex' | 'azure';
+export interface LLMConfig {
+    provider: LLMProvider;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+export interface AskOptions {
+    temperature?: number;
+    topP?: number;
+    seed?: number;
+}
+export declare function deterministicSeed(input: string): number;
+export declare const DEFAULT_MODELS: Record<LLMProvider, string>;
+export interface ImageInput {
+    base64: string;
+    mediaType: string;
+}
+export interface AskResult {
+    text: string;
+    usage: LLMUsage;
+}
+/**
+ * Thrown in place of a raw provider SDK error. Provider error messages
+ * sometimes echo back a snippet of the request/prompt content (e.g. a
+ * content-policy rejection quoting the offending text), which may include a
+ * customer's crawled screen content or other sensitive data - safe to log
+ * server-side, not safe to return verbatim to whoever's looking at the API
+ * response. sanitizeProviderError() below preserves the actionable signal
+ * (rate limited vs. bad auth vs. bad request) without the raw text.
+ */
+export declare class LLMProviderError extends Error {
+    /** True when the error is not transient and retrying will not help. */
+    readonly nonRetryable: boolean;
+    constructor(message?: string, nonRetryable?: boolean);
+}
+export declare function sanitizeProviderError(e: unknown): LLMProviderError;
+/** Mock-aware entry point (see llm-mock.ts). With ZETA_LLM_MOCK unset this is a pass-through. */
+export declare function askTrackedWithImages(system: string, user: string, images: ImageInput[], llmCfg?: LLMConfig, options?: AskOptions): Promise<AskResult>;
+export declare function askWithImages(system: string, user: string, images: ImageInput[], llmCfg?: LLMConfig): Promise<string>;
+export type SystemBlock = {
+    text: string;
+    cache?: boolean;
+};
+/** Mock-aware entry point (see llm-mock.ts). With ZETA_LLM_MOCK unset this is a pass-through. */
+export declare function askTracked(system: string | SystemBlock[], user: string, llmCfg?: LLMConfig, maxTokens?: number, schema?: any, options?: AskOptions): Promise<AskResult>;
+export declare function ask(system: string, user: string, llmCfg?: LLMConfig, maxTokens?: number): Promise<string>;