@taodigital/agents 0.0.0-stage → 1.2.1

package/dist/label-assignment-agent.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.labelTestCasesBatch = labelTestCasesBatch;
+const llm_js_1 = require("./llm.js");
+const llm_config_js_1 = require("./llm-config.js");
+const core_1 = require("@taodigital/core");
+const BATCH_SIZE = 50;
+const SYSTEM_PROMPT = `You are a QA test case classification assistant. Given a batch of test cases, assign each one:
+- priority: one of CRITICAL, HIGH, MEDIUM, LOW
+- inferredKind: one of POSITIVE, NEGATIVE, EDGE, SECURITY, ACCESSIBILITY
+- tags: relevant string tags (e.g. ["@auth","@login"])
+- confidence: a float 0–1 representing classification confidence
+
+Priority heuristic rules:
+- CRITICAL: test involves login, auth, payment, checkout, password, billing, security, or accessibility
+- HIGH: test involves register, signup, onboard, data loss, or is a negative test with more than 5 steps
+- MEDIUM: test involves filter, sort, search, export, or navigation
+- LOW: everything else
+
+Kind inference rules:
+- NEGATIVE: title or steps contain keywords: invalid, wrong, incorrect, fail, error, reject
+- SECURITY: title or steps contain keywords: security, xss, injection, csrf, permission, unauthorized
+- ACCESSIBILITY: title or steps contain keywords: accessibility, a11y, screen reader, wcag, aria
+- EDGE: title or steps contain keywords: boundary, edge, limit, maximum, minimum, empty, null
+- POSITIVE: default when none of the above apply
+
+Return ONLY a valid JSON array with no markdown, no explanation, no code fences. Each element must have exactly these keys: id, priority, inferredKind, tags, confidence.
+
+Example output format:
+[{"id":"tc-1","priority":"HIGH","inferredKind":"POSITIVE","tags":["@auth","@login"],"confidence":0.85}]`;
+function buildUserPrompt(batch) {
+    const items = batch.map((c) => ({
+        id: c.id,
+        title: c.title,
+        stepsText: c.stepsText,
+        expectedResult: c.expectedResult ?? '',
+    }));
+    return `Classify the following test cases:\n${JSON.stringify(items, null, 2)}`;
+}
+function fallbackResults(batch) {
+    return batch.map((c) => ({
+        id: c.id,
+        priority: 'HIGH',
+        inferredKind: 'POSITIVE',
+        tags: [],
+        confidence: 0.5,
+        needsReview: true,
+    }));
+}
+function parseRaw(raw, batch) {
+    let cleaned = raw.trim();
+    // Strip markdown code fences if present
+    cleaned = cleaned.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '').trim();
+    let parsed;
+    try {
+        parsed = JSON.parse(cleaned);
+    }
+    catch {
+        return fallbackResults(batch);
+    }
+    if (!Array.isArray(parsed))
+        return fallbackResults(batch);
+    const idSet = new Set(batch.map((c) => c.id));
+    const resultMap = new Map();
+    for (const item of parsed) {
+        if (!item?.id || !idSet.has(item.id))
+            continue;
+        const confidence = typeof item.confidence === 'number' ? Math.min(1, Math.max(0, item.confidence)) : 0.5;
+        resultMap.set(item.id, {
+            id: item.id,
+            priority: ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW'].includes(item.priority) ? item.priority : 'HIGH',
+            inferredKind: typeof item.inferredKind === 'string' && item.inferredKind ? item.inferredKind : 'POSITIVE',
+            tags: Array.isArray(item.tags) ? item.tags.filter((t) => typeof t === 'string') : [],
+            confidence,
+            needsReview: confidence < 0.70,
+        });
+    }
+    // Fill in any cases the LLM omitted with fallback
+    return batch.map((c) => resultMap.get(c.id) ?? { ...fallbackResults([c])[0] });
+}
+async function labelTestCasesBatch(cases, tenantId) {
+    const llmCfg = await (0, llm_config_js_1.getTaskAgentLLMConfig)(tenantId, 'testGeneration');
+    const results = [];
+    for (let i = 0; i < cases.length; i += BATCH_SIZE) {
+        const batch = cases.slice(i, i + BATCH_SIZE);
+        try {
+            const seed = (0, llm_js_1.deterministicSeed)(`${tenantId}:label-batch:${batch.map((c) => c.id).join('|')}`);
+            const effectiveSystem = await (0, core_1.getAgentPrompt)('label-assignment', undefined, tenantId) ?? SYSTEM_PROMPT;
+            const { text } = await (0, llm_js_1.askTracked)(effectiveSystem, buildUserPrompt(batch), llmCfg, 4096, undefined, { temperature: 0, seed });
+            results.push(...parseRaw(text, batch));
+        }
+        catch {
+            results.push(...fallbackResults(batch));
+        }
+    }
+    return results;
+}

package/dist/llm-config.d.ts

@@ -0,0 +1,38 @@
+import type { LLMTaskType } from '@taodigital/app-brain';
+import type { LLMConfig } from './llm.js';
+export declare function getAllowedModels(extra?: string[]): string[];
+export declare function getLLMBaseUrls(overrides?: Record<string, string>): {
+    gemini: string;
+    grok: string;
+    groq: string;
+    mistral: string;
+    deepseek: string;
+    perplexity: string;
+    togetherai: string;
+    openrouter: string;
+    nvidia: string;
+    ollama: string;
+};
+/** Load the tenant's configured LLM provider. Falls back to platform admin config, then env vars. */
+export declare function getAgentLLMConfig(tenantId: string): Promise<LLMConfig | undefined>;
+/**
+ * Returns the platform admin LLM config directly, bypassing tenant config.
+ * Use this when a tenant-configured key fails at call time (401/403) to retry with the platform key.
+ */
+export declare function getPlatformFallbackConfig(): Promise<LLMConfig | undefined>;
+/**
+ * Runs `fn(tenantCfg)` and, if the provider rejects the API key (401/403), retries with
+ * platform admin config. Prevents a single expired workspace key from blocking all agent jobs.
+ *
+ * When tenantCfg is undefined (no tenant or platform config resolved), fails fast with a
+ * clear "No AI provider configured" error rather than making a live call with empty credentials.
+ */
+export declare function tryWithPlatformFallback<T>(tenantCfg: LLMConfig | undefined, fn: (cfg: LLMConfig | undefined) => Promise<T>): Promise<T>;
+/** Load workspace domain config for a project — used to inject domain hints into prompts. */
+export declare function getWorkspaceDomainContext(tenantId: string, projectId: string): Promise<string>;
+/** Returns true if the workspace has PHI masking enabled — redact real PII field values before LLM calls. */
+export declare function getWorkspacePhiMasking(tenantId: string, projectId: string): Promise<boolean>;
+/** Returns the tenant's LLM config swapped to the provider's fast mini-model for lower-cost passes. */
+export declare function getFastPassLLMConfig(tenantId: string): Promise<LLMConfig | undefined>;
+/** Load the tenant's LLM config for a specific task, respecting per-task overrides. */
+export declare function getTaskAgentLLMConfig(tenantId: string, task: LLMTaskType): Promise<LLMConfig | undefined>;

package/dist/llm-config.js

@@ -0,0 +1,287 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAllowedModels = getAllowedModels;
+exports.getLLMBaseUrls = getLLMBaseUrls;
+exports.getAgentLLMConfig = getAgentLLMConfig;
+exports.getPlatformFallbackConfig = getPlatformFallbackConfig;
+exports.tryWithPlatformFallback = tryWithPlatformFallback;
+exports.getWorkspaceDomainContext = getWorkspaceDomainContext;
+exports.getWorkspacePhiMasking = getWorkspacePhiMasking;
+exports.getFastPassLLMConfig = getFastPassLLMConfig;
+exports.getTaskAgentLLMConfig = getTaskAgentLLMConfig;
+const app_brain_1 = require("@taodigital/app-brain");
+const core_1 = require("@taodigital/core");
+const ALLOWED_MODELS_BASE = [
+    'claude-3-5-sonnet-20241022',
+    'claude-3-5-haiku-20241022',
+    'claude-3-opus-20240229',
+    'claude-sonnet-4-6',
+    'claude-haiku-4-5-20251001',
+    'gpt-4o',
+    'gpt-4o-mini',
+    'gpt-4-turbo',
+    'gemini-2.5-flash',
+    'gemini-1.5-flash',
+];
+const ALLOWED_MODELS_EXTRA_ENV = process.env.ALLOWED_MODELS_EXTRA
+    ? process.env.ALLOWED_MODELS_EXTRA.split(',').map((m) => m.trim()).filter(Boolean)
+    : [];
+function getAllowedModels(extra) {
+    return [...ALLOWED_MODELS_BASE, ...ALLOWED_MODELS_EXTRA_ENV, ...(extra ?? [])];
+}
+const ALLOWED_MODELS = new Set([...ALLOWED_MODELS_BASE, ...ALLOWED_MODELS_EXTRA_ENV]);
+function assertAllowedModel(model) {
+    if (!ALLOWED_MODELS.has(model)) {
+        throw new Error(`Model '${model}' is not allowed. Set ALLOWED_MODELS_EXTRA env var to add models.`);
+    }
+}
+const LLM_BASE_URLS_DEFAULT = {
+    gemini: process.env.GEMINI_BASE_URL ?? 'https://generativelanguage.googleapis.com/v1beta/openai/',
+    grok: process.env.GROK_BASE_URL ?? 'https://api.x.ai/v1',
+    groq: process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1',
+    mistral: process.env.MISTRAL_BASE_URL ?? 'https://api.mistral.ai/v1',
+    deepseek: process.env.DEEPSEEK_BASE_URL ?? 'https://api.deepseek.com/v1',
+    perplexity: process.env.PERPLEXITY_BASE_URL ?? 'https://api.perplexity.ai',
+    togetherai: process.env.TOGETHERAI_BASE_URL ?? 'https://api.together.xyz/v1',
+    openrouter: process.env.OPENROUTER_BASE_URL ?? 'https://openrouter.ai/api/v1',
+    nvidia: process.env.NVIDIA_BASE_URL ?? 'https://integrate.api.nvidia.com/v1',
+    ollama: process.env.OLLAMA_BASE_URL ?? 'http://localhost:11434/v1',
+};
+function getLLMBaseUrls(overrides) {
+    return { ...LLM_BASE_URLS_DEFAULT, ...overrides };
+}
+const LLM_BASE_URLS = LLM_BASE_URLS_DEFAULT;
+const PLATFORM_DEFAULT_MODELS = {
+    openai: process.env.DEFAULT_MODEL_OPENAI ?? 'gpt-4o',
+    anthropic: process.env.DEFAULT_MODEL_ANTHROPIC ?? 'claude-sonnet-4-6',
+    gemini: process.env.DEFAULT_MODEL_GEMINI ?? 'gemini-2.5-pro',
+    grok: process.env.DEFAULT_MODEL_GROK ?? 'grok-3',
+    groq: process.env.DEFAULT_MODEL_GROQ ?? 'llama-3.3-70b-versatile',
+    mistral: process.env.DEFAULT_MODEL_MISTRAL ?? 'mistral-large-latest',
+    deepseek: process.env.DEFAULT_MODEL_DEEPSEEK ?? 'deepseek-chat',
+    perplexity: process.env.DEFAULT_MODEL_PERPLEXITY ?? 'llama-3.1-sonar-large-128k-online',
+    togetherai: process.env.DEFAULT_MODEL_TOGETHERAI ?? 'meta-llama/Llama-3.3-70B-Instruct-Turbo',
+    openrouter: process.env.DEFAULT_MODEL_OPENROUTER ?? 'anthropic/claude-sonnet-4-6',
+    nvidia: process.env.DEFAULT_MODEL_NVIDIA ?? 'meta/llama-3.1-70b-instruct',
+    ollama: process.env.DEFAULT_MODEL_OLLAMA ?? 'llama3.1',
+};
+// Platform admin LLM categories in priority order (first configured wins).
+// apiKeyField: which DB field holds the API key (default: 'apiKey'); used for NVIDIA multi-key slots.
+// noApiKey: provider works without an API key (ollama — check baseUrl instead).
+const PLATFORM_LLM_CATEGORIES = [
+    { category: 'llm_openai', provider: 'openai', defaultModel: PLATFORM_DEFAULT_MODELS.openai },
+    { category: 'llm', provider: 'anthropic', defaultModel: PLATFORM_DEFAULT_MODELS.anthropic },
+    { category: 'llm_gemini', provider: 'gemini', defaultModel: PLATFORM_DEFAULT_MODELS.gemini,
+        baseUrl: LLM_BASE_URLS.gemini },
+    { category: 'llm_grok', provider: 'grok', defaultModel: PLATFORM_DEFAULT_MODELS.grok,
+        baseUrl: LLM_BASE_URLS.grok },
+    { category: 'llm_groq', provider: 'openrouter', defaultModel: PLATFORM_DEFAULT_MODELS.groq,
+        baseUrl: LLM_BASE_URLS.groq },
+    { category: 'llm_mistral', provider: 'mistral', defaultModel: PLATFORM_DEFAULT_MODELS.mistral,
+        baseUrl: LLM_BASE_URLS.mistral },
+    { category: 'llm_deepseek', provider: 'deepseek', defaultModel: PLATFORM_DEFAULT_MODELS.deepseek,
+        baseUrl: LLM_BASE_URLS.deepseek },
+    { category: 'llm_perplexity', provider: 'perplexity', defaultModel: PLATFORM_DEFAULT_MODELS.perplexity,
+        baseUrl: LLM_BASE_URLS.perplexity },
+    { category: 'llm_togetherai', provider: 'togetherai', defaultModel: PLATFORM_DEFAULT_MODELS.togetherai,
+        baseUrl: LLM_BASE_URLS.togetherai },
+    { category: 'llm_openrouter', provider: 'openrouter', defaultModel: PLATFORM_DEFAULT_MODELS.openrouter,
+        baseUrl: LLM_BASE_URLS.openrouter },
+    // NVIDIA NIM — 10 independent API key slots, tried in order for failover
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey2' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey3' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey4' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey5' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey6' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey7' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey8' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey9' },
+    { category: 'llm_nvidia', provider: 'nvidia', defaultModel: PLATFORM_DEFAULT_MODELS.nvidia,
+        baseUrl: LLM_BASE_URLS.nvidia, apiKeyField: 'apiKey10' },
+    // Ollama — no API key required; configured when admin sets a custom base URL
+    { category: 'llm_ollama', provider: 'ollama', defaultModel: PLATFORM_DEFAULT_MODELS.ollama,
+        baseUrl: LLM_BASE_URLS.ollama, noApiKey: true },
+];
+/** Resolve platform admin LLM config — first configured provider wins. */
+async function getPlatformLLMConfig() {
+    // Cache per-category configs to avoid redundant DB calls for NVIDIA multi-key slots
+    const configCache = new Map();
+    for (const entry of PLATFORM_LLM_CATEGORIES) {
+        const { category, provider, defaultModel, apiKeyField, noApiKey } = entry;
+        try {
+            let cfg = configCache.get(category);
+            if (!cfg) {
+                cfg = await (0, core_1.getSystemConfig)(category);
+                configCache.set(category, cfg);
+            }
+            const resolvedBaseUrl = cfg.baseUrl || entry.baseUrl;
+            if (noApiKey) {
+                // For ollama: only activate if admin explicitly set a base URL or has a model configured
+                if (cfg.model || (resolvedBaseUrl && resolvedBaseUrl !== entry.baseUrl)) {
+                    const model = cfg.model || defaultModel;
+                    return { provider, model, apiKey: '', baseUrl: resolvedBaseUrl };
+                }
+                continue;
+            }
+            const keyField = apiKeyField ?? 'apiKey';
+            const apiKey = cfg[keyField];
+            if (apiKey) {
+                const model = cfg.model || defaultModel;
+                // assertAllowedModel not called — platform admin may configure any provider model
+                return { provider, model, apiKey, baseUrl: resolvedBaseUrl };
+            }
+        }
+        catch { /* try next */ }
+    }
+    // ENV fallback — last resort when neither workspace nor platform admin is configured.
+    if (process.env.OPENAI_API_KEY) {
+        return { provider: 'openai', model: PLATFORM_DEFAULT_MODELS.openai, apiKey: process.env.OPENAI_API_KEY };
+    }
+    if (process.env.ANTHROPIC_API_KEY) {
+        return { provider: 'anthropic', model: PLATFORM_DEFAULT_MODELS.anthropic, apiKey: process.env.ANTHROPIC_API_KEY };
+    }
+    return undefined;
+}
+/** Load the tenant's configured LLM provider. Falls back to platform admin config, then env vars. */
+async function getAgentLLMConfig(tenantId) {
+    await (0, app_brain_1.enforceTokenBudget)(tenantId);
+    try {
+        const stored = await app_brain_1.AppBrain.getTenantLLMConfig(tenantId);
+        // apiKey check uses !== undefined so a decryption failure (undefined) correctly falls through,
+        // but a legitimate empty-string key doesn't silently drop the tenant config.
+        // assertAllowedModel is NOT called here — tenant-configured models are validated at save time
+        // and may include custom providers (NVIDIA NIM, Ollama, OpenRouter) not in the platform allowlist.
+        if (stored?.provider && stored?.model && stored?.apiKey !== undefined && stored?.apiKey !== null) {
+            return {
+                provider: stored.provider,
+                model: stored.model,
+                apiKey: stored.apiKey,
+                baseUrl: stored.baseUrl,
+            };
+        }
+    }
+    catch (e) {
+        console.warn('[llm-config] Failed to load tenant LLM config, falling back to platform config:', e);
+    }
+    // Fall back to platform admin config (set in Admin → Config → AI/LLM Providers)
+    return getPlatformLLMConfig();
+}
+/**
+ * Returns the platform admin LLM config directly, bypassing tenant config.
+ * Use this when a tenant-configured key fails at call time (401/403) to retry with the platform key.
+ */
+async function getPlatformFallbackConfig() {
+    return getPlatformLLMConfig();
+}
+/**
+ * Runs `fn(tenantCfg)` and, if the provider rejects the API key (401/403), retries with
+ * platform admin config. Prevents a single expired workspace key from blocking all agent jobs.
+ *
+ * When tenantCfg is undefined (no tenant or platform config resolved), fails fast with a
+ * clear "No AI provider configured" error rather than making a live call with empty credentials.
+ */
+async function tryWithPlatformFallback(tenantCfg, fn) {
+    if (!tenantCfg) {
+        // getTaskAgentLLMConfig already tried tenant → platform → env. All paths returned undefined.
+        throw new Error('No AI provider configured. Go to Admin → AI/LLM Providers and add an API key, ' +
+            'or set OPENAI_API_KEY / ANTHROPIC_API_KEY in your environment.');
+    }
+    try {
+        return await fn(tenantCfg);
+    }
+    catch (e) {
+        const isAuthError = e?.name === 'LLMProviderError' && /API key|Unauthorized|rejected/i.test(e?.message ?? '');
+        if (!isAuthError || !tenantCfg.apiKey)
+            throw e;
+        const platformCfg = await getPlatformLLMConfig();
+        // Only retry if platform config has a different key — otherwise we'd loop with same failure.
+        if (!platformCfg || platformCfg.apiKey === tenantCfg.apiKey)
+            throw e;
+        console.warn('[llm-config] Workspace API key rejected (401/403), retrying with platform admin config');
+        return await fn(platformCfg);
+    }
+}
+/** Load workspace domain config for a project — used to inject domain hints into prompts. */
+async function getWorkspaceDomainContext(tenantId, projectId) {
+    try {
+        const project = await app_brain_1.AppBrain.getProject(tenantId, projectId);
+        if (!project?.workspaceId)
+            return '';
+        // Load workspace settings directly via AppBrain
+        const ws = await app_brain_1.AppBrain.getWorkspaceSettings(tenantId, project.workspaceId);
+        if (!ws)
+            return '';
+        const parts = [];
+        if (ws.vertical)
+            parts.push(`Domain vertical: ${ws.vertical}`);
+        if (ws.industry)
+            parts.push(`Industry: ${ws.industry}`);
+        if (ws.complianceFrameworks?.length)
+            parts.push(`Compliance: ${ws.complianceFrameworks.join(', ')}`);
+        if (ws.requirementTypes?.length)
+            parts.push(`Requirement types: ${ws.requirementTypes.join(', ')}`);
+        return parts.length ? `\n\n## Workspace Domain Context\n${parts.join('\n')}` : '';
+    }
+    catch {
+        return '';
+    }
+}
+/** Returns true if the workspace has PHI masking enabled — redact real PII field values before LLM calls. */
+async function getWorkspacePhiMasking(tenantId, projectId) {
+    try {
+        const project = await app_brain_1.AppBrain.getProject(tenantId, projectId);
+        if (!project?.workspaceId)
+            return false;
+        const ws = await app_brain_1.AppBrain.getWorkspaceSettings(tenantId, project.workspaceId);
+        return !!ws?.phiMaskingEnabled;
+    }
+    catch {
+        return false;
+    }
+}
+const PROVIDER_FAST_MODELS = {
+    anthropic: process.env.FAST_MODEL_ANTHROPIC ?? 'claude-haiku-4-5-20251001',
+    openai: process.env.FAST_MODEL_OPENAI ?? 'gpt-4o-mini',
+    gemini: process.env.FAST_MODEL_GEMINI ?? 'gemini-2.5-flash',
+    openrouter: process.env.FAST_MODEL_OPENROUTER ?? 'llama-3.3-70b-versatile',
+};
+/** Returns the tenant's LLM config swapped to the provider's fast mini-model for lower-cost passes. */
+async function getFastPassLLMConfig(tenantId) {
+    const base = await getAgentLLMConfig(tenantId);
+    if (!base)
+        return undefined;
+    const fastModel = PROVIDER_FAST_MODELS[base.provider];
+    if (!fastModel)
+        return base;
+    return { ...base, model: fastModel };
+}
+/** Load the tenant's LLM config for a specific task, respecting per-task overrides. */
+async function getTaskAgentLLMConfig(tenantId, task) {
+    await (0, app_brain_1.enforceTokenBudget)(tenantId);
+    try {
+        const stored = await app_brain_1.AppBrain.getTaskLLMConfig(tenantId, task);
+        if (!stored?.provider || !stored?.model || stored?.apiKey === undefined || stored?.apiKey === null) {
+            return getAgentLLMConfig(tenantId);
+        }
+        return {
+            provider: stored.provider,
+            model: stored.model,
+            apiKey: stored.apiKey,
+            baseUrl: stored.baseUrl,
+        };
+    }
+    catch (e) {
+        console.warn('[llm-config] Failed to load task LLM override, falling back to primary config:', e);
+        return getAgentLLMConfig(tenantId);
+    }
+}

package/dist/llm-mock.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Mock / record / replay layer for the LLM choke points in llm.ts.
+ *
+ * Purpose: let agent flows (routing, dedup, persistence, token accounting) run
+ * deterministically in CI without LLM spend. See docs/120-AGENT-E2E-HARNESS-PLAN.md.
+ *
+ * Env flags
+ *   ZETA_LLM_MOCK        '1' | 'replay'  → replay fixtures (fallback if no fixture)
+ *                        'record'        → replay when a fixture exists, otherwise call
+ *                                          the real provider and save the response
+ *   ZETA_LLM_MOCK_DIR    fixture directory (default: <cwd>/eval/fixtures/llm)
+ *   ZETA_LLM_MOCK_FAIL   '429' | '503' | 'malformed-json' | 'empty'  → failure injection
+ *
+ * This module intentionally does NOT import llm.ts (avoids a circular import);
+ * llm.ts turns a `{ kind: 'error' }` outcome into a sanitized LLMProviderError.
+ */
+import type { LLMUsage } from '@taodigital/core';
+export type MockMode = 'off' | 'replay' | 'record';
+export interface MockAskResult {
+    text: string;
+    usage: LLMUsage;
+}
+export type MockOutcome = {
+    kind: 'result';
+    res: MockAskResult;
+    source: 'fixture' | 'fallback' | 'injected';
+} | {
+    kind: 'error';
+    status: number;
+}
+/** record mode only: no fixture exists, caller must hit the real provider then call saveFixture(). */
+ | {
+    kind: 'miss';
+};
+export type MockSystem = string | Array<{
+    text: string;
+    cache?: boolean;
+}>;
+export interface LlmCallStats {
+    calls: number;
+    promptTokens: number;
+    completionTokens: number;
+    estimatedCostUsd: number;
+    /** mock keys seen this window → how each was answered (fixture | fallback | injected | recorded | real) */
+    mockKeys: Record<string, string>;
+}
+export declare function resetLlmStats(): void;
+export declare function snapshotLlmStats(): LlmCallStats;
+export declare function recordLlmCall(usage: Partial<LLMUsage> | undefined, key?: string, source?: string): void;
+interface FixtureFile {
+    key: string;
+    text: string;
+    usage?: Partial<LLMUsage>;
+    recordedAt?: string;
+}
+export declare function getMockMode(): MockMode;
+export declare function fixtureDir(): string;
+/**
+ * Stable agent/pass identifier derived from the pass-specific (last) system block.
+ * e.g. "…ZeTa Test Standard v2.0 Pass 3: Security…" → "pass-3-security"
+ * Falls back to a short hash of the block's first line so unknown agents still get a key.
+ */
+export declare function agentId(system: MockSystem): string;
+/**
+ * Key = <agent/pass id>__<screen slug or head-of-prompt hash>.
+ * Keys deliberately avoid hashing the full prompt so wording tweaks to prompts
+ * don't invalidate every fixture; only a different agent or screen changes the key.
+ */
+export declare function mockKey(system: MockSystem, user: string, kind?: 'text' | 'vision'): string;
+export declare function loadFixture(key: string): FixtureFile | null;
+export declare function saveFixture(key: string, res: MockAskResult): void;
+/** Decide how to answer a call while mocking is on. Never performs a network call itself. */
+export declare function resolveMock(mode: Exclude<MockMode, 'off'>, system: MockSystem, user: string, opts?: {
+    schema?: unknown;
+    kind?: 'text' | 'vision';
+}): MockOutcome;
+export {};