@gzoo/cortex 0.5.9

package/packages/llm/dist/providers/openai-compatible.js

@@ -0,0 +1,164 @@
+import OpenAI from 'openai';
+import { LLMTask, CortexError, LLM_AUTH_FAILED, LLM_PROVIDER_UNAVAILABLE, LLM_RATE_LIMITED, LLM_TIMEOUT, createLogger, } from '@cortex/core';
+const logger = createLogger('llm:openai-compatible');
+export class OpenAICompatibleProvider {
+    name = 'openai-compatible';
+    type = 'cloud';
+    client;
+    primaryModel;
+    fastModel;
+    isGemini;
+    capabilities = {
+        supportedTasks: [
+            LLMTask.ENTITY_EXTRACTION,
+            LLMTask.RELATIONSHIP_INFERENCE,
+            LLMTask.CONTRADICTION_DETECTION,
+            LLMTask.CONVERSATIONAL_QUERY,
+            LLMTask.CONTEXT_RANKING,
+        ],
+        maxContextTokens: 128_000,
+        supportsStructuredOutput: true,
+        supportsStreaming: true,
+        estimatedTokensPerSecond: 80,
+        // Set to 0 — pricing varies by provider; budget tracking is approximate
+        costPerMillionInputTokens: 0,
+        costPerMillionOutputTokens: 0,
+    };
+    constructor(options) {
+        if (!options.apiKey) {
+            throw new CortexError(LLM_AUTH_FAILED, 'critical', 'llm', 'OpenAI-compatible API key not found. Check llm.cloud.apiKeySource in your config.', undefined, 'Set the environment variable specified in llm.cloud.apiKeySource.', false, 401);
+        }
+        this.client = new OpenAI({
+            apiKey: options.apiKey,
+            baseURL: options.baseUrl,
+            timeout: options.timeoutMs ?? 60_000,
+            maxRetries: options.maxRetries ?? 3,
+        });
+        this.primaryModel = options.primaryModel ?? 'gpt-4o';
+        this.fastModel = options.fastModel ?? 'gpt-4o-mini';
+        this.isGemini = options.baseUrl.includes('generativelanguage.googleapis.com');
+        logger.info('OpenAI-compatible provider initialized', {
+            baseUrl: options.baseUrl,
+            primaryModel: this.primaryModel,
+            fastModel: this.fastModel,
+        });
+    }
+    getModel(preference = 'primary') {
+        return preference === 'fast' ? this.fastModel : this.primaryModel;
+    }
+    /** Gemini uses max_completion_tokens; others use max_tokens */
+    tokenLimitParams(maxTokens) {
+        return this.isGemini
+            ? { max_completion_tokens: maxTokens }
+            : { max_tokens: maxTokens };
+    }
+    async complete(prompt, options) {
+        const result = await this.completeWithSystem(undefined, prompt, options);
+        return result.content;
+    }
+    async completeWithSystem(systemPrompt, userPrompt, options, modelPreference = 'primary') {
+        const model = this.getModel(modelPreference);
+        try {
+            const messages = [];
+            if (systemPrompt) {
+                messages.push({ role: 'system', content: systemPrompt });
+            }
+            messages.push({ role: 'user', content: userPrompt });
+            const response = await this.client.chat.completions.create({
+                model,
+                ...this.tokenLimitParams(options?.maxTokens ?? 4096),
+                temperature: options?.temperature ?? 0.7,
+                messages,
+                ...(options?.stopSequences?.length && { stop: options.stopSequences }),
+            });
+            const content = response.choices[0]?.message?.content ?? '';
+            return {
+                content,
+                inputTokens: response.usage?.prompt_tokens ?? 0,
+                outputTokens: response.usage?.completion_tokens ?? 0,
+                model,
+            };
+        }
+        catch (err) {
+            throw this.mapError(err);
+        }
+    }
+    async completeStructured(prompt, _schema, options) {
+        const result = await this.complete(prompt, options);
+        return JSON.parse(result);
+    }
+    async *stream(prompt, options) {
+        yield* this.streamWithSystem(undefined, prompt, options);
+    }
+    async *streamWithSystem(systemPrompt, userPrompt, options, modelPreference = 'primary') {
+        const model = this.getModel(modelPreference);
+        try {
+            const messages = [];
+            if (systemPrompt) {
+                messages.push({ role: 'system', content: systemPrompt });
+            }
+            messages.push({ role: 'user', content: userPrompt });
+            const stream = await this.client.chat.completions.create({
+                model,
+                ...this.tokenLimitParams(options?.maxTokens ?? 4096),
+                temperature: options?.temperature ?? 0.7,
+                messages,
+                stream: true,
+                stream_options: { include_usage: true },
+                ...(options?.stopSequences?.length && { stop: options.stopSequences }),
+            });
+            let inputTokens = 0;
+            let outputTokens = 0;
+            for await (const chunk of stream) {
+                const delta = chunk.choices[0]?.delta?.content;
+                if (delta) {
+                    yield delta;
+                }
+                // Usage is reported in the final chunk when stream_options.include_usage = true
+                if (chunk.usage) {
+                    inputTokens = chunk.usage.prompt_tokens;
+                    outputTokens = chunk.usage.completion_tokens;
+                }
+            }
+            return { inputTokens, outputTokens, model };
+        }
+        catch (err) {
+            throw this.mapError(err);
+        }
+    }
+    async embed(_texts) {
+        throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'medium', 'llm', 'OpenAI-compatible provider does not handle embeddings. Use local embedding model.');
+    }
+    async isAvailable() {
+        try {
+            await this.client.chat.completions.create({
+                model: this.fastModel,
+                ...this.tokenLimitParams(1),
+                messages: [{ role: 'user', content: 'ping' }],
+            });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    mapError(err) {
+        if (err instanceof OpenAI.AuthenticationError) {
+            return new CortexError(LLM_AUTH_FAILED, 'critical', 'llm', 'OpenAI-compatible API authentication failed. Check your API key.', undefined, 'Verify the environment variable in llm.cloud.apiKeySource is correct.', false, 401);
+        }
+        if (err instanceof OpenAI.RateLimitError) {
+            return new CortexError(LLM_RATE_LIMITED, 'medium', 'llm', 'OpenAI-compatible API rate limit exceeded.', undefined, 'Wait and retry with backoff.', true, 429);
+        }
+        if (err instanceof OpenAI.APIConnectionTimeoutError) {
+            return new CortexError(LLM_TIMEOUT, 'medium', 'llm', 'OpenAI-compatible API request timed out.', undefined, 'Retry the request or increase llm.cloud.timeoutMs.', true, 504);
+        }
+        if (err instanceof OpenAI.APIError) {
+            const body = typeof err.error === 'object' ? JSON.stringify(err.error) : String(err.error ?? '');
+            logger.debug('API error details', { status: err.status, body, headers: err.headers });
+            return new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', `OpenAI-compatible API error: ${err.status} ${err.message}${body ? ` — ${body}` : ''}`, { status: err.status }, 'Retry or check your provider status page.', true, err.status);
+        }
+        const message = err instanceof Error ? err.message : String(err);
+        return new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', `OpenAI-compatible provider error: ${message}`, undefined, 'Check network connectivity and llm.cloud.baseUrl configuration.', true);
+    }
+}
+//# sourceMappingURL=openai-compatible.js.map

package/packages/llm/dist/providers/openai-compatible.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-compatible.js","sourceRoot":"","sources":["../../src/providers/openai-compatible.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAIL,OAAO,EACP,WAAW,EACX,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,WAAW,EACX,YAAY,GACb,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,CAAC,CAAC;AAWrD,MAAM,OAAO,wBAAwB;IAC1B,IAAI,GAAG,mBAAmB,CAAC;IAC3B,IAAI,GAAG,OAAgB,CAAC;IAEzB,MAAM,CAAS;IACf,YAAY,CAAS;IACrB,SAAS,CAAS;IAClB,QAAQ,CAAU;IAEjB,YAAY,GAAyB;QAC5C,cAAc,EAAE;YACd,OAAO,CAAC,iBAAiB;YACzB,OAAO,CAAC,sBAAsB;YAC9B,OAAO,CAAC,uBAAuB;YAC/B,OAAO,CAAC,oBAAoB;YAC5B,OAAO,CAAC,eAAe;SACxB;QACD,gBAAgB,EAAE,OAAO;QACzB,wBAAwB,EAAE,IAAI;QAC9B,iBAAiB,EAAE,IAAI;QACvB,wBAAwB,EAAE,EAAE;QAC5B,wEAAwE;QACxE,yBAAyB,EAAE,CAAC;QAC5B,0BAA0B,EAAE,CAAC;KAC9B,CAAC;IAEF,YAAY,OAAwC;QAClD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,WAAW,CACnB,eAAe,EACf,UAAU,EACV,KAAK,EACL,mFAAmF,EACnF,SAAS,EACT,mEAAmE,EACnE,KAAK,EACL,GAAG,CACJ,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;YACpC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;SACpC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC;QACpD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;QAE9E,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE;YACpD,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,aAAiC,SAAS;QACjD,OAAO,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IACpE,CAAC;IAED,+DAA+D;IACvD,gBAAgB,CAAC,SAAiB;QACxC,OAAO,IAAI,CAAC,QAAQ;YAClB,CAAC,CAAC,EAAE,qBAAqB,EAAE,SAAS,EAAE;YACtC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,OAA2B;QACxD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACzE,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,YAAgC,EAChC,UAAkB,EAClB,OAA2B,EAC3B,kBAAsC,SAAS;QAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAE7C,IAAI,CAAC;YACH,MAAM,QAAQ,GAA6C,EAAE,CAAC;YAC9D,IAAI,YAAY,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACzD,KAAK;gBACL,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;gBACpD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,GAAG;gBACxC,QAAQ;gBACR,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC;aACvE,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YAE5D,OAAO;gBACL,OAAO;gBACP,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC;gBAC/C,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC;gBACpD,KAAK;aACN,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,MAAc,EACd,OAAgC,EAChC,OAA2B;QAE3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CACX,MAAc,EACd,OAA2B;QAE3B,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,CAAC,gBAAgB,CACrB,YAAgC,EAChC,UAAkB,EAClB,OAA2B,EAC3B,kBAAsC,SAAS;QAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAE7C,IAAI,CAAC;YACH,MAAM,QAAQ,GAA6C,EAAE,CAAC;YAC9D,IAAI,YAAY,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACvD,KAAK;gBACL,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;gBACpD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,GAAG;gBACxC,QAAQ;gBACR,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE;gBACvC,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC;aACvE,CAAC,CAAC;YAEH,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,IAAI,YAAY,GAAG,CAAC,CAAC;YAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC;gBAC/C,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,gFAAgF;gBAChF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAChB,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC;oBACxC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,iBAAiB,CAAC;gBAC/C,CAAC;YACH,CAAC;YAED,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAgB;QAC1B,MAAM,IAAI,WAAW,CACnB,wBAAwB,EACxB,QAAQ,EACR,KAAK,EACL,mFAAmF,CACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;gBACxC,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;aAC9C,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,GAAY;QAC3B,IAAI,GAAG,YAAY,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC9C,OAAO,IAAI,WAAW,CACpB,eAAe,EAAE,UAAU,EAAE,KAAK,EAClC,kEAAkE,EAClE,SAAS,EAAE,uEAAuE,EAAE,KAAK,EAAE,GAAG,CAC/F,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,YAAY,MAAM,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO,IAAI,WAAW,CACpB,gBAAgB,EAAE,QAAQ,EAAE,KAAK,EACjC,4CAA4C,EAC5C,SAAS,EAAE,8BAA8B,EAAE,IAAI,EAAE,GAAG,CACrD,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,YAAY,MAAM,CAAC,yBAAyB,EAAE,CAAC;YACpD,OAAO,IAAI,WAAW,CACpB,WAAW,EAAE,QAAQ,EAAE,KAAK,EAC5B,0CAA0C,EAC1C,SAAS,EAAE,oDAAoD,EAAE,IAAI,EAAE,GAAG,CAC3E,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACjG,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACtF,OAAO,IAAI,WAAW,CACpB,wBAAwB,EAAE,MAAM,EAAE,KAAK,EACvC,gCAAgC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EACtF,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,EACtB,2CAA2C,EAAE,IAAI,EACjD,GAAG,CAAC,MAAM,CACX,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,IAAI,WAAW,CACpB,wBAAwB,EAAE,MAAM,EAAE,KAAK,EACvC,qCAAqC,OAAO,EAAE,EAC9C,SAAS,EAAE,iEAAiE,EAAE,IAAI,CACnF,CAAC;IACJ,CAAC;CACF"}

package/packages/llm/dist/router.d.ts

@@ -0,0 +1,87 @@
+import type { ZodSchema } from 'zod';
+import { LLMTask, type CortexConfig } from '@cortex/core';
+import { AnthropicProvider } from './providers/anthropic.js';
+import { OllamaProvider } from './providers/ollama.js';
+import { OpenAICompatibleProvider } from './providers/openai-compatible.js';
+import { TokenTracker } from './token-tracker.js';
+import { ResponseCache } from './cache.js';
+export type LLMMode = 'cloud-first' | 'hybrid' | 'local-first' | 'local-only';
+export type TaskRouting = 'auto' | 'local' | 'cloud';
+export interface RouterOptions {
+    config: CortexConfig;
+    apiKey?: string;
+}
+export interface CompleteRequest {
+    systemPrompt?: string;
+    userPrompt: string;
+    promptId: string;
+    promptVersion: string;
+    task: LLMTask;
+    modelPreference?: 'primary' | 'fast';
+    temperature?: number;
+    maxTokens?: number;
+    contentHash?: string;
+    forceProvider?: 'local' | 'cloud';
+}
+export interface CompleteResult {
+    content: string;
+    model: string;
+    inputTokens: number;
+    outputTokens: number;
+    cached: boolean;
+    latencyMs: number;
+    costUsd: number;
+    provider: 'anthropic' | 'ollama' | 'openai-compatible';
+}
+interface ProviderWithSystem {
+    completeWithSystem(systemPrompt: string | undefined, userPrompt: string, options?: {
+        temperature?: number;
+        maxTokens?: number;
+    }, modelPreference?: 'primary' | 'fast'): Promise<{
+        content: string;
+        inputTokens: number;
+        outputTokens: number;
+        model: string;
+    }>;
+    streamWithSystem(systemPrompt: string | undefined, userPrompt: string, options?: {
+        temperature?: number;
+        maxTokens?: number;
+    }, modelPreference?: 'primary' | 'fast'): AsyncGenerator<string, {
+        inputTokens: number;
+        outputTokens: number;
+        model: string;
+    }>;
+    getModel(preference?: 'primary' | 'fast'): string;
+    isAvailable(): Promise<boolean>;
+}
+type CloudProvider = AnthropicProvider | OpenAICompatibleProvider;
+export declare class Router {
+    private cloudProvider;
+    private localProvider;
+    private mode;
+    private taskRouting;
+    private tracker;
+    private cache;
+    private config;
+    private availabilityCache;
+    private static readonly AVAILABILITY_TTL_MS;
+    constructor(options: RouterOptions);
+    /**
+     * Select provider based on mode, task routing, and availability
+     */
+    private selectProvider;
+    get provider(): ProviderWithSystem;
+    complete(request: CompleteRequest): Promise<CompleteResult>;
+    completeStructured<T>(request: CompleteRequest, schema: ZodSchema<T>): Promise<{
+        data: T;
+    } & CompleteResult>;
+    stream(request: Omit<CompleteRequest, 'contentHash'>): AsyncGenerator<string, CompleteResult>;
+    getTracker(): TokenTracker;
+    getCache(): ResponseCache;
+    getLocalProvider(): OllamaProvider | null;
+    getCloudProvider(): CloudProvider | null;
+    getMode(): LLMMode;
+    isAvailable(): Promise<boolean>;
+}
+export {};
+//# sourceMappingURL=router.d.ts.map

package/packages/llm/dist/router.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AACrC,OAAO,EACL,OAAO,EACP,KAAK,YAAY,EAMlB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAK3C,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG,QAAQ,GAAG,aAAa,GAAG,YAAY,CAAC;AAC9E,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,eAAe,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,WAAW,GAAG,QAAQ,GAAG,mBAAmB,CAAC;CACxD;AAED,UAAU,kBAAkB;IAC1B,kBAAkB,CAChB,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,EACtD,eAAe,CAAC,EAAE,SAAS,GAAG,MAAM,GACnC,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1F,gBAAgB,CACd,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,EACtD,eAAe,CAAC,EAAE,SAAS,GAAG,MAAM,GACnC,cAAc,CAAC,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxF,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAClD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CACjC;AAED,KAAK,aAAa,GAAG,iBAAiB,GAAG,wBAAwB,CAAC;AAiBlE,qBAAa,MAAM;IACjB,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,iBAAiB,CAAuD;IAChF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAU;gBAEzC,OAAO,EAAE,aAAa;IAyElC;;OAEG;YACW,cAAc;IAoJ5B,IAAI,QAAQ,IAAI,kBAAkB,CAEjC;IAEK,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IA0G3D,kBAAkB,CAAC,CAAC,EACxB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC;QAAE,IAAI,EAAE,CAAC,CAAA;KAAE,GAAG,cAAc,CAAC;IAiEjC,MAAM,CACX,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,GAC5C,cAAc,CAAC,MAAM,EAAE,cAAc,CAAC;IAmFzC,UAAU,IAAI,YAAY;IAI1B,QAAQ,IAAI,aAAa;IAIzB,gBAAgB,IAAI,cAAc,GAAG,IAAI;IAIzC,gBAAgB,IAAI,aAAa,GAAG,IAAI;IAIxC,OAAO,IAAI,OAAO;IAIZ,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;CA0BtC"}

package/packages/llm/dist/router.js

@@ -0,0 +1,399 @@
+import { LLMTask, CortexError, LLM_BUDGET_EXHAUSTED, LLM_PROVIDER_UNAVAILABLE, createLogger, eventBus, } from '@cortex/core';
+import { AnthropicProvider } from './providers/anthropic.js';
+import { OllamaProvider } from './providers/ollama.js';
+import { OpenAICompatibleProvider } from './providers/openai-compatible.js';
+import { TokenTracker } from './token-tracker.js';
+import { ResponseCache } from './cache.js';
+import { parseStructuredOutput, buildCorrectionPrompt } from './output-parser.js';
+const logger = createLogger('llm:router');
+function resolveApiKeySource(source) {
+    if (source.startsWith('env:')) {
+        return process.env[source.slice(4)];
+    }
+    // Warn if a raw key was provided instead of a reference format
+    if (source && !source.startsWith('keychain:') && !source.startsWith('file:')) {
+        logger.warn('apiKeySource appears to be a raw key. Use "env:VAR_NAME" format instead. ' +
+            'Raw keys in config files are a security risk.');
+        return source;
+    }
+    return undefined;
+}
+export class Router {
+    cloudProvider = null;
+    localProvider = null;
+    mode;
+    taskRouting;
+    tracker;
+    cache;
+    config;
+    availabilityCache = null;
+    static AVAILABILITY_TTL_MS = 60_000; // 1 minute
+    constructor(options) {
+        const { config } = options;
+        this.config = config;
+        this.mode = config.llm.mode;
+        this.taskRouting = config.llm.taskRouting;
+        // Initialize providers based on mode
+        if (this.mode !== 'local-only') {
+            try {
+                if (config.llm.cloud.provider === 'openai-compatible') {
+                    const baseUrl = config.llm.cloud.baseUrl;
+                    if (!baseUrl) {
+                        logger.warn('openai-compatible provider requires llm.cloud.baseUrl — skipping cloud');
+                    }
+                    else {
+                        this.cloudProvider = new OpenAICompatibleProvider({
+                            baseUrl,
+                            apiKey: options.apiKey ?? resolveApiKeySource(config.llm.cloud.apiKeySource),
+                            primaryModel: config.llm.cloud.models.primary,
+                            fastModel: config.llm.cloud.models.fast,
+                            timeoutMs: config.llm.cloud.timeoutMs,
+                            maxRetries: config.llm.cloud.maxRetries,
+                        });
+                    }
+                }
+                else {
+                    this.cloudProvider = new AnthropicProvider({
+                        apiKey: options.apiKey,
+                        primaryModel: config.llm.cloud.models.primary,
+                        fastModel: config.llm.cloud.models.fast,
+                        timeoutMs: config.llm.cloud.timeoutMs,
+                        maxRetries: config.llm.cloud.maxRetries,
+                        promptCaching: config.llm.cloud.promptCaching,
+                    });
+                }
+            }
+            catch (err) {
+                // Cloud provider initialization failed (e.g., no API key)
+                if (this.mode === 'cloud-first') {
+                    throw err; // Cloud is required in cloud-first mode
+                }
+                logger.warn('Cloud provider unavailable, falling back to local-only', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        if (this.mode !== 'cloud-first' || !this.cloudProvider) {
+            this.localProvider = new OllamaProvider({
+                host: config.llm.local.host,
+                model: config.llm.local.model,
+                embeddingModel: config.llm.local.embeddingModel,
+                numCtx: config.llm.local.numCtx,
+                numGpu: config.llm.local.numGpu,
+                timeoutMs: config.llm.local.timeoutMs,
+                keepAlive: config.llm.local.keepAlive,
+            });
+        }
+        this.tracker = new TokenTracker(config.llm.budget.monthlyLimitUsd, config.llm.budget.warningThresholds);
+        this.cache = new ResponseCache({
+            enabled: config.llm.cache.enabled,
+            ttlMs: config.llm.cache.ttlDays * 24 * 60 * 60 * 1000,
+        });
+        logger.info('Router initialized', {
+            mode: this.mode,
+            hasCloud: !!this.cloudProvider,
+            hasLocal: !!this.localProvider,
+        });
+    }
+    /**
+     * Select provider based on mode, task routing, and availability
+     */
+    async selectProvider(task, forceProvider) {
+        const cloudName = () => (this.cloudProvider?.name ?? 'anthropic');
+        // Handle forced provider
+        if (forceProvider === 'cloud') {
+            if (!this.cloudProvider) {
+                throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'Cloud provider requested but not available.', { mode: this.mode }, 'Set your cloud API key or change LLM mode.', false);
+            }
+            return { provider: this.cloudProvider, name: cloudName() };
+        }
+        if (forceProvider === 'local') {
+            if (!this.localProvider) {
+                throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'Local provider requested but not configured.', { mode: this.mode }, 'Change LLM mode to include local provider.', false);
+            }
+            return { provider: this.localProvider, name: 'ollama' };
+        }
+        // Check task-specific routing
+        const taskRoute = this.taskRouting[task] ?? 'auto';
+        if (taskRoute === 'cloud' && this.cloudProvider) {
+            return { provider: this.cloudProvider, name: cloudName() };
+        }
+        if (taskRoute === 'local' && this.localProvider) {
+            return { provider: this.localProvider, name: 'ollama' };
+        }
+        // Auto routing based on mode
+        switch (this.mode) {
+            case 'local-only':
+                if (!this.localProvider) {
+                    throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'Local-only mode but Ollama provider not available.', undefined, 'Ensure Ollama is running with `ollama serve`.', false);
+                }
+                return { provider: this.localProvider, name: 'ollama' };
+            case 'local-first':
+                // Try local first, fall back to cloud
+                if (this.localProvider && await this.localProvider.isAvailable()) {
+                    return { provider: this.localProvider, name: 'ollama' };
+                }
+                if (this.cloudProvider) {
+                    logger.info('Local provider unavailable, falling back to cloud');
+                    return { provider: this.cloudProvider, name: cloudName() };
+                }
+                throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'No LLM provider available.', { mode: this.mode }, 'Start Ollama or configure cloud API key.', false);
+            case 'hybrid':
+                // Use local for cheap/high-volume tasks, cloud for reasoning-heavy tasks
+                const cheapTasks = [
+                    LLMTask.ENTITY_EXTRACTION,
+                    LLMTask.CONTEXT_RANKING,
+                    LLMTask.EMBEDDING_GENERATION,
+                ];
+                if (cheapTasks.includes(task) && this.localProvider && await this.localProvider.isAvailable()) {
+                    logger.debug('Hybrid routing to local provider', { task });
+                    return { provider: this.localProvider, name: 'ollama' };
+                }
+                if (this.cloudProvider) {
+                    logger.debug('Hybrid routing to cloud provider', { task });
+                    return { provider: this.cloudProvider, name: cloudName() };
+                }
+                // Cloud unavailable — fall back to local for everything
+                if (this.localProvider) {
+                    logger.warn('Cloud provider unavailable in hybrid mode, falling back to local', { task });
+                    return { provider: this.localProvider, name: 'ollama' };
+                }
+                throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'No LLM provider available.', { mode: this.mode }, 'Configure cloud API key or start Ollama.', false);
+            case 'cloud-first':
+            default:
+                // Prefer cloud, fall back to local if budget exhausted
+                if (this.cloudProvider && !this.tracker.isBudgetExhausted()) {
+                    return { provider: this.cloudProvider, name: cloudName() };
+                }
+                if (this.localProvider && this.config.llm.budget.enforcementAction === 'fallback-local') {
+                    logger.info('Budget exhausted or cloud unavailable, falling back to local');
+                    return { provider: this.localProvider, name: 'ollama' };
+                }
+                if (!this.cloudProvider) {
+                    throw new CortexError(LLM_PROVIDER_UNAVAILABLE, 'high', 'llm', 'Cloud provider not available.', { mode: this.mode }, 'Set your cloud API key or change LLM mode.', false);
+                }
+                // Cloud available but budget exhausted and no fallback
+                throw new CortexError(LLM_BUDGET_EXHAUSTED, 'high', 'llm', 'Monthly budget exhausted.', { spent: this.tracker.getCurrentMonthSpend() }, 'Increase budget, wait for next month, or enable local fallback.', false, 402);
+        }
+    }
+    // Keep legacy provider getter for backward compatibility
+    get provider() {
+        return this.cloudProvider ?? this.localProvider;
+    }
+    async complete(request) {
+        // Select provider based on mode and task
+        const { provider, name: providerName } = await this.selectProvider(request.task, request.forceProvider);
+        // Check cache
+        if (request.contentHash) {
+            const cached = this.cache.get(request.contentHash, request.promptId, request.promptVersion);
+            if (cached) {
+                return {
+                    content: cached.response,
+                    model: cached.model,
+                    inputTokens: cached.inputTokens,
+                    outputTokens: cached.outputTokens,
+                    cached: true,
+                    latencyMs: 0,
+                    costUsd: 0,
+                    provider: providerName,
+                };
+            }
+        }
+        // Emit start event
+        const requestId = crypto.randomUUID();
+        eventBus.emit({
+            type: 'llm.request.start',
+            payload: { requestId, task: request.task, provider: providerName },
+            timestamp: new Date().toISOString(),
+            source: 'llm:router',
+        });
+        const startMs = performance.now();
+        const result = await provider.completeWithSystem(request.systemPrompt, request.userPrompt, {
+            temperature: request.temperature,
+            maxTokens: request.maxTokens,
+        }, request.modelPreference ?? 'primary');
+        const latencyMs = Math.round(performance.now() - startMs);
+        // Track tokens
+        const usageRecord = this.tracker.record(requestId, request.task, providerName, result.model, result.inputTokens, result.outputTokens, latencyMs);
+        // Cache the result
+        if (request.contentHash) {
+            this.cache.set(request.contentHash, request.promptId, request.promptVersion, result.content, result.model, result.inputTokens, result.outputTokens);
+        }
+        // Emit complete event
+        eventBus.emit({
+            type: 'llm.request.complete',
+            payload: {
+                requestId,
+                task: request.task,
+                provider: providerName,
+                model: result.model,
+                usage: {
+                    inputTokens: result.inputTokens,
+                    outputTokens: result.outputTokens,
+                    estimatedCostUsd: usageRecord.estimatedCostUsd,
+                },
+                latencyMs,
+            },
+            timestamp: new Date().toISOString(),
+            source: 'llm:router',
+        });
+        return {
+            content: result.content,
+            model: result.model,
+            inputTokens: result.inputTokens,
+            outputTokens: result.outputTokens,
+            cached: false,
+            latencyMs,
+            costUsd: usageRecord.estimatedCostUsd,
+            provider: providerName,
+        };
+    }
+    async completeStructured(request, schema) {
+        const result = await this.complete(request);
+        try {
+            const data = parseStructuredOutput(result.content, schema);
+            // Step 9: local-first confidence escalation
+            // If the local provider returned low-confidence entities, retry with cloud
+            if (this.mode === 'local-first' &&
+                result.provider === 'ollama' &&
+                this.cloudProvider &&
+                !request.forceProvider) {
+                const entities = data.entities;
+                if (Array.isArray(entities) && entities.length > 0) {
+                    const confidences = entities
+                        .map((e) => (typeof e.confidence === 'number' ? e.confidence : null))
+                        .filter((c) => c !== null)
+                        .sort((a, b) => a - b);
+                    if (confidences.length > 0) {
+                        const mid = Math.floor(confidences.length / 2);
+                        const median = confidences.length % 2 !== 0
+                            ? confidences[mid]
+                            : ((confidences[mid - 1] + confidences[mid]) / 2);
+                        if (median < 0.6) {
+                            logger.info('Local confidence below threshold, escalating to cloud', {
+                                median: Math.round(median * 100) / 100,
+                                task: request.task,
+                            });
+                            const cloudResult = await this.complete({ ...request, forceProvider: 'cloud', contentHash: undefined });
+                            const cloudData = parseStructuredOutput(cloudResult.content, schema);
+                            return { ...cloudResult, data: cloudData };
+                        }
+                    }
+                }
+            }
+            return { ...result, data };
+        }
+        catch (firstErr) {
+            // Retry once with correction prompt
+            logger.warn('Structured output parse failed, retrying with correction', {
+                promptId: request.promptId,
+                error: firstErr instanceof Error ? firstErr.message : String(firstErr),
+            });
+            const correctedPrompt = buildCorrectionPrompt(request.userPrompt, result.content, firstErr instanceof Error ? firstErr.message : String(firstErr));
+            const retryResult = await this.complete({
+                ...request,
+                userPrompt: correctedPrompt,
+                contentHash: undefined, // Don't cache correction attempts
+            });
+            const data = parseStructuredOutput(retryResult.content, schema);
+            return { ...retryResult, data };
+        }
+    }
+    async *stream(request) {
+        // Select provider based on mode and task
+        const { provider, name: providerName } = await this.selectProvider(request.task, request.forceProvider);
+        const requestId = crypto.randomUUID();
+        eventBus.emit({
+            type: 'llm.request.start',
+            payload: { requestId, task: request.task, provider: providerName },
+            timestamp: new Date().toISOString(),
+            source: 'llm:router',
+        });
+        const startMs = performance.now();
+        let fullContent = '';
+        const gen = provider.streamWithSystem(request.systemPrompt, request.userPrompt, {
+            temperature: request.temperature,
+            maxTokens: request.maxTokens,
+        }, request.modelPreference ?? 'primary');
+        let streamResult;
+        while (true) {
+            const { value, done } = await gen.next();
+            if (done) {
+                streamResult = value;
+                break;
+            }
+            fullContent += value;
+            yield value;
+        }
+        const latencyMs = Math.round(performance.now() - startMs);
+        const tokens = streamResult ?? { inputTokens: 0, outputTokens: 0, model: provider.getModel() };
+        const usageRecord = this.tracker.record(requestId, request.task, providerName, tokens.model, tokens.inputTokens, tokens.outputTokens, latencyMs);
+        eventBus.emit({
+            type: 'llm.request.complete',
+            payload: {
+                requestId,
+                task: request.task,
+                provider: providerName,
+                model: tokens.model,
+                usage: {
+                    inputTokens: tokens.inputTokens,
+                    outputTokens: tokens.outputTokens,
+                    estimatedCostUsd: usageRecord.estimatedCostUsd,
+                },
+                latencyMs,
+            },
+            timestamp: new Date().toISOString(),
+            source: 'llm:router',
+        });
+        return {
+            content: fullContent,
+            model: tokens.model,
+            inputTokens: tokens.inputTokens,
+            outputTokens: tokens.outputTokens,
+            cached: false,
+            latencyMs,
+            costUsd: usageRecord.estimatedCostUsd,
+            provider: providerName,
+        };
+    }
+    getTracker() {
+        return this.tracker;
+    }
+    getCache() {
+        return this.cache;
+    }
+    getLocalProvider() {
+        return this.localProvider;
+    }
+    getCloudProvider() {
+        return this.cloudProvider;
+    }
+    getMode() {
+        return this.mode;
+    }
+    async isAvailable() {
+        // Return cached result if still valid
+        if (this.availabilityCache && Date.now() < this.availabilityCache.expiresAt) {
+            return this.availabilityCache.result;
+        }
+        let result;
+        switch (this.mode) {
+            case 'local-only':
+                result = await this.localProvider?.isAvailable() ?? false;
+                break;
+            case 'cloud-first':
+                result = await this.cloudProvider?.isAvailable() ?? false;
+                break;
+            default: {
+                // hybrid or local-first: either provider works
+                const localAvailable = await this.localProvider?.isAvailable() ?? false;
+                const cloudAvailable = await this.cloudProvider?.isAvailable() ?? false;
+                result = localAvailable || cloudAvailable;
+                break;
+            }
+        }
+        this.availabilityCache = { result, expiresAt: Date.now() + Router.AVAILABILITY_TTL_MS };
+        return result;
+    }
+}
+//# sourceMappingURL=router.js.map