hazo_llm_api 1.2.13 → 1.3.0

package/dist/lib/llm_api/index.js

@@ -25,11 +25,19 @@ import { hazo_llm_image_image_text as hazo_llm_image_image_text_internal } from
 import { hazo_llm_prompt_chain as hazo_llm_prompt_chain_internal } from './hazo_llm_prompt_chain.js';
 import { hazo_llm_document_text as hazo_llm_document_text_internal } from './hazo_llm_document_text.js';
 import { hazo_llm_dynamic_data_extract as hazo_llm_dynamic_data_extract_internal } from './hazo_llm_dynamic_data_extract.js';
+import { hazo_llm_embed_internal } from './hazo_llm_embed.js';
+import { configure_embed_cache } from './embed_cache.js';
 import { register_provider, set_enabled_llms, set_primary_llm, get_primary_llm, get_provider, } from '../providers/registry.js';
 import { GeminiProvider } from '../providers/gemini/index.js';
 import { QwenProvider } from '../providers/qwen/index.js';
+import { AnthropicProvider } from '../providers/anthropic/index.js';
+import { OpenAIProvider } from '../providers/openai/index.js';
+import { DeepSeekProvider } from '../providers/deepseek/index.js';
 import { SERVICE_TYPES } from '../providers/types.js';
 import { create_request_context, call_before_request_hook, call_after_response_hook, call_on_error_hook, detect_error_code, } from './provider_helper.js';
+import { ensure_api_log_table } from '../database/init_api_log.js';
+import { create_default_after_response_hook } from '../observability/log_writer.js';
+import { load_pricing, set_pricing_overrides } from '../pricing/pricing.js';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as ini from 'ini';
@@ -40,6 +48,8 @@ let initialized = false;
 let db_auto_initialized = false;
 let current_config = null;
 let stored_hazo_connect = null;
+let stored_cascade_config = null;
+let stored_cost_cap_config = null;
 // =============================================================================
 // Default Logger
 // =============================================================================
@@ -403,7 +413,7 @@ function load_api_key_from_env(provider_name) {
 }
 /**
  * Read LLM global config from [llm] section
- * @returns Object with enabled_llms and primary_llm
+ * @returns Object with enabled_llms, primary_llm, sqlite_path, and optional cascade config
  */
 function get_llm_global_config() {
     const config_path = find_config_file();
@@ -424,10 +434,37 @@ function get_llm_global_config() {
         const enabled_llms = parse_enabled_llms(llm_section.enabled_llms);
         const primary_llm = llm_section.primary_llm || default_primary;
         const sqlite_path = llm_section.sqlite_path || default_sqlite;
+        const cascade_section = config.llm_cascade;
+        let parsed_cascade;
+        if (cascade_section) {
+            parsed_cascade = {};
+            if (cascade_section.providers) {
+                parsed_cascade.providers = String(cascade_section.providers)
+                    .split(',')
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+            }
+            if (cascade_section.timeout_ms_per_attempt) {
+                const ms = parseInt(String(cascade_section.timeout_ms_per_attempt), 10);
+                if (!isNaN(ms) && ms > 0) {
+                    parsed_cascade.timeout_ms_per_attempt = ms;
+                }
+            }
+            if (cascade_section.cascade_on_codes) {
+                const codes = String(cascade_section.cascade_on_codes)
+                    .split(',')
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+                if (codes.length > 0) {
+                    parsed_cascade.cascade_on_codes = codes;
+                }
+            }
+        }
         return {
             enabled_llms: enabled_llms.length > 0 ? enabled_llms : default_enabled,
             primary_llm,
             sqlite_path,
+            cascade: parsed_cascade,
         };
     }
     catch {
@@ -435,6 +472,7 @@ function get_llm_global_config() {
             enabled_llms: default_enabled,
             primary_llm: default_primary,
             sqlite_path: default_sqlite,
+            cascade: undefined,
         };
     }
 }
@@ -660,6 +698,220 @@ function load_qwen_provider_from_config(logger) {
         return null;
     }
 }
+/**
+ * Load Anthropic provider from config file
+ * Reads [llm_anthropic] section + ANTHROPIC_API_KEY env var
+ */
+function load_anthropic_provider_from_config(logger) {
+    const config_path = find_config_file();
+    if (!config_path) {
+        logger.warn('Config file not found, cannot load Anthropic provider', {
+            file: 'index.ts',
+            line: 0,
+        });
+        return null;
+    }
+    try {
+        const config_content = fs.readFileSync(config_path, 'utf-8');
+        const config = ini.parse(config_content);
+        const section = config.llm_anthropic || {};
+        const env_var_name = section.api_key_env || 'ANTHROPIC_API_KEY';
+        const api_key = process.env[env_var_name];
+        if (!api_key) {
+            logger.error(`${env_var_name} not found in environment variables`, {
+                file: 'index.ts',
+                line: 0,
+                data: { config_path, env_var_name },
+            });
+            return null;
+        }
+        const capabilities = parse_capabilities(section.capabilities);
+        const generation_config = {};
+        let has_config = false;
+        if (section.text_max_tokens !== undefined) {
+            const v = parseInt(section.text_max_tokens, 10);
+            if (!isNaN(v)) {
+                generation_config.max_tokens = v;
+                has_config = true;
+            }
+        }
+        if (section.text_temperature !== undefined) {
+            const v = parseFloat(section.text_temperature);
+            if (!isNaN(v)) {
+                generation_config.temperature = v;
+                has_config = true;
+            }
+        }
+        const provider_config = {
+            api_key,
+            api_url: section.api_url,
+            api_version: section.api_version,
+            model_text_text: section.model_text_text,
+            model_image_text: section.model_image_text,
+            model_document_text: section.model_document_text,
+            generation_config: has_config ? generation_config : undefined,
+            capabilities: capabilities.length > 0 ? capabilities : undefined,
+            logger,
+        };
+        return new AnthropicProvider(provider_config);
+    }
+    catch (error) {
+        const error_message = error instanceof Error ? error.message : String(error);
+        logger.error('Failed to load Anthropic provider from config', {
+            file: 'index.ts',
+            line: 0,
+            data: { error: error_message, config_path },
+        });
+        return null;
+    }
+}
+/**
+ * Load OpenAI provider from config file
+ * Reads [llm_openai] section + OPENAI_API_KEY env var
+ */
+function load_openai_provider_from_config(logger) {
+    const config_path = find_config_file();
+    if (!config_path) {
+        logger.warn('Config file not found, cannot load OpenAI provider', {
+            file: 'index.ts',
+            line: 0,
+        });
+        return null;
+    }
+    try {
+        const config_content = fs.readFileSync(config_path, 'utf-8');
+        const config = ini.parse(config_content);
+        const section = config.llm_openai || {};
+        const env_var_name = section.api_key_env || 'OPENAI_API_KEY';
+        const api_key = process.env[env_var_name];
+        if (!api_key) {
+            logger.error(`${env_var_name} not found in environment variables`, {
+                file: 'index.ts',
+                line: 0,
+                data: { config_path, env_var_name },
+            });
+            return null;
+        }
+        const capabilities = parse_capabilities(section.capabilities);
+        const generation_config = {};
+        let has_config = false;
+        if (section.text_temperature !== undefined) {
+            const v = parseFloat(section.text_temperature);
+            if (!isNaN(v)) {
+                generation_config.temperature = v;
+                has_config = true;
+            }
+        }
+        if (section.text_top_p !== undefined) {
+            const v = parseFloat(section.text_top_p);
+            if (!isNaN(v)) {
+                generation_config.top_p = v;
+                has_config = true;
+            }
+        }
+        if (section.text_max_tokens !== undefined) {
+            const v = parseInt(section.text_max_tokens, 10);
+            if (!isNaN(v)) {
+                generation_config.max_tokens = v;
+                has_config = true;
+            }
+        }
+        const provider_config = {
+            api_key,
+            api_url: section.api_url,
+            api_url_image: section.api_url_image,
+            api_url_embed: section.api_url_embed,
+            model_text_text: section.model_text_text,
+            model_image_text: section.model_image_text,
+            model_text_image: section.model_text_image,
+            model_embed: section.model_embed,
+            generation_config: has_config ? generation_config : undefined,
+            capabilities: capabilities.length > 0 ? capabilities : undefined,
+            logger,
+        };
+        return new OpenAIProvider(provider_config);
+    }
+    catch (error) {
+        const error_message = error instanceof Error ? error.message : String(error);
+        logger.error('Failed to load OpenAI provider from config', {
+            file: 'index.ts',
+            line: 0,
+            data: { error: error_message, config_path },
+        });
+        return null;
+    }
+}
+/**
+ * Load DeepSeek provider from config file
+ * Reads [llm_deepseek] section + DEEPSEEK_API_KEY env var
+ */
+function load_deepseek_provider_from_config(logger) {
+    const config_path = find_config_file();
+    if (!config_path) {
+        logger.warn('Config file not found, cannot load DeepSeek provider', {
+            file: 'index.ts',
+            line: 0,
+        });
+        return null;
+    }
+    try {
+        const config_content = fs.readFileSync(config_path, 'utf-8');
+        const config = ini.parse(config_content);
+        const section = config.llm_deepseek || {};
+        const env_var_name = section.api_key_env || 'DEEPSEEK_API_KEY';
+        const api_key = process.env[env_var_name];
+        if (!api_key) {
+            logger.error(`${env_var_name} not found in environment variables`, {
+                file: 'index.ts',
+                line: 0,
+                data: { config_path, env_var_name },
+            });
+            return null;
+        }
+        const capabilities = parse_capabilities(section.capabilities);
+        const generation_config = {};
+        let has_config = false;
+        if (section.text_temperature !== undefined) {
+            const v = parseFloat(section.text_temperature);
+            if (!isNaN(v)) {
+                generation_config.temperature = v;
+                has_config = true;
+            }
+        }
+        if (section.text_top_p !== undefined) {
+            const v = parseFloat(section.text_top_p);
+            if (!isNaN(v)) {
+                generation_config.top_p = v;
+                has_config = true;
+            }
+        }
+        if (section.text_max_tokens !== undefined) {
+            const v = parseInt(section.text_max_tokens, 10);
+            if (!isNaN(v)) {
+                generation_config.max_tokens = v;
+                has_config = true;
+            }
+        }
+        const provider_config = {
+            api_key,
+            api_url: section.api_url,
+            model_text_text: section.model_text_text,
+            generation_config: has_config ? generation_config : undefined,
+            capabilities: capabilities.length > 0 ? capabilities : undefined,
+            logger,
+        };
+        return new DeepSeekProvider(provider_config);
+    }
+    catch (error) {
+        const error_message = error instanceof Error ? error.message : String(error);
+        logger.error('Failed to load DeepSeek provider from config', {
+            file: 'index.ts',
+            line: 0,
+            data: { error: error_message, config_path },
+        });
+        return null;
+    }
+}
 /**
  * Load and register all enabled providers from config file
  * @param logger - Logger instance
@@ -697,7 +949,46 @@ function load_and_register_providers(logger) {
                 });
             }
         }
-        // Future: Add other providers here (OpenAI, Anthropic, etc.)
+        else if (llm_name.toLowerCase() === 'anthropic') {
+            const provider = load_anthropic_provider_from_config(logger);
+            if (provider) {
+                register_provider(provider);
+            }
+            else {
+                logger.warn('Anthropic provider is enabled in config but failed to load. Check ANTHROPIC_API_KEY in environment variables.', {
+                    file: 'index.ts',
+                    line: 0,
+                    data: { llm_name: llm_name.toLowerCase() },
+                });
+            }
+        }
+        else if (llm_name.toLowerCase() === 'openai') {
+            const provider = load_openai_provider_from_config(logger);
+            if (provider) {
+                register_provider(provider);
+            }
+            else {
+                logger.warn('OpenAI provider is enabled in config but failed to load. Check OPENAI_API_KEY in environment variables.', {
+                    file: 'index.ts',
+                    line: 0,
+                    data: { llm_name: llm_name.toLowerCase() },
+                });
+            }
+        }
+        else if (llm_name.toLowerCase() === 'deepseek') {
+            const provider = load_deepseek_provider_from_config(logger);
+            if (provider) {
+                register_provider(provider);
+            }
+            else {
+                logger.warn('DeepSeek provider is enabled in config but failed to load. Check DEEPSEEK_API_KEY in environment variables.', {
+                    file: 'index.ts',
+                    line: 0,
+                    data: { llm_name: llm_name.toLowerCase() },
+                });
+            }
+        }
+        // Future: Add other providers here
     }
 }
 // =============================================================================
@@ -758,10 +1049,12 @@ export async function initialize_llm_api(config = {}) {
     const logger = config.logger || default_logger;
     // Store the logger for use by other functions
     set_logger(logger);
-    // Store hooks if provided
-    if (config.hooks) {
-        set_hooks(config.hooks);
-    }
+    // Reset hooks — will be set correctly after connect is established below
+    set_hooks({});
+    // Reset cascade config — will be set correctly below
+    stored_cascade_config = null;
+    // Reset cost cap config — will be set correctly below
+    stored_cost_cap_config = null;
     // Get global config from file
     const global_config = get_llm_global_config();
     // Use provided sqlite_path or fall back to config file value
@@ -803,6 +1096,66 @@ export async function initialize_llm_api(config = {}) {
         }
         initialized = true;
         current_config = final_config;
+        // Apply pricing table and any consumer overrides
+        load_pricing();
+        if (config.pricing_overrides) {
+            set_pricing_overrides(config.pricing_overrides);
+        }
+        // Store resolved cascade config: init-time config takes precedence over INI
+        stored_cascade_config = config.cascade ?? global_config.cascade ?? null;
+        // Store cost cap config from consumer
+        stored_cost_cap_config = config.cost_cap ?? null;
+        // Configure embed cache from config
+        if (config.embed_cache) {
+            configure_embed_cache({
+                max_size: config.embed_cache.max_size,
+                keyv: config.embed_cache.keyv,
+            });
+        }
+        // Set up the log table + default afterResponse hook unless explicitly disabled
+        const resolved_connect = stored_hazo_connect;
+        const api_log_enabled = config.api_log?.enabled !== false;
+        const consumer_after = config.hooks?.afterResponse;
+        if (api_log_enabled && resolved_connect) {
+            try {
+                await ensure_api_log_table(resolved_connect, logger);
+                // Ensure cost cap index exists for fast session-based SUM queries
+                const { ensure_cost_cap_index } = await import('../cost_cap/cost_cap.js');
+                await ensure_cost_cap_index(resolved_connect, logger);
+                const default_hook = create_default_after_response_hook({
+                    connect: resolved_connect,
+                    logger,
+                    extract_context: config.api_log?.extract_context,
+                });
+                // Chain: default hook first, then consumer hook (if provided)
+                const chained = consumer_after
+                    ? async (ctx) => {
+                        await default_hook(ctx);
+                        await consumer_after(ctx);
+                    }
+                    : default_hook;
+                set_hooks({
+                    ...get_hooks(), // preserve any other hooks (beforeRequest, onError)
+                    ...config.hooks, // merge consumer hooks
+                    afterResponse: chained,
+                });
+            }
+            catch (err) {
+                logger.warn('api_log setup failed; calls will not be logged', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+                // Fall through — install consumer hooks only, don't override with failing default
+                if (config.hooks) {
+                    set_hooks(config.hooks);
+                }
+            }
+        }
+        else {
+            // api_log disabled or no connect: install consumer hooks as-is (or leave empty)
+            if (config.hooks) {
+                set_hooks(config.hooks);
+            }
+        }
         // LLM API initialized successfully
     }
     catch (error) {
@@ -1116,6 +1469,36 @@ export async function hazo_llm_dynamic_data_extract(params, llm) {
     }
 }
 // =============================================================================
+// Embed Function
+// =============================================================================
+/**
+ * Generate vector embeddings for one or more text inputs.
+ * Caches results in-memory (or via BYO Keyv) and deduplicates repeated texts in batches.
+ *
+ * @param params - Text string or array of strings to embed, plus optional model override
+ * @returns EmbedResponse with vectors, dimensions, model, and cache flags
+ *
+ * @example
+ * ```typescript
+ * import { hazo_llm_embed } from 'hazo_llm_api/server';
+ *
+ * const result = await hazo_llm_embed({ text: 'Hello world' });
+ * if (result.success) {
+ *   console.log(result.vectors?.[0]); // number[]
+ * }
+ * ```
+ */
+export async function hazo_llm_embed(params, _llm) {
+    try {
+        const config = check_initialized();
+        const logger = config.logger || default_logger;
+        return hazo_llm_embed_internal(params, logger);
+    }
+    catch (error) {
+        return { success: false, error: error instanceof Error ? error.message : String(error) };
+    }
+}
+// =============================================================================
 // Streaming Functions
 // =============================================================================
 /**
@@ -1173,6 +1556,21 @@ export async function* hazo_llm_text_text_stream(params, llm) {
             };
             return;
         }
+        // Cost cap pre-call gate
+        const cost_cap_config = get_cost_cap_config();
+        if (cost_cap_config) {
+            const { check_cost_cap_gate } = await import('../cost_cap/cost_cap.js');
+            const cap_block = await check_cost_cap_gate(stored_hazo_connect, cost_cap_config, logger);
+            if (cap_block !== null) {
+                yield {
+                    text: '',
+                    done: true,
+                    error: cap_block.error ?? cap_block.error_info?.message,
+                    error_info: cap_block.error_info,
+                };
+                return;
+            }
+        }
         // Call hooks and streaming method
         const request_context = create_request_context(SERVICE_TYPES.TEXT_TEXT, provider.get_name(), {
             prompt: params.prompt,
@@ -1267,6 +1665,21 @@ export async function* hazo_llm_image_text_stream(params, llm) {
             };
             return;
         }
+        // Cost cap pre-call gate
+        const cost_cap_config = get_cost_cap_config();
+        if (cost_cap_config) {
+            const { check_cost_cap_gate } = await import('../cost_cap/cost_cap.js');
+            const cap_block = await check_cost_cap_gate(stored_hazo_connect, cost_cap_config, logger);
+            if (cap_block !== null) {
+                yield {
+                    text: '',
+                    done: true,
+                    error: cap_block.error ?? cap_block.error_info?.message,
+                    error_info: cap_block.error_info,
+                };
+                return;
+            }
+        }
         // Call hooks and streaming method
         const request_context = create_request_context(SERVICE_TYPES.IMAGE_TEXT, provider.get_name(), {
             prompt: params.prompt,
@@ -1329,6 +1742,20 @@ export async function* hazo_llm_image_text_stream(params, llm) {
 export function is_initialized() {
     return initialized;
 }
+/**
+ * Get the current cascade configuration (from init or INI).
+ * Returns null if no cascade config has been set.
+ */
+export function get_cascade_config() {
+    return stored_cascade_config;
+}
+/**
+ * Get the current cost cap configuration (set during initialize_llm_api).
+ * Returns null if no cost cap config has been set.
+ */
+export function get_cost_cap_config() {
+    return stored_cost_cap_config;
+}
 /**
  * Get the current configuration (without sensitive logger)
  * @returns Current configuration or null if not initialized