ai-consultants 2.10.1

package/scripts/lib/api_query.sh

@@ -0,0 +1,228 @@
+#!/bin/bash
+# api_query.sh - API mode query execution for AI Consultants v2.6
+#
+# This module provides a unified API query function that handles the
+# differences between API formats (OpenAI, Anthropic, Google AI).
+#
+# Used by query_*.sh scripts when they are in API mode instead of CLI mode.
+
+# Guard against double-sourcing
+if [[ -n "${_API_QUERY_SH_SOURCED:-}" ]]; then
+    return 0 2>/dev/null || exit 0
+fi
+_API_QUERY_SH_SOURCED=1
+
+# Load dependencies
+SCRIPT_DIR_API_QUERY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR_API_QUERY/common.sh"
+source "$SCRIPT_DIR_API_QUERY/api.sh"
+
+# =============================================================================
+# API QUERY EXECUTION
+# =============================================================================
+
+# Execute an API query for a consultant in API mode
+#
+# Usage: run_api_mode_query <consultant_name> <model> <query> <output_file> <timeout_sec>
+#
+# This function:
+# 1. Determines the correct API format based on consultant name
+# 2. Builds the appropriate request body
+# 3. Executes the API call with retry logic
+# 4. Parses the response
+# 5. Returns the raw text response
+#
+# Returns:
+#   0 on success (response written to output_file)
+#   1 on auth failure
+#   2 on rate limit
+#   3 on server error
+#   124 on timeout
+run_api_mode_query() {
+    local consultant_name="$1"
+    local model="$2"
+    local query="$3"
+    local output_file="$4"
+    local timeout_seconds="${5:-180}"
+
+    # Get API configuration
+    local api_format
+    api_format=$(get_api_format "$consultant_name")
+    local api_key_var
+    api_key_var=$(get_api_key_var "$consultant_name")
+    local api_url
+    api_url=$(get_api_url "$consultant_name")
+
+    # Validate API key (except for Google AI where key is in URL)
+    if [[ "$api_format" != "google_ai" && -z "${!api_key_var:-}" ]]; then
+        log_error "[$consultant_name] API key not set: $api_key_var"
+        return 1
+    fi
+
+    # Build request body based on format
+    local request_body
+    local final_api_url="$api_url"
+    local auth_style="bearer"
+
+    case "$api_format" in
+        google_ai)
+            request_body=$(build_google_ai_request "$query")
+            # Google AI appends model to URL; use x-goog-api-key header for security
+            final_api_url="${api_url}/${model}:generateContent"
+            auth_style="google_ai"
+            ;;
+        anthropic)
+            request_body=$(build_anthropic_request "$query" "$model")
+            auth_style="anthropic"
+            ;;
+        qwen)
+            request_body=$(build_qwen_request "$query" "$model")
+            ;;
+        *)  # openai format
+            request_body=$(build_openai_request "$query" "$model")
+            ;;
+    esac
+
+    log_debug "[$consultant_name] API format: $api_format"
+    log_debug "[$consultant_name] API URL: $final_api_url"
+
+    # Create temp file for raw API response
+    local temp_response
+    temp_response=$(mktemp)
+
+    # Execute API call using unified run_api_query with auth_style
+    run_api_query "$consultant_name" "$temp_response" "$timeout_seconds" "$final_api_url" "$api_key_var" "$request_body" "$auth_style"
+    local exit_code=$?
+
+    if [[ $exit_code -ne 0 ]]; then
+        rm -f "$temp_response"
+        return $exit_code
+    fi
+
+    # Parse response based on format
+    local raw_response
+    raw_response=$(cat "$temp_response")
+
+    local parsed_content
+    case "$api_format" in
+        google_ai)
+            parsed_content=$(parse_google_ai_response "$raw_response")
+            ;;
+        anthropic)
+            parsed_content=$(parse_anthropic_response "$raw_response")
+            ;;
+        qwen)
+            parsed_content=$(parse_qwen_response "$raw_response")
+            ;;
+        *)  # openai format
+            parsed_content=$(parse_openai_response "$raw_response")
+            ;;
+    esac
+
+    if [[ -z "$parsed_content" ]]; then
+        log_error "[$consultant_name] Failed to parse API response"
+        rm -f "$temp_response"
+        return 1
+    fi
+
+    # Write parsed content to output
+    echo "$parsed_content" > "$output_file"
+
+    # Extract and log token usage
+    local tokens
+    tokens=$(extract_token_usage "$raw_response" "$api_format")
+    log_debug "[$consultant_name] Tokens used: $tokens"
+
+    rm -f "$temp_response"
+    return 0
+}
+
+# =============================================================================
+# GENERIC API CONSULTANT FUNCTION (v2.6)
+# =============================================================================
+
+# Run a consultation using API mode for any API-based consultant
+# This function is called by consult_all.sh for custom API agents and
+# as a fallback when no dedicated query script exists.
+#
+# Usage: run_api_consultant <consultant_name> <query> <context_file> <output_file>
+#
+# Parameters:
+#   consultant_name - Name of the consultant (e.g., "Qwen3", "DeepSeek")
+#   query           - The query text (can be empty if context_file provides all content)
+#   context_file    - Path to context file (can be empty)
+#   output_file     - Path to write the JSON response
+#
+# Returns:
+#   0 on success
+#   1 on failure
+run_api_consultant() {
+    local consultant_name="$1"
+    local query="$2"
+    local context_file="$3"
+    local output_file="$4"
+
+    local consultant_upper
+    consultant_upper=$(to_upper "$consultant_name")
+
+    # Get configuration for this consultant
+    local model_var="${consultant_upper}_MODEL"
+    local timeout_var="${consultant_upper}_TIMEOUT_SECONDS"
+
+    local model="${!model_var:-}"
+    local timeout_seconds="${!timeout_var:-180}"
+
+    # Build full query from query + context
+    local full_query
+    full_query=$(build_full_query "$query" "$context_file")
+
+    if [[ -z "$full_query" ]]; then
+        log_error "[$consultant_name] No query to send"
+        return 1
+    fi
+
+    local start_time
+    start_time=$(get_timestamp_ms)
+
+    # Create temp file for API response
+    local temp_response
+    temp_response=$(mktemp)
+
+    # Use run_api_mode_query which handles all the API format/auth logic
+    run_api_mode_query "$consultant_name" "$model" "$full_query" "$temp_response" "$timeout_seconds"
+    local exit_code=$?
+
+    local end_time
+    end_time=$(get_timestamp_ms)
+    local latency=$((end_time - start_time))
+
+    if [[ $exit_code -ne 0 ]]; then
+        build_error_response "$consultant_name" "${model:-unknown}" "API Consultant" \
+            "API query failed (exit code: $exit_code)" "$latency" > "$output_file"
+        rm -f "$temp_response"
+        return 1
+    fi
+
+    # Read parsed content from temp file (run_api_mode_query already parsed it)
+    local parsed_content
+    parsed_content=$(cat "$temp_response")
+    rm -f "$temp_response"
+
+    if [[ -z "$parsed_content" ]]; then
+        build_error_response "$consultant_name" "${model:-unknown}" "API Consultant" \
+            "Empty response from API" "$latency" > "$output_file"
+        return 1
+    fi
+
+    # Try to parse as structured JSON response
+    if echo "$parsed_content" | jq -e '.response' >/dev/null 2>&1; then
+        build_structured_response "$consultant_name" "${model:-unknown}" "API Consultant" \
+            "$parsed_content" "$latency" > "$output_file"
+    else
+        build_fallback_response "$consultant_name" "${model:-unknown}" "API Consultant" \
+            "$parsed_content" "$latency" > "$output_file"
+    fi
+
+    log_success "[$consultant_name] Response generated"
+    return 0
+}

package/scripts/lib/cache.sh

@@ -0,0 +1,338 @@
+#!/bin/bash
+# cache.sh - Semantic caching for AI Consultants
+#
+# Caches consultation responses based on semantic fingerprints
+# to avoid redundant API calls for similar queries.
+
+set -euo pipefail
+
+# =============================================================================
+# CONFIGURATION
+# =============================================================================
+
+CACHE_DIR="${CACHE_DIR:-/tmp/ai_consultants_cache}"
+CACHE_TTL_HOURS="${CACHE_TTL_HOURS:-24}"
+
+# =============================================================================
+# CACHE INITIALIZATION
+# =============================================================================
+
+# Initialize cache directory
+# Usage: init_cache
+init_cache() {
+    if [[ ! -d "$CACHE_DIR" ]]; then
+        mkdir -p "$CACHE_DIR"
+        chmod 700 "$CACHE_DIR"
+    fi
+}
+
+# =============================================================================
+# FINGERPRINT GENERATION
+# =============================================================================
+
+# Normalize query for consistent hashing
+# Usage: _normalize_query <query>
+_normalize_query() {
+    local query="$1"
+    # Lowercase, remove extra whitespace, trim
+    echo "$query" | tr '[:upper:]' '[:lower:]' | sed 's/[[:space:]]\+/ /g' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
+}
+
+# Calculate checksum for a file
+# Usage: _file_checksum <file_path>
+_file_checksum() {
+    local file="$1"
+    if [[ -f "$file" ]]; then
+        if command -v md5sum &>/dev/null; then
+            md5sum "$file" 2>/dev/null | cut -d' ' -f1
+        elif command -v md5 &>/dev/null; then
+            md5 -q "$file" 2>/dev/null
+        else
+            # Fallback: use file size and mtime
+            stat -f "%z%m" "$file" 2>/dev/null || stat -c "%s%Y" "$file" 2>/dev/null || echo "0"
+        fi
+    else
+        echo ""
+    fi
+}
+
+# Generate semantic fingerprint for a query + context
+# Usage: generate_fingerprint <query> <category> [context_file]
+generate_fingerprint() {
+    local query="$1"
+    local category="${2:-GENERAL}"
+    local context_file="${3:-}"
+
+    # Normalize query
+    local normalized_query
+    normalized_query=$(_normalize_query "$query")
+
+    # Build fingerprint components
+    local fingerprint_input="${normalized_query}|${category}"
+
+    # Add context checksum if provided
+    if [[ -n "$context_file" && -f "$context_file" ]]; then
+        local context_checksum
+        context_checksum=$(_file_checksum "$context_file")
+        fingerprint_input+="|${context_checksum}"
+    fi
+
+    # Generate hash
+    if command -v sha256sum &>/dev/null; then
+        echo -n "$fingerprint_input" | sha256sum | cut -d' ' -f1
+    elif command -v shasum &>/dev/null; then
+        echo -n "$fingerprint_input" | shasum -a 256 | cut -d' ' -f1
+    else
+        # Fallback to md5
+        if command -v md5sum &>/dev/null; then
+            echo -n "$fingerprint_input" | md5sum | cut -d' ' -f1
+        elif command -v md5 &>/dev/null; then
+            echo -n "$fingerprint_input" | md5
+        else
+            # Last resort: simple hash
+            echo -n "$fingerprint_input" | cksum | cut -d' ' -f1
+        fi
+    fi
+}
+
+# =============================================================================
+# CACHE OPERATIONS
+# =============================================================================
+
+# Check if semantic caching is enabled
+# Usage: is_cache_enabled
+is_cache_enabled() {
+    [[ "${ENABLE_SEMANTIC_CACHE:-true}" == "true" ]]
+}
+
+# Get cache file path for a fingerprint
+# Usage: _get_cache_path <fingerprint> <consultant>
+_get_cache_path() {
+    local fingerprint="$1"
+    local consultant="${2:-all}"
+    echo "${CACHE_DIR}/${consultant}_${fingerprint}.json"
+}
+
+# Get cache age in hours
+# Usage: _get_cache_age_hours <cache_file>
+_get_cache_age_hours() {
+    local cache_file="$1"
+
+    if [[ ! -f "$cache_file" ]]; then
+        echo "999999"  # Very old
+        return
+    fi
+
+    local now
+    local file_time
+    local age_seconds
+
+    now=$(date +%s)
+
+    # Get file modification time
+    if stat -f %m "$cache_file" &>/dev/null; then
+        # macOS
+        file_time=$(stat -f %m "$cache_file")
+    elif stat -c %Y "$cache_file" &>/dev/null; then
+        # Linux
+        file_time=$(stat -c %Y "$cache_file")
+    else
+        echo "999999"
+        return
+    fi
+
+    age_seconds=$((now - file_time))
+    echo $((age_seconds / 3600))
+}
+
+# Check cache for a response
+# Usage: check_cache <query> <category> <consultant> [context_file]
+# Returns: cached response JSON if found and valid, empty otherwise
+check_cache() {
+    local query="$1"
+    local category="${2:-GENERAL}"
+    local consultant="${3:-all}"
+    local context_file="${4:-}"
+
+    # Check if caching is enabled
+    if ! is_cache_enabled; then
+        return 1
+    fi
+
+    init_cache
+
+    local fingerprint
+    fingerprint=$(generate_fingerprint "$query" "$category" "$context_file")
+
+    local cache_file
+    cache_file=$(_get_cache_path "$fingerprint" "$consultant")
+
+    # Check if cache file exists
+    if [[ ! -f "$cache_file" ]]; then
+        return 1
+    fi
+
+    # Check TTL
+    local age_hours
+    age_hours=$(_get_cache_age_hours "$cache_file")
+    local ttl="${CACHE_TTL_HOURS:-24}"
+
+    if [[ $age_hours -ge $ttl ]]; then
+        # Cache expired
+        rm -f "$cache_file" 2>/dev/null || true
+        return 1
+    fi
+
+    # Return cached content
+    cat "$cache_file"
+    return 0
+}
+
+# Store response in cache
+# Usage: store_cache <query> <category> <consultant> <response_json> [context_file]
+store_cache() {
+    local query="$1"
+    local category="${2:-GENERAL}"
+    local consultant="${3:-all}"
+    local response="$4"
+    local context_file="${5:-}"
+
+    # Check if caching is enabled
+    if ! is_cache_enabled; then
+        return 0
+    fi
+
+    init_cache
+
+    local fingerprint
+    fingerprint=$(generate_fingerprint "$query" "$category" "$context_file")
+
+    local cache_file
+    cache_file=$(_get_cache_path "$fingerprint" "$consultant")
+
+    # Store response with metadata
+    local timestamp
+    timestamp=$(date -Iseconds)
+
+    # Add cache metadata to response
+    local cached_response
+    cached_response=$(echo "$response" | jq --arg ts "$timestamp" --arg fp "$fingerprint" \
+        '. + {cache_metadata: {cached_at: $ts, fingerprint: $fp, from_cache: false}}' 2>/dev/null || echo "$response")
+
+    # Atomic write: write to temp file first, then move to final location
+    # This prevents race conditions with parallel consultants
+    local temp_file
+    temp_file=$(mktemp "${CACHE_DIR}/cache_tmp.XXXXXX")
+    echo "$cached_response" > "$temp_file"
+    chmod 600 "$temp_file"
+    mv "$temp_file" "$cache_file"
+}
+
+# Invalidate cache for a specific query
+# Usage: invalidate_cache <query> <category> [consultant] [context_file]
+invalidate_cache() {
+    local query="$1"
+    local category="${2:-GENERAL}"
+    local consultant="${3:-}"
+    local context_file="${4:-}"
+
+    init_cache
+
+    local fingerprint
+    fingerprint=$(generate_fingerprint "$query" "$category" "$context_file")
+
+    if [[ -n "$consultant" ]]; then
+        # Invalidate specific consultant cache
+        local cache_file
+        cache_file=$(_get_cache_path "$fingerprint" "$consultant")
+        rm -f "$cache_file" 2>/dev/null || true
+    else
+        # Invalidate all caches for this fingerprint
+        rm -f "${CACHE_DIR}"/*_"${fingerprint}".json 2>/dev/null || true
+    fi
+}
+
+# Clear all expired cache entries
+# Usage: cleanup_cache
+cleanup_cache() {
+    init_cache
+
+    local ttl="${CACHE_TTL_HOURS:-24}"
+    local count=0
+
+    for cache_file in "$CACHE_DIR"/*.json; do
+        if [[ -f "$cache_file" ]]; then
+            local age_hours
+            age_hours=$(_get_cache_age_hours "$cache_file")
+
+            if [[ $age_hours -ge $ttl ]]; then
+                rm -f "$cache_file" 2>/dev/null || true
+                ((count++)) || true
+            fi
+        fi
+    done
+
+    echo "$count"
+}
+
+# Clear all cache entries
+# Usage: clear_cache
+clear_cache() {
+    init_cache
+    rm -f "$CACHE_DIR"/*.json 2>/dev/null || true
+}
+
+# =============================================================================
+# CACHE STATISTICS
+# =============================================================================
+
+# Get cache statistics
+# Usage: get_cache_stats
+get_cache_stats() {
+    init_cache
+
+    local total_files=0
+    local total_size=0
+    local expired_files=0
+    local ttl="${CACHE_TTL_HOURS:-24}"
+
+    for cache_file in "$CACHE_DIR"/*.json; do
+        if [[ -f "$cache_file" ]]; then
+            ((total_files++)) || true
+
+            local file_size
+            if stat -f %z "$cache_file" &>/dev/null; then
+                file_size=$(stat -f %z "$cache_file")
+            else
+                file_size=$(stat -c %s "$cache_file" 2>/dev/null || echo 0)
+            fi
+            total_size=$((total_size + file_size))
+
+            local age_hours
+            age_hours=$(_get_cache_age_hours "$cache_file")
+            if [[ $age_hours -ge $ttl ]]; then
+                ((expired_files++)) || true
+            fi
+        fi
+    done
+
+    local size_kb=$((total_size / 1024))
+
+    cat << EOF
+{
+  "total_entries": $total_files,
+  "expired_entries": $expired_files,
+  "total_size_kb": $size_kb,
+  "cache_dir": "$CACHE_DIR",
+  "ttl_hours": $ttl,
+  "enabled": $(is_cache_enabled && echo "true" || echo "false")
+}
+EOF
+}
+
+# Mark response as from cache (for logging/metrics)
+# Usage: echo "$json" | mark_from_cache   OR   mark_from_cache "$json"
+mark_from_cache() {
+    local response="${1:-$(cat)}"
+    echo "$response" | jq '.cache_metadata.from_cache = true' 2>/dev/null || echo "$response"
+}