@agile-vibe-coding/avc 0.1.1 → 0.3.1

package/cli/llm-local.js

@@ -0,0 +1,493 @@
+import OpenAI from 'openai';
+import { jsonrepair } from 'jsonrepair';
+import { LLMProvider } from './llm-provider.js';
+
+/**
+ * Known local inference servers and their default endpoints.
+ * All expose an OpenAI-compatible Chat Completions API.
+ */
+const KNOWN_SERVERS = [
+  { app: 'lmstudio',  url: 'http://localhost:1234/v1',   label: 'LM Studio' },
+  { app: 'ollama',    url: 'http://localhost:11434/v1',   label: 'Ollama' },
+  { app: 'llamacpp',  url: 'http://localhost:8080/v1',    label: 'llama.cpp' },
+  { app: 'vllm',      url: 'http://localhost:8000/v1',    label: 'vLLM' },
+  { app: 'localai',   url: 'http://localhost:8081/v1',    label: 'LocalAI' },
+];
+
+export { KNOWN_SERVERS };
+
+/**
+ * Probe a single server: GET /models with a short timeout.
+ * @param {string} baseURL - e.g. 'http://localhost:1234/v1'
+ * @param {number} timeoutMs
+ * @returns {Promise<Array<{id:string, owned_by?:string, context_length?:number}>>}
+ */
+async function probeServer(baseURL, timeoutMs = 1500) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const resp = await fetch(`${baseURL}/models`, { signal: controller.signal });
+    if (!resp.ok) return [];
+    const body = await resp.json();
+    return body.data || [];
+  } catch {
+    return [];
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Discover all running local inference servers and their loaded models.
+ * Also checks LOCAL_LLM_URL env var for custom endpoints.
+ * @returns {Promise<Array<{app:string, label:string, url:string, models:Array}>>}
+ */
+export async function discoverLocalServers() {
+  const servers = [...KNOWN_SERVERS];
+
+  // Add custom endpoint from env if set and not already in the list
+  const customUrl = process.env.LOCAL_LLM_URL;
+  if (customUrl) {
+    const normalized = customUrl.replace(/\/+$/, '');
+    const base = normalized.endsWith('/v1') ? normalized : `${normalized}/v1`;
+    if (!servers.some(s => s.url === base)) {
+      servers.unshift({ app: 'custom', url: base, label: 'Custom Local' });
+    }
+  }
+
+  const results = await Promise.all(
+    servers.map(async (server) => {
+      const rawModels = await probeServer(server.url);
+      if (rawModels.length === 0) return null;
+      const models = rawModels.map(m => ({
+        id: m.id,
+        ownedBy: m.owned_by || null,
+        contextLength: m.context_length || null,
+      }));
+      return { ...server, models };
+    })
+  );
+
+  return results.filter(Boolean);
+}
+
+/**
+ * Strip any XML-like tag blocks that local models use for internal reasoning.
+ * Models use a variety of tags: <think>, <reasoning>, <reflection>, <scratchpad>,
+ * <internal>, <thought>, <analysis>, <planning>, <chain_of_thought>, etc.
+ * Instead of maintaining an allowlist, we strip ALL matched <word>...</word> blocks
+ * that appear before the actual content.
+ */
+function cleanLocalResponse(text) {
+  let cleaned = text;
+
+  // Pass 1: remove all <single_word>...</single_word> blocks (case-insensitive, multi-line).
+  // These are almost universally reasoning/thinking wrappers in local models.
+  // Matches tags like <think>, <reasoning>, <reflection>, <scratchpad>, <thought>,
+  // <analysis>, <planning>, <internal>, <chain_of_thought>, etc.
+  // Uses a backreference to ensure opening and closing tags match.
+  cleaned = cleaned.replace(/<([a-z][a-z0-9_]*)>[\s\S]*?<\/\1>\s*/gi, '');
+
+  return cleaned.trim();
+}
+
+/**
+ * For JSON responses: extract the JSON payload from whatever the model emitted.
+ * Handles code fences, residual text before/after JSON, and nested structures.
+ */
+function extractJSON(text) {
+  let s = text.trim();
+
+  // Strip markdown code fences
+  if (s.startsWith('```')) {
+    s = s.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?\s*```\s*$/, '').trim();
+  }
+
+  // If it already looks like JSON, return as-is
+  if (s.startsWith('{') || s.startsWith('[')) return s;
+
+  // Otherwise, find the first { or [ and extract to its matching closer.
+  // This handles models that emit preamble text before the JSON.
+  const objStart = s.indexOf('{');
+  const arrStart = s.indexOf('[');
+  let start = -1;
+  let openChar, closeChar;
+
+  if (objStart >= 0 && (arrStart < 0 || objStart < arrStart)) {
+    start = objStart; openChar = '{'; closeChar = '}';
+  } else if (arrStart >= 0) {
+    start = arrStart; openChar = '['; closeChar = ']';
+  }
+
+  if (start < 0) return s; // no JSON found, return as-is for error reporting
+
+  // Walk forward counting nesting depth, respecting strings
+  let depth = 0;
+  let inString = false;
+  let escape = false;
+  for (let i = start; i < s.length; i++) {
+    const ch = s[i];
+    if (escape) { escape = false; continue; }
+    if (ch === '\\' && inString) { escape = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+    if (ch === openChar) depth++;
+    if (ch === closeChar) { depth--; if (depth === 0) return s.slice(start, i + 1); }
+  }
+
+  // Unbalanced — return from start to end, let JSON.parse report the real error
+  return s.slice(start);
+}
+
+/**
+ * LocalProvider — connects to any OpenAI-compatible local inference server.
+ * Uses the `openai` SDK pointed at a local base URL.
+ */
+export class LocalProvider extends LLMProvider {
+  /**
+   * @param {string} model - Model ID loaded on the local server
+   * @param {string} [baseURL] - Override base URL (defaults to auto-detect or LOCAL_LLM_URL)
+   */
+  constructor(model = 'default', baseURL = null) {
+    super('local', model);
+    this._baseURL = baseURL || process.env.LOCAL_LLM_URL || null;
+  }
+
+  /**
+   * Auto-detect which server has the requested model, or use configured base URL.
+   */
+  async _resolveBaseURL() {
+    if (this._baseURL) {
+      const normalized = this._baseURL.replace(/\/+$/, '');
+      return normalized.endsWith('/v1') ? normalized : `${normalized}/v1`;
+    }
+
+    // Auto-detect: probe known servers for the model
+    const servers = await discoverLocalServers();
+    for (const server of servers) {
+      if (server.models.some(m => m.id === this.model)) {
+        this._baseURL = server.url;
+        return server.url;
+      }
+    }
+
+    // Fallback: if any server is running, use the first one
+    if (servers.length > 0) {
+      this._baseURL = servers[0].url;
+      return servers[0].url;
+    }
+
+    throw new Error(
+      `No local inference server found. Start LM Studio, Ollama, or another local server, ` +
+      `or set LOCAL_LLM_URL in your .env file.`
+    );
+  }
+
+  _createClient() {
+    // Deferred — actual client creation happens in _ensureClient() after async base URL resolution
+    return null;
+  }
+
+  async _ensureClient() {
+    if (this._client) return;
+    const baseURL = await this._resolveBaseURL();
+    this._client = new OpenAI({
+      baseURL,
+      apiKey: 'not-needed', // Local servers don't require API keys
+      timeout: 30 * 60_000, // 30 min — local models on consumer hardware can be very slow
+      maxRetries: 0,
+    });
+  }
+
+  /**
+   * Stream a chat completion and collect the full response.
+   * Streaming prevents server-side idle timeouts (e.g. LM Studio's 5-min default)
+   * because tokens flow continuously over the connection.
+   * @returns {{ content: string, usage: object|null }}
+   */
+  async _streamCompletion(params) {
+    const stream = await this._client.chat.completions.create({
+      ...params,
+      stream: true,
+      stream_options: { include_usage: true },
+    });
+
+    const chunks = [];
+    let usage = null;
+    for await (const chunk of stream) {
+      const delta = chunk.choices?.[0]?.delta?.content;
+      if (delta) chunks.push(delta);
+      // The last chunk with stream_options.include_usage carries the usage object
+      if (chunk.usage) usage = chunk.usage;
+    }
+    return { content: chunks.join(''), usage };
+  }
+
+  async _callProvider(prompt, maxTokens, systemInstructions) {
+    await this._ensureClient();
+
+    const messages = [];
+    if (systemInstructions) {
+      messages.push({ role: 'system', content: systemInstructions });
+    }
+    messages.push({ role: 'user', content: prompt });
+
+    const params = {
+      model: this.model,
+      messages,
+      // Don't send max_tokens to local servers — let the server use its own
+      // context window limit. Artificially capping output truncates large responses.
+    };
+
+    const { content, usage } = await this._streamCompletion(params);
+    this._trackTokens(usage);
+    return cleanLocalResponse(content);
+  }
+
+  async generateJSON(prompt, agentInstructions = null, cachedContext = null) {
+    await this._ensureClient();
+
+    const JSON_SYSTEM = 'You are a helpful assistant that always returns valid JSON. Your response must be a valid JSON object or array, nothing else. Do not include any thinking, reasoning, or explanation — only the JSON.';
+    const systemParts = [JSON_SYSTEM];
+    if (agentInstructions) systemParts.push(agentInstructions);
+    if (cachedContext) systemParts.push(`---\n\n${cachedContext}`);
+
+    const messages = [
+      { role: 'system', content: systemParts.join('\n\n') },
+      { role: 'user',   content: prompt },
+    ];
+
+    const params = {
+      model: this.model,
+      messages,
+      // Don't send max_tokens to local servers — let the server use its own
+      // context window limit. Artificially capping output truncates large responses.
+    };
+
+    // Try JSON mode — not all local servers support it; fall back gracefully
+    let useJsonMode = true;
+
+    const _t0 = Date.now();
+    let content;
+    let usage;
+    try {
+      ({ content, usage } = await this._withRetry(
+        () => this._streamCompletion({
+          ...params,
+          ...(useJsonMode ? { response_format: { type: 'json_object' } } : {}),
+        }),
+        'JSON generation (Local)'
+      ));
+    } catch (err) {
+      // If JSON mode is not supported, retry without it
+      if (useJsonMode && (err.message?.includes('response_format') || err.status === 400)) {
+        useJsonMode = false;
+        ({ content, usage } = await this._withRetry(
+          () => this._streamCompletion(params),
+          'JSON generation (Local, no json_mode)'
+        ));
+      } else {
+        throw err;
+      }
+    }
+
+    this._trackTokens(usage, {
+      prompt,
+      agentInstructions: agentInstructions ?? null,
+      response: content,
+      elapsed: Date.now() - _t0,
+    });
+
+    // Strip reasoning tags, code fences, and extract JSON payload
+    let jsonStr = extractJSON(cleanLocalResponse(content));
+
+    try {
+      return JSON.parse(jsonStr);
+    } catch (firstError) {
+      if (jsonStr.startsWith('{') || jsonStr.startsWith('[')) {
+        try { return JSON.parse(jsonrepair(jsonStr)); } catch { /* fall through */ }
+      }
+      throw new Error(`Failed to parse JSON response from local model: ${firstError.message}\n\nResponse was:\n${content}`);
+    }
+  }
+
+  /**
+   * Generate JSON with tool-calling support.
+   * Runs a tool-call loop: if the model emits tool_calls, they are executed
+   * via the provided dispatcher and results fed back until the model returns
+   * a final JSON content response.
+   *
+   * @param {string} prompt - User prompt
+   * @param {string|null} agentInstructions - System instructions
+   * @param {Array} tools - OpenAI-format tool definitions
+   * @param {Function} toolDispatcher - async (name, args) => string
+   * @param {number} [maxRounds=5] - Max tool-call rounds to prevent infinite loops
+   * @returns {Promise<Object>} Parsed JSON response
+   */
+  async generateJSONWithTools(prompt, agentInstructions, tools, toolDispatcher, maxRounds = 5) {
+    await this._ensureClient();
+
+    const JSON_SYSTEM = 'You are a helpful assistant that always returns valid JSON. When you need accurate API details for external services, use the fetch_api_reference tool before writing your response. Your final response must be a valid JSON object or array, nothing else.';
+    const systemParts = [JSON_SYSTEM];
+    if (agentInstructions) systemParts.push(agentInstructions);
+
+    const messages = [
+      { role: 'system', content: systemParts.join('\n\n') },
+      { role: 'user',   content: prompt },
+    ];
+
+    const _t0 = Date.now();
+    let totalUsage = { prompt_tokens: 0, completion_tokens: 0 };
+    let toolCallCount = 0;
+
+    let forceNoTools = false;
+    for (let round = 0; round < maxRounds + 2; round++) {
+      const params = {
+        model: this.model,
+        messages,
+        ...(forceNoTools ? {} : { tools }),
+      };
+
+      let response;
+      try {
+        if (forceNoTools) {
+          // Final forced round: use streaming to prevent LM Studio's server-side idle timeout
+          // (non-streaming requests can timeout after ~5 min under load).
+          // No tools in params, so we only expect content back — streaming is safe.
+          const { content, usage } = await this._streamCompletion(params);
+          response = {
+            usage,
+            choices: [{ finish_reason: 'stop', message: { content, tool_calls: null } }],
+          };
+        } else {
+          // Tool-call rounds: use non-streaming (streaming tool call support is inconsistent across local servers)
+          response = await this._withRetry(
+            () => this._client.chat.completions.create(params),
+            `JSON+tools generation (Local, round ${round + 1})`
+          );
+        }
+      } catch (err) {
+        // If tools not supported, fall back to regular generateJSON
+        if (err.message?.includes('tools') || err.status === 400) {
+          console.log('[DEBUG] Local server does not support tools — falling back to generateJSON');
+          return this.generateJSON(prompt, agentInstructions);
+        }
+        throw err;
+      }
+
+      // Accumulate token usage
+      if (response.usage) {
+        totalUsage.prompt_tokens += response.usage.prompt_tokens || 0;
+        totalUsage.completion_tokens += response.usage.completion_tokens || 0;
+      }
+
+      const choice = response.choices?.[0];
+      if (!choice) {
+        throw new Error('No response choice from local model');
+      }
+
+      // If the model wants to call tools, execute them and loop
+      if (choice.finish_reason === 'tool_calls' || choice.message?.tool_calls?.length > 0) {
+        if (round >= maxRounds) {
+          console.log(`[DEBUG] Tool call loop exceeded ${maxRounds} rounds — forcing final response without tools`);
+          // Re-request without tools to force a content response
+          messages.push(choice.message);
+          messages.push({ role: 'user', content: 'Please provide your final JSON response now, without calling any more tools.' });
+          forceNoTools = true;
+          continue;
+        }
+
+        // Append assistant message with tool calls
+        messages.push(choice.message);
+
+        // Execute each tool call
+        for (const toolCall of choice.message.tool_calls) {
+          toolCallCount++;
+          const fnName = toolCall.function?.name;
+          let fnArgs;
+          try {
+            fnArgs = JSON.parse(toolCall.function?.arguments || '{}');
+          } catch {
+            fnArgs = {};
+          }
+
+          console.log(`[DEBUG] Tool call #${toolCallCount}: ${fnName}(${JSON.stringify(fnArgs)})`);
+
+          let result;
+          try {
+            result = await toolDispatcher(fnName, fnArgs);
+          } catch (err) {
+            result = `Tool error: ${err.message}`;
+          }
+
+          // Append tool result
+          messages.push({
+            role: 'tool',
+            tool_call_id: toolCall.id,
+            content: typeof result === 'string' ? result : JSON.stringify(result),
+          });
+        }
+
+        continue; // Loop back for the model to process tool results
+      }
+
+      // Model returned content — extract and parse JSON
+      const content = choice.message?.content || '';
+      this._trackTokens(totalUsage, {
+        prompt,
+        agentInstructions: agentInstructions ?? null,
+        response: content,
+        elapsed: Date.now() - _t0,
+      });
+
+      if (toolCallCount > 0) {
+        console.log(`[DEBUG] Tool-augmented generation complete: ${toolCallCount} tool call(s) in ${round + 1} round(s)`);
+      }
+
+      const jsonStr = extractJSON(cleanLocalResponse(content));
+      try {
+        return JSON.parse(jsonStr);
+      } catch (firstError) {
+        if (jsonStr.startsWith('{') || jsonStr.startsWith('[')) {
+          try { return JSON.parse(jsonrepair(jsonStr)); } catch { /* fall through */ }
+        }
+        throw new Error(`Failed to parse JSON response from local model (with tools): ${firstError.message}\n\nResponse was:\n${content}`);
+      }
+    }
+
+    throw new Error('Tool call loop exhausted without final response');
+  }
+
+  async generateText(prompt, agentInstructions = null, cachedContext = null) {
+    await this._ensureClient();
+
+    const systemParts = [];
+    if (agentInstructions) systemParts.push(agentInstructions);
+    if (cachedContext) systemParts.push(cachedContext);
+
+    const messages = [];
+    if (systemParts.length > 0) {
+      messages.push({ role: 'system', content: systemParts.join('\n\n') });
+    }
+    messages.push({ role: 'user', content: prompt });
+
+    const _t0 = Date.now();
+    const { content, usage } = await this._withRetry(
+      () => this._streamCompletion({
+        model: this.model,
+        messages,
+        // Don't send max_tokens to local servers — let the server manage limits.
+      }),
+      'Text generation (Local)'
+    );
+
+    const textContent = cleanLocalResponse(content);
+    this._trackTokens(usage, {
+      prompt,
+      agentInstructions: agentInstructions ?? null,
+      response: textContent,
+      elapsed: Date.now() - _t0,
+    });
+    return textContent;
+  }
+}