@contentgrowth/llm-service 0.4.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@contentgrowth/llm-service",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Unified LLM Service for Content Growth",
   "main": "src/index.js",
   "type": "module",

package/src/llm/json-utils.js

@@ -0,0 +1,80 @@
+/**
+ * Extracts and parses JSON from a text response (e.g., from an LLM).
+ * Handles JSON in markdown code blocks or plain JSON objects.
+ * 
+ * TODO: improveme for better performance
+ * 
+ * @param {string} text - The text containing JSON
+ * @returns {object|null} - The parsed JSON object, or null if no valid JSON found
+ */
+export function extractJsonFromResponse(text) {
+  if (!text || typeof text !== 'string') {
+    return null;
+  }
+
+  // Helper function to attempt JSON parsing with escape sequence normalization
+  function tryParseJson(jsonStr) {
+    // First, try to parse as-is
+    try {
+      return JSON.parse(jsonStr);
+    } catch (e) {
+      // If that fails, check if the LLM over-escaped the content
+      // This is a common issue where LLMs return \\\\n instead of \\n
+
+      // Only attempt normalization if we detect the problematic pattern
+      if (jsonStr.includes('\\\\\\\\')) {
+        // Log the first parse attempt failure for debugging
+        console.warn('Initial JSON parse failed, attempting normalization:', e.message);
+
+        try {
+          // Strategy: The LLM sometimes escapes strings that are already escaped
+          // For example: "content": "text\\\\nmore" should be "content": "text\\nmore"
+          // Replace quadruple backslashes with double (handles over-escaping)
+          let normalized = jsonStr.replace(/\\\\\\\\/g, '\\\\');
+
+          return JSON.parse(normalized);
+        } catch (e2) {
+          // Log this failure too
+          console.warn('Normalized JSON parse also failed:', e2.message);
+          throw e; // Throw original error
+        }
+      } else {
+        // No over-escaping pattern detected, throw original error
+        throw e;
+      }
+    }
+  }
+
+  // Regular expression to find a JSON object within markdown code fences.
+  // It's flexible with or without the 'json' language specifier.
+  const jsonRegex = /```(?:json)?\s*({[\s\S]*?})\s*```/;
+  const match = text.match(jsonRegex);
+
+  // If a fenced JSON block is found, try to parse it.
+  if (match && match[1]) {
+    try {
+      return tryParseJson(match[1]);
+    } catch (e) {
+      // If parsing fails, log the error and fall through to the next method.
+      console.warn('Could not parse the content of a matched JSON block.', e.message);
+    }
+  }
+
+  // Fallback for cases where the AI might not use markdown fences correctly.
+  // Find the first opening brace and the last closing brace.
+  const firstBrace = text.indexOf('{');
+  const lastBrace = text.lastIndexOf('}');
+
+  if (firstBrace !== -1 && lastBrace > firstBrace) {
+    const potentialJson = text.substring(firstBrace, lastBrace + 1);
+    try {
+      return tryParseJson(potentialJson);
+    } catch (e) {
+      // This substring is not valid JSON.
+      console.error('Error parsing JSON extracted in { and }', e);
+    }
+  }
+
+  // If no valid JSON could be extracted by any method, return null.
+  return null;
+}

package/src/llm/providers/gemini-provider.js

@@ -1,6 +1,7 @@
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import { BaseLLMProvider } from './base-provider.js';
 import { LLMServiceException } from '../../llm-service.js';
+import { extractJsonFromResponse } from '../json-utils.js';
 
 export class GeminiProvider extends BaseLLMProvider {
     constructor(config) {
@@ -146,15 +147,27 @@ export class GeminiProvider extends BaseLLMProvider {
             maxOutputTokens: options.maxTokens ?? maxTokens,
         };
 
-        switch (options.responseFormat) {
-            case 'json':
-            case 'json_schema':
+        // Handle responseFormat as an object with type and schema properties
+        if (options.responseFormat) {
+            const formatType = typeof options.responseFormat === 'string'
+                ? options.responseFormat
+                : options.responseFormat.type;
+
+            const schema = typeof options.responseFormat === 'object'
+                ? options.responseFormat.schema
+                : null;
+
+            if (formatType === 'json' || formatType === 'json_schema') {
                 config.responseMimeType = 'application/json';
 
-                if (options.responseSchema) {
-                    config.responseSchema = this._convertToGeminiSchema(options.responseSchema);
+                // CRITICAL: Must provide schema for "Strict Mode" to avoid markdown wrappers
+                if (schema) {
+                    config.responseSchema = this._convertToGeminiSchema(schema);
+                    console.log('[GeminiProvider] Using Strict JSON mode with schema');
+                } else {
+                    console.warn('[GeminiProvider] Using legacy JSON mode without schema - may produce markdown wrappers');
                 }
-                break;
+            }
         }
 
         return config;
@@ -223,12 +236,22 @@ export class GeminiProvider extends BaseLLMProvider {
 
     _safeJsonParse(content) {
         if (!content) return null;
-        try {
-            return JSON.parse(content);
-        } catch (e) {
-            console.warn('[GeminiProvider] Failed to auto-parse JSON response:', e.message);
-            return null;
+
+        // Use the robust JSON extractor that handles:
+        // - Markdown code blocks (```json ... ```)
+        // - Plain JSON objects
+        // - Over-escaped content (\\\\n instead of \\n)
+        // - Brace extraction as fallback
+        const parsed = extractJsonFromResponse(content);
+
+        if (parsed) {
+            console.log('[GeminiProvider] Successfully parsed JSON from response');
+        } else {
+            console.error('[GeminiProvider] Failed to extract valid JSON from response');
+            console.error('[GeminiProvider] Content preview:', content.substring(0, 200));
         }
+
+        return parsed;
     }
 
     async executeTools(tool_calls, messages, tenantId, toolImplementations, env) {

package/src/llm/providers/openai-provider.js

@@ -1,5 +1,6 @@
 import OpenAI from 'openai';
 import { BaseLLMProvider } from './base-provider.js';
+import { extractJsonFromResponse } from '../json-utils.js';
 
 export class OpenAIProvider extends BaseLLMProvider {
     constructor(config) {
@@ -71,20 +72,48 @@ export class OpenAIProvider extends BaseLLMProvider {
     }
 
     _buildResponseFormat(options) {
-        switch (options.responseFormat) {
+        if (!options.responseFormat) {
+            return undefined;
+        }
+
+        // Handle responseFormat as either string or object { type, schema }
+        const formatType = typeof options.responseFormat === 'string'
+            ? options.responseFormat
+            : options.responseFormat.type;
+
+        const schema = typeof options.responseFormat === 'object'
+            ? options.responseFormat.schema
+            : null;
+
+        switch (formatType) {
             case 'json':
-                return { type: 'json_object' };
+                // If schema is provided, use strict mode; otherwise use legacy json_object
+                if (schema) {
+                    console.log('[OpenAIProvider] Using Strict JSON mode with schema');
+                    return {
+                        type: 'json_schema',
+                        json_schema: {
+                            name: options.schemaName || 'response_schema',
+                            strict: options.strictSchema ?? true,
+                            schema: schema
+                        }
+                    };
+                } else {
+                    console.warn('[OpenAIProvider] Using legacy json_object mode without schema - may produce markdown wrappers');
+                    return { type: 'json_object' };
+                }
 
             case 'json_schema':
-                if (!options.responseSchema) {
-                    throw new Error('responseSchema required when using json_schema format');
+                if (!schema) {
+                    throw new Error('schema required when using json_schema format');
                 }
+                console.log('[OpenAIProvider] Using Strict JSON mode with schema');
                 return {
                     type: 'json_schema',
                     json_schema: {
                         name: options.schemaName || 'response_schema',
                         strict: options.strictSchema ?? true,
-                        schema: options.responseSchema
+                        schema: schema
                     }
                 };
 
@@ -99,12 +128,22 @@ export class OpenAIProvider extends BaseLLMProvider {
 
     _safeJsonParse(content) {
         if (!content) return null;
-        try {
-            return JSON.parse(content);
-        } catch (e) {
-            console.warn('[OpenAIProvider] Failed to auto-parse JSON response:', e.message);
-            return null;
+
+        // Use the robust JSON extractor that handles:
+        // - Markdown code blocks (```json ... ```)
+        // - Plain JSON objects
+        // - Over-escaped content (\\\\n instead of \\n)
+        // - Brace extraction as fallback
+        const parsed = extractJsonFromResponse(content);
+
+        if (parsed) {
+            console.log('[OpenAIProvider] Successfully parsed JSON from response');
+        } else {
+            console.error('[OpenAIProvider] Failed to extract valid JSON from response');
+            console.error('[OpenAIProvider] Content preview:', content.substring(0, 200));
         }
+
+        return parsed;
     }
 
     async executeTools(tool_calls, messages, tenantId, toolImplementations, env) {

package/src/llm-service.js

@@ -162,20 +162,27 @@ export class LLMService {
 
     /**
      * Wrap of chatCompletion to handle toolcalls from LLM.
+     * @param {Array} messages - Conversation messages
+     * @param {string} tenantId - Tenant identifier
+     * @param {string} systemPrompt - System instructions
+     * @param {Array} tools - Tools array
+     * @param {Object} options - Options object (for responseFormat, etc.)
+     * @returns {Object} Response with content, tool_calls, and optionally parsedContent
      */
-    async chatWithTools(messages, tenantId, systemPrompt, tools = []) {
+    async chatWithTools(messages, tenantId, systemPrompt, tools = [], options = {}) {
         const provider = await this._getProvider(tenantId);
 
         let currentMessages = [...messages];
 
-        // Initial call
+        // Initial call - pass options to enable JSON mode, etc.
         const initialResponse = await provider.chatCompletion(
             currentMessages,
             systemPrompt,
-            tools
+            tools,
+            options
         );
 
-        let { content, tool_calls } = initialResponse;
+        let { content, tool_calls, parsedContent } = initialResponse;
 
         // Tool execution loop
         while (tool_calls) {
@@ -185,18 +192,21 @@ export class LLMService {
             // Execute tools using the provider's helper (which formats results for that provider)
             await provider.executeTools(tool_calls, currentMessages, tenantId, this.toolImplementations, this.env);
 
-            // Next call
+            // Next call - also pass options
             const nextResponse = await provider.chatCompletion(
                 currentMessages,
                 systemPrompt,
-                tools
+                tools,
+                options
             );
 
             content = nextResponse.content;
             tool_calls = nextResponse.tool_calls;
+            parsedContent = nextResponse.parsedContent; // Preserve parsedContent from final response
         }
 
-        return { content };
+        // Return both content and parsedContent (if available)
+        return { content, parsedContent, toolCalls: tool_calls };
     }
 
     /**