json-object-editor 0.10.664 → 0.10.665

package/CHANGELOG.md

@@ -1,4 +1,10 @@
-## 0.10.663
+## 0.10.665
+
+- AI Response JSON extraction and audit trail improvements
+  - **JSON extraction**: Enhanced `extractJsonText` to handle cases where tool call logs and reasoning text are concatenated before the actual JSON response. The function now finds all JSON objects in the response, filters out tool call logs (objects starting with `{"tool":`), and returns the largest valid JSON object that matches the expected response structure.
+  - **Raw response preservation**: Added `response_raw` field to `ai_response` schema to preserve the complete original response from OpenAI (including tool logs, reasoning text, etc.) for full audit/debugging while `response` contains the cleaned JSON string for merges.
+  - **Merge reliability**: Updated `compareResponseToObject` and `updateObjectFromResponse` methods to prefer `response_json` (already parsed) when available, with fallback to parsing `response` string, reducing double-parsing and improving reliability.
+  - **Response storage**: `executeJOEAiPrompt` now stores both `response_raw` (original) and `response` (cleaned JSON) to maintain complete audit trail while ensuring merge operations work with clean, parseable JSON.## 0.10.663
 
 - Plugin inventory debug page
   - Added `_www/plugins-test.html`, a secured debug page that lists all active plugins, their async/top-level methods, and which apps use them, sourced from the `plugin-utils` plugin.

package/css/joe.css

@@ -1,6 +1,6 @@
 /* --------------------------------------------------------
  *
- *  json-object-editor - v0.10.664
+ *  json-object-editor - v0.10.665
  *  Created by: Corey Hadden
  *
  * -------------------------------------------------------- */

package/js/joe.js

@@ -1,6 +1,6 @@
 /* --------------------------------------------------------
  *
- *  json-object-editor - v0.10.664
+ *  json-object-editor - v0.10.665
  *  Created by: Corey Hadden
  *
  * -------------------------------------------------------- */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "json-object-editor",
-  "version": "0.10.664",
+  "version": "0.10.665",
   "description": "JOE the Json Object Editor | Platform Edition",
   "main": "app.js",
   "scripts": {

package/server/plugins/chatgpt.js

@@ -887,6 +887,7 @@ function shrinkUnderstandObjectMessagesForTokens(messages) {
     // in markdown fences (```json ... ```), and may prepend/append prose. This
     // helper strips fences and tries to isolate the first well-formed JSON
     // object/array substring so JSON.parse has the best chance of succeeding.
+    // Handles cases where tool call logs are concatenated before the actual JSON.
     function extractJsonText(raw) {
         if (!raw) { return ''; }
         let t = String(raw).trim();
@@ -906,19 +907,100 @@ function shrinkUnderstandObjectMessagesForTokens(messages) {
             }
             t = t.trim();
         }
-        // If there's extra prose around the JSON, slice from first {/[ to last }/]
+        
+        // Handle cases where tool call logs (small JSON objects like {"tool":"..."}) 
+        // are concatenated before the actual response JSON (larger JSON object).
+        // Find all JSON objects and pick the largest one that's not a tool log.
+        const jsonCandidates = [];
+        const firstBrace = t.indexOf('{');
+        const firstBracket = t.indexOf('[');
+        const lastBrace = Math.max(t.lastIndexOf('}'), t.lastIndexOf(']'));
+        
+        if (firstBrace === -1 && firstBracket === -1) {
+            return '';
+        }
+        
+        const startPos = (firstBrace === -1) ? firstBracket : 
+                        ((firstBracket === -1) ? firstBrace : Math.min(firstBrace, firstBracket));
+        
+        if (startPos === -1 || lastBrace === -1 || lastBrace <= startPos) {
+            return t.trim();
+        }
+        
+        // Find all potential JSON objects
+        for (let i = startPos; i <= lastBrace; i++) {
+            if (t[i] !== '{' && t[i] !== '[') continue;
+            
+            // Find matching closing brace/bracket
+            let depth = 0;
+            let inString = false;
+            let escape = false;
+            let endPos = -1;
+            
+            for (let j = i; j <= lastBrace; j++) {
+                const char = t[j];
+                if (escape) {
+                    escape = false;
+                    continue;
+                }
+                if (char === '\\') {
+                    escape = true;
+                    continue;
+                }
+                if (char === '"') {
+                    inString = !inString;
+                    continue;
+                }
+                if (!inString) {
+                    if (char === '{' || char === '[') {
+                        depth++;
+                    } else if (char === '}' || char === ']') {
+                        depth--;
+                        if (depth === 0) {
+                            endPos = j;
+                            break;
+                        }
+                    }
+                }
+            }
+            
+            if (endPos !== -1) {
+                const candidate = t.substring(i, endPos + 1);
+                // Skip tool call logs - they match pattern {"tool":"..."}
+                const isToolLog = /^\s*{\s*"tool"\s*:/.test(candidate);
+                try {
+                    JSON.parse(candidate);
+                    jsonCandidates.push({
+                        text: candidate,
+                        length: candidate.length,
+                        isToolLog: isToolLog
+                    });
+                } catch (e) {
+                    // Not valid JSON, skip
+                }
+            }
+        }
+        
+        // Find the largest non-tool-log JSON object, or largest overall if all are tool logs
+        if (jsonCandidates.length > 0) {
+            // Filter out tool logs first
+            const nonToolLogs = jsonCandidates.filter(c => !c.isToolLog);
+            const candidatesToUse = nonToolLogs.length > 0 ? nonToolLogs : jsonCandidates;
+            
+            // Sort by length (descending) and return the largest
+            candidatesToUse.sort((a, b) => b.length - a.length);
+            return candidatesToUse[0].text.trim();
+        }
+        
+        // Fallback: try simple first-to-last extraction
         if (t[0] !== '{' && t[0] !== '[') {
-            const firstBrace = t.indexOf('{');
-            const firstBracket = t.indexOf('[');
-            let first = -1;
-            if (firstBrace === -1) { first = firstBracket; }
-            else if (firstBracket === -1) { first = firstBrace; }
-            else { first = Math.min(firstBrace, firstBracket); }
-            const lastBrace = Math.max(t.lastIndexOf('}'), t.lastIndexOf(']'));
-            if (first !== -1 && lastBrace !== -1 && lastBrace > first) {
-                t = t.slice(first, lastBrace + 1);
+            const first = startPos;
+            const last = lastBrace;
+            if (first !== -1 && last !== -1 && last > first) {
+                t = t.slice(first, last + 1);
             }
         }
+        
         return t.trim();
     }
 
@@ -1357,9 +1439,27 @@ this.executeJOEAiPrompt = async function(data, req, res) {
 
         // const response = await openai.chat.completions.create(payload);
 
+        // Extract JSON from response to strip tool logs, reasoning text, and other non-JSON content.
+        // This is critical for prompts that explicitly require JSON-only output.
+        const rawResponseText = response.output_text || "";
+        let cleanedResponseText = rawResponseText;
+        try {
+            const extractedJson = extractJsonText(rawResponseText);
+            if (extractedJson && extractedJson.trim().length > 0) {
+                // Validate it's actually valid JSON
+                JSON.parse(extractedJson);
+                cleanedResponseText = extractedJson;
+            }
+        } catch (e) {
+            // If extraction fails or JSON is invalid, fall back to raw text
+            // (some prompts may not be JSON-formatted)
+            console.warn('[chatgpt.executeJOEAiPrompt] Failed to extract JSON from response, using raw text:', e.message);
+        }
+
         const saved = await saveAiResponseRefactor({
             prompt,
-            ai_response_content: response.output_text || "",
+            ai_response_content: cleanedResponseText,
+            ai_response_raw: rawResponseText,
             user_prompt: finalInput || '',
             params,
             referenced_object_ids: referencedObjectIds,
@@ -1382,7 +1482,7 @@ this.executeJOEAiPrompt = async function(data, req, res) {
             }
         }catch(_e){}
 
-        return { success: true, ai_response_id: saved._id,response:response.output_text || "",usage:response.usage };
+        return { success: true, ai_response_id: saved._id,response:cleanedResponseText,usage:response.usage };
     } catch (e) {
         console.error('❌ executeJOEAiPrompt error:', e);
         return { error: "Failed to execute AI prompt.",message: e.message };
@@ -1402,7 +1502,7 @@ this.executeJOEAiPrompt = async function(data, req, res) {
             max_tokens: prompt.max_tokens ?? 1200
         };
     }
-    async function saveAiResponseRefactor({ prompt, ai_response_content, user_prompt, params, referenced_object_ids,response_id,usage,user,ai_assistant_id, mcp_enabled, mcp_toolset, mcp_selected_tools, mcp_instructions_mode, mcp_tools_used }) {
+    async function saveAiResponseRefactor({ prompt, ai_response_content, ai_response_raw, user_prompt, params, referenced_object_ids,response_id,usage,user,ai_assistant_id, mcp_enabled, mcp_toolset, mcp_selected_tools, mcp_instructions_mode, mcp_tools_used }) {
         var response_keys = [];
         try {
             response_keys = Object.keys(JSON.parse(ai_response_content));
@@ -1437,6 +1537,7 @@ this.executeJOEAiPrompt = async function(data, req, res) {
             prompt_name: prompt.name,
             prompt_method:prompt.prompt_method,
             response: ai_response_content,
+            response_raw: ai_response_raw || null,
             response_json: parsedResponse,
             response_keys: response_keys,
             response_id:response_id||'',

package/server/schemas/ai_response.js

@@ -30,6 +30,7 @@ var schema = {
             { name:'model_used', type:'string' },
             { name:'response_type', type:'string' },
             { name:'response', type:'string' },
+            { name:'response_raw', type:'string' },
             { name:'response_json', type:'object' },
             { name:'response_keys', type:'string', isArray:true },
             { name:'response_id', type:'string' },