json-object-editor 0.10.657 → 0.10.662

package/server/modules/Sites.js

@@ -48,6 +48,9 @@ function getInclude(req,res,next){
         case 'css':
             res.set('Content-Type', 'text/css');
             break;
+        case 'json':
+            res.set('Content-Type', 'application/json');
+            break;
     }
     res.send(content);
 }
@@ -536,8 +539,31 @@ var routeWithoutSlash = route.startsWith('/') ? route.slice(1) : route;
                 page.content = Editor.wrap(page.content,'jpe-content',{name:'page content'})
             }
 
-            var template = layout.template.replace('${this.PAGE.content}',content.PAGE.content);
-            var html = fillTemplate(template,content);
+            // First, process page content with fillTemplate to resolve variables like ${INCLUDES}, ${this.PAGE.name}, etc.
+            // This is needed because these variables are INSIDE the page content
+            var pageContent = content.PAGE.content || '';
+            pageContent = pageContent.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+            
+            // Preserve newlines before fillTemplate strips them
+            // Replace newlines with a temporary marker that we'll restore later
+            var newlineMarker = '__JOE_NEWLINE__' + Date.now() + '__';
+            var pageContentWithMarkers = pageContent.replace(/\n/g, newlineMarker);
+            
+            // Process page content with fillTemplate to resolve template variables
+            var processedPageContent = fillTemplate(pageContentWithMarkers, content);
+            
+            // Restore newlines after fillTemplate processing
+            processedPageContent = processedPageContent.replace(new RegExp(newlineMarker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '\n');
+            
+            // Use a placeholder for ${this.PAGE.content} in layout template
+            var placeholder = '__JOE_PAGE_CONTENT_PLACEHOLDER__' + Date.now() + '__';
+            var template = layout.template.replace(/\$\{this\.PAGE\.content\}/g, placeholder);
+            
+            // Process layout template with fillTemplate (which will strip newlines from layout, but our placeholder remains)
+            var html = fillTemplate(template, content);
+            
+            // Replace placeholder with processed page content (which has newlines preserved)
+            html = html.replace(new RegExp(placeholder.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), processedPageContent);
             if(editor){
                 html = html.replace('</head>',Editor.styles+'</head>')
                 .replace('</body>',Editor.gui(layout)+Editor.gui(page)+Editor.menu

package/server/plugins/chatgpt.js

@@ -19,7 +19,28 @@ function ChatGPT() {
     var self = this;
     this.async ={};
     function coloredLog(message){
-        console.log(JOE.Utils.color('[chatgpt]', 'plugin', false), message);
+        try{
+            // Only emit verbose plugin logs in non‑production environments.
+            // This keeps consoles clean in production while preserving rich
+            // traces (assistant resolution, MCP config, systemText) during
+            // local/dev debugging.
+            var env = null;
+            if (typeof JOE !== 'undefined' && JOE && JOE.webconfig && JOE.webconfig.env){
+                env = JOE.webconfig.env;
+            } else if (typeof process !== 'undefined' && process.env && process.env.NODE_ENV){
+                env = process.env.NODE_ENV;
+            }
+            if (env && env.toLowerCase() === 'production'){
+                return;
+            }
+            console.log(JOE.Utils.color('[chatgpt]', 'plugin', false), message);
+        }catch(_e){
+            // If anything goes wrong determining env, default to logging so
+            // that development debugging is not silently broken.
+            try{
+                console.log('[chatgpt]', message);
+            }catch(__e){}
+        }
     }
     //xx -setup and send a test prompt to chatgpt
     //xx get the api key from joe settings
@@ -58,6 +79,23 @@ function ChatGPT() {
         const summary = JOE.Schemas && JOE.Schemas.summary && JOE.Schemas.summary[name];
         return { full, summary };
     }
+    function buildMcpToolsFromConfig(cfg) {
+        if (!cfg || !cfg.mcp_enabled) {
+            return { tools: null, names: [] };
+        }
+        try {
+            const names = MCP.getToolNamesForToolset(
+                cfg.mcp_toolset || 'read-only',
+                Array.isArray(cfg.mcp_selected_tools) ? cfg.mcp_selected_tools : null
+            );
+            const defs = MCP.getToolDefinitions(names);
+            return { tools: defs, names: names };
+        } catch (e) {
+            console.warn('[chatgpt] buildMcpToolsFromConfig failed', e);
+            return { tools: null, names: [] };
+        }
+    }
+
     /**
      * callMCPTool
      *
@@ -270,29 +308,32 @@ function shrinkUnderstandObjectMessagesForTokens(messages) {
         const attachmentsMode = opts.attachments_mode || null;
         const openaiFileIds = opts.openai_file_ids || null;
 
-        // Normalize tools: in many schemas tools may be stored as a JSON string;
-        // here we accept either an array or a JSON-stringified array.
-        let tools = null;
+        // Debug/trace: log the effective system instructions going into this
+        // Responses+tools call. This helps verify assistant + MCP instructions
+        // wiring across prompts, assists, autofill, and widget chat.
+        try{
+            coloredLog('runWithTools systemText:\n' + systemText);
+        }catch(_e){}
+
+        // Normalize tools: manual assistant.tools plus optional MCP tools
+        let manualTools = null;
         if (assistant && assistant.tools) {
             if (Array.isArray(assistant.tools)) {
-                tools = assistant.tools;
+                manualTools = assistant.tools;
             } else if (typeof assistant.tools === 'string') {
                 try {
                     const parsed = JSON.parse(assistant.tools);
                     if (Array.isArray(parsed)) {
-                        tools = parsed;
+                        manualTools = parsed;
                     }
                 } catch (e) {
                     console.error('[chatgpt] Failed to parse assistant.tools JSON', e);
                 }
             }
         }
-        // Normalize tool definitions for the Responses API. The assistant UI
-        // uses the Assistants-style shape ({ type:'function', function:{...} }),
-        // but Responses expects the name/description/parameters at the top level:
-        //   { type:'function', name:'x', description:'...', parameters:{...} }
-        if (Array.isArray(tools)) {
-            tools = tools.map(function (t) {
+        // Flatten any Assistants-style function definitions
+        if (Array.isArray(manualTools)) {
+            manualTools = manualTools.map(function (t) {
                 if (t && t.type === 'function' && t.function && !t.name) {
                     const fn = t.function || {};
                     return {
@@ -306,6 +347,26 @@ function shrinkUnderstandObjectMessagesForTokens(messages) {
             });
         }
 
+        // Merge manual tools with MCP tools (manual wins on name collisions).
+        let tools = null;
+        const mergedByName = {};
+        const mcp = buildMcpToolsFromConfig(assistant || {});
+        const mcpTools = Array.isArray(mcp.tools) ? mcp.tools : null;
+
+        if (Array.isArray(manualTools) || Array.isArray(mcpTools)) {
+            tools = [];
+            (manualTools || []).forEach(function(t){
+                if (!t) { return; }
+                if (t.name) { mergedByName[t.name] = true; }
+                tools.push(t);
+            });
+            (mcpTools || []).forEach(function(t){
+                if (!t || !t.name) { return; }
+                if (mergedByName[t.name]) { return; }
+                tools.push(t);
+            });
+        }
+
         // No tools configured – do a simple single Responses call.
         if (!tools) {
             const resp = await openai.responses.create({
@@ -1096,6 +1157,24 @@ this.executeJOEAiPrompt = async function(data, req, res) {
             return { error: "Prompt not found." };
         }
 
+        // If this prompt run is associated with a JOE ai_assistant, log which
+        // assistant is being used so we can debug "which agent handled this?"
+        try{
+            const aiAssistantId = data.ai_assistant_id || null;
+            if (aiAssistantId) {
+                let asst = null;
+                try{
+                    // Prefer explicit ai_assistant schema lookup, then fallback
+                    // to a generic get in case datasets are flattened.
+                    asst = $J.get(aiAssistantId,'ai_assistant') || $J.get(aiAssistantId);
+                }catch(_e){}
+                const label = asst && (asst.name || asst.title || asst.info || asst._id) || aiAssistantId;
+                coloredLog('[prompt] executeJOEAiPrompt using ai_assistant: '
+                    + label + ' [' + aiAssistantId + ']'
+                    + ' for prompt: ' + (prompt.name || prompt._id));
+            }
+        }catch(_e){}
+
         let instructions = prompt.instructions || "";
         let finalInstructions=instructions;
         let finalInput='';
@@ -1474,9 +1553,15 @@ this.executeJOEAiPrompt = async function(data, req, res) {
                 system: system,
                 messages: [],
                 source: body.source || "widget",
+                // Optional scope for object-scoped widget chats
+                scope_itemtype: body.scope_itemtype || null,
+                scope_id: body.scope_id || null,
                 created: new Date().toISOString(),
                 joeUpdated: new Date().toISOString()
             };
+            if (body.name && !convo.name) {
+                convo.name = String(body.name);
+            }
 
             const saved = await new Promise(function (resolve, reject) {
                 // Widget conversations are lightweight and do not need full history diffs.
@@ -1598,7 +1683,76 @@ this.executeJOEAiPrompt = async function(data, req, res) {
                 return { success: false, error: "Conversation not found" };
             }
 
+            // Best-effort: if this is an object-scoped conversation and we have
+            // not yet attached any files, walk the scoped object for uploader
+            // style files that have OpenAI ids and cache them on the convo.
+            try{
+                if ((!convo.attached_openai_file_ids || !convo.attached_openai_file_ids.length) &&
+                    convo.scope_itemtype && convo.scope_id) {
+                    var scopedObj = null;
+                    try{
+                        scopedObj = $J.get(convo.scope_id, convo.scope_itemtype) || $J.get(convo.scope_id);
+                    }catch(_e){}
+                    if (scopedObj && typeof scopedObj === 'object') {
+                        var ids = [];
+                        var meta = [];
+                        Object.keys(scopedObj).forEach(function(field){
+                            var val = scopedObj[field];
+                            if (!Array.isArray(val)) { return; }
+                            val.forEach(function(f){
+                                if (f && f.openai_file_id) {
+                                    ids.push(f.openai_file_id);
+                                    meta.push({
+                                        itemtype: scopedObj.itemtype || convo.scope_itemtype,
+                                        field: field,
+                                        name: f.filename || '',
+                                        role: f.file_role || null,
+                                        openai_file_id: f.openai_file_id
+                                    });
+                                }
+                            });
+                        });
+                        if (ids.length) {
+                            convo.attached_openai_file_ids = ids;
+                            convo.attached_files_meta = meta;
+                        }
+                    }
+                }
+            }catch(_e){ /* non-fatal */ }
+
             convo.messages = normalizeMessages(convo.messages);
+
+            // On the very first turn of an object-scoped widget conversation,
+            // pre-load a slimmed understandObject snapshot so the assistant
+            // immediately knows which record "this client/task/..." refers to
+            // without having to remember to call MCP. We keep this snapshot
+            // concise via slimUnderstandObjectResult and only inject it once.
+            try{
+                var isObjectChat = (convo.source === 'object_chat') && convo.scope_id;
+                var hasMessages = Array.isArray(convo.messages) && convo.messages.length > 0;
+                if (isObjectChat && !hasMessages){
+                    const uo = await callMCPTool('understandObject', {
+                        _id: convo.scope_id,
+                        itemtype: convo.scope_itemtype || undefined,
+                        depth: 1,
+                        slim: true
+                    }, { req });
+                    const slimmed = slimUnderstandObjectResult(uo);
+                    if (slimmed) {
+                        convo.messages = convo.messages || [];
+                        convo.messages.push({
+                            role: 'system',
+                            content: JSON.stringify({
+                                tool: 'understandObject',
+                                scope_object: slimmed
+                            })
+                        });
+                    }
+                }
+            }catch(_e){
+                console.warn('[chatgpt] widgetMessage understandObject preload failed', _e && _e.message || _e);
+            }
+
             const nowIso = new Date().toISOString();
 
             // Append user message
@@ -1630,37 +1784,163 @@ this.executeJOEAiPrompt = async function(data, req, res) {
                 }
             }
 
-            const assistantId = body.assistant_id || convo.assistant_id || null;
-            // NOTE: assistantId here is the OpenAI assistant_id, not the JOE cuid.
-            // We do NOT pass assistant_id to the Responses API (it is not supported in the
-            // version we are using); instead we look up the JOE ai_assistant by assistant_id
-            // and inject its configuration (model, instructions, tools) into the request.
-            var assistantObj = null;
-            if (assistantId && JOE && JOE.Data && Array.isArray(JOE.Data.ai_assistant)) {
-                assistantObj = JOE.Data.ai_assistant.find(function (a) {
-                    return a && a.assistant_id === assistantId;
+        // Resolve the JOE ai_assistant driving this conversation. We support
+        // both the modern flow (ai_assistant_id / convo.assistant, which are
+        // JOE cuid references) and the legacy OpenAI Assistants flow
+        // (assistant_id / convo.assistant_id, which are OpenAI ids).
+        var assistantObj = null;
+        var joeAssistantId = body.ai_assistant_id || convo.assistant || null; // JOE cuid
+        if (joeAssistantId) {
+            try{
+                // Prefer a direct lookup via the ai_assistant schema, but fall
+                // back to scanning the in-memory dataset if needed. In some
+                // server contexts $J.get may not be wired for ai_assistant yet,
+                // while JOE.Data.ai_assistant is available.
+                assistantObj = $J.get(joeAssistantId,'ai_assistant') || $J.get(joeAssistantId) || null;
+            }catch(_e){}
+            if (!assistantObj && JOE && JOE.Data && Array.isArray(JOE.Data.ai_assistant)) {
+                assistantObj = JOE.Data.ai_assistant.find(function(a){
+                    return a && (a._id === joeAssistantId);
                 }) || null;
             }
+        }
+        const assistantId = body.assistant_id || convo.assistant_id || (assistantObj && assistantObj.assistant_id) || null;
+        // Legacy fallback: if we only have an OpenAI assistant_id, try to
+        // locate the JOE ai_assistant by that id.
+        if (!assistantObj && assistantId && JOE && JOE.Data && Array.isArray(JOE.Data.ai_assistant)) {
+            assistantObj = JOE.Data.ai_assistant.find(function (a) {
+                return a && a.assistant_id === assistantId;
+            }) || null;
+        }
+
+        // Log which ai_assistant (if any) is being used for this widget
+        // conversation so we can easily confirm the active agent when
+        // debugging object chat vs AI Hub behavior.
+        try{
+            coloredLog('[widget] assistant resolution: '
+                + 'convo.assistant=' + String(convo.assistant || '')
+                + ' convo.assistant_id=' + String(convo.assistant_id || '')
+                + ' body.ai_assistant_id=' + String(body.ai_assistant_id || '')
+                + ' body.assistant_id=' + String(body.assistant_id || '')
+                + ' resolvedJoe=' + String(assistantObj && assistantObj._id || ''));
+            if (assistantObj) {
+                var asstLabel = assistantObj.name || assistantObj.title || assistantObj.info || assistantObj._id || assistantId;
+                coloredLog('[widget] widgetMessage using ai_assistant: '
+                    + asstLabel + ' [' + (assistantObj._id || assistantId || '') + ']'
+                    + ' source=' + String(convo.source || 'widget')
+                    + ' convo=' + String(convo._id || ''));
+            } else if (assistantId) {
+                coloredLog('[widget] widgetMessage assistant_id (no JOE ai_assistant found): '
+                    + assistantId
+                    + ' source=' + String(convo.source || 'widget')
+                    + ' convo=' + String(convo._id || ''));
+            }
+        }catch(_e){}
+
             const openai = newClient();
             const model = (assistantObj && assistantObj.ai_model) || convo.model || body.model || "gpt-5.1";
 
             // Prefer explicit system text on the conversation, then assistant instructions.
-            const systemText = (convo.system && String(convo.system)) ||
+            const baseSystemText = (convo.system && String(convo.system)) ||
                 (assistantObj && assistantObj.instructions) ||
                 "";
+
+            // When this conversation was launched from an object ("Start Chat"
+            // on a record), include a small scope hint so the assistant knows
+            // which object id/itemtype to use with MCP tools like
+            // understandObject/search. We keep this concise to avoid
+            // unnecessary tokens but still make the scope unambiguous.
+            let systemText = baseSystemText;
+            try{
+                if (convo.source === 'object_chat' && convo.scope_id) {
+                    const scopeLine = '\n\n---\nJOE scope_object:\n'
+                        + '- itemtype: ' + String(convo.scope_itemtype || 'unknown') + '\n'
+                        + '- _id: ' + String(convo.scope_id) + '\n'
+                        + 'When you need this object\'s details, call the MCP tool "understandObject" '
+                        + 'with these identifiers, or search for related records using the MCP search tools.\n';
+                    systemText = (baseSystemText || '') + scopeLine;
+                }
+            }catch(_e){ /* non-fatal */ }
+
+            // Append MCP tool instructions for assistants that have MCP enabled,
+            // using the same helper as other MCP-aware surfaces.
+            try{
+                if (assistantObj && assistantObj.mcp_enabled) {
+                    const mcpCfg = {
+                        mcp_enabled: assistantObj.mcp_enabled,
+                        mcp_toolset: assistantObj.mcp_toolset,
+                        mcp_selected_tools: assistantObj.mcp_selected_tools,
+                        mcp_instructions_mode: assistantObj.mcp_instructions_mode || 'auto'
+                    };
+                    const mcp = buildMcpToolsFromConfig(mcpCfg);
+                    coloredLog('[widget] MCP config for assistant '
+                        + String(assistantObj._id || '') + ': enabled=' + String(mcpCfg.mcp_enabled)
+                        + ' toolset=' + String(mcpCfg.mcp_toolset || '')
+                        + ' names=' + JSON.stringify(mcp.names || []));
+                    if (mcp.names && mcp.names.length) {
+                        const txt = MCP.buildToolInstructions(mcp.names, mcpCfg.mcp_instructions_mode || 'auto');
+                        if (txt) {
+                            coloredLog('[widget] appending MCP instructions block to systemText');
+                            systemText = (systemText || '') + '\n\n' + txt;
+                        }
+                    }
+                } else {
+                    coloredLog('[widget] MCP disabled for this assistant or assistant missing; no MCP block appended.');
+                }
+            }catch(_e){ /* non-fatal */ }
+
+            // Build the messages array for the model. We deliberately separate
+            // the stored `convo.messages` from the model-facing payload so we
+            // can annotate the latest user turn with uploaded_files metadata
+            // without altering the persisted history.
             const messagesForModel = convo.messages.map(function (m) {
                 return { role: m.role, content: m.content };
             });
+            // If we have attached file metadata, wrap the latest user turn in a
+            // small JSON envelope so the model can see which files exist and how
+            // they are labeled (role, name, origin field) while still receiving
+            // the raw user input as `input`.
+            try{
+                if (convo.attached_files_meta && convo.attached_files_meta.length && messagesForModel.length) {
+                    var lastMsg = messagesForModel[messagesForModel.length - 1];
+                    if (lastMsg && lastMsg.role === role && typeof lastMsg.content === 'string') {
+                        lastMsg.content = JSON.stringify({
+                            uploaded_files: convo.attached_files_meta,
+                            input: lastMsg.content
+                        }, null, 2);
+                    }
+                }
+            }catch(_e){ /* non-fatal */ }
+
+            // Collect OpenAI file ids from scoped object attachments and any
+            // assistant-level files so they are available to the model via the
+            // shared attachFilesToResponsesPayload helper inside runWithTools.
+            var openaiFileIds = [];
+            if (Array.isArray(convo.attached_openai_file_ids) && convo.attached_openai_file_ids.length){
+                openaiFileIds = openaiFileIds.concat(convo.attached_openai_file_ids);
+            }
+            try{
+                if (assistantObj && Array.isArray(assistantObj.assistant_files)) {
+                    assistantObj.assistant_files.forEach(function(f){
+                        if (f && f.openai_file_id) {
+                            openaiFileIds.push(f.openai_file_id);
+                        }
+                    });
+                }
+            }catch(_e){}
 
             // Use runWithTools so that, when an assistant has tools configured,
-            // we let the model call those tools via MCP before generating a
-            // final response.
+            // we let the model call those tools via MCP / function tools before
+            // generating a final response. Attach any discovered OpenAI files
+            // so the model can read from them as needed.
             const runResult = await runWithTools({
                 openai: openai,
                 model: model,
                 systemText: systemText,
                 messages: messagesForModel,
                 assistant: assistantObj,
+                attachments_mode: (body.attachments_mode || 'direct'),
+                openai_file_ids: openaiFileIds.length ? openaiFileIds : null,
                 req: req
             });
 

package/server/plugins/formBuilder.js

@@ -335,6 +335,104 @@ function FormBuilder(){
         }
     }//end bodycontent
 
+    /**
+     * Get form definition JSON from an include
+     * GET /API/plugin/formBuilder/definition?include_id={id}
+     * GET /API/plugin/formBuilder/definition?formid={form_id}&field=json_definition_include
+     * GET /API/plugin/formBuilder/definition?formid={form_id}&pageid={page_id} (auto-finds JSON from page includes)
+     */
+    this.definition = function(data, req, res) {
+        var includeId = data.include_id;
+        var formId = data.formid;
+        var pageId = data.pageid;
+        var includeField = data.field || 'json_definition_include';
+        
+        // If formid provided, look up the include from form's meta, page includes, or specified field
+        if (formId && !includeId) {
+            var form = JOE.Cache.findByID('form', formId);
+            if (!form) {
+                return res.status(404).json({errors: 'form not found', failedat: 'formBuilder'});
+            }
+            
+            // First try: Check if form has a reference to include in meta or a direct field
+            if (form.meta && form.meta[includeField]) {
+                includeId = form.meta[includeField];
+            } else if (form[includeField]) {
+                includeId = form[includeField];
+            } 
+            // Second try: If pageid provided, find JSON include from page's includes
+            else if (pageId) {
+                var page = JOE.Cache.findByID('page', pageId);
+                if (page && page.includes && Array.isArray(page.includes)) {
+                    for (var i = 0; i < page.includes.length; i++) {
+                        var inc = JOE.Cache.findByID('include', page.includes[i]);
+                        if (inc && inc.filetype === 'json') {
+                            includeId = inc._id;
+                            break;
+                        }
+                    }
+                }
+            }
+            
+            if (!includeId) {
+                return res.status(404).json({errors: 'form does not have json definition include reference. Set it in form.meta.json_definition_include or add a JSON include to the page', failedat: 'formBuilder'});
+            }
+        }
+        
+        if (!includeId) {
+            return res.status(400).json({errors: 'include_id or formid required', failedat: 'formBuilder'});
+        }
+        
+        var include = JOE.Cache.findByID('include', includeId);
+        if (!include) {
+            return res.status(404).json({errors: 'include not found', failedat: 'formBuilder'});
+        }
+        
+        if (include.filetype !== 'json') {
+            return res.status(400).json({errors: 'include is not a JSON file', failedat: 'formBuilder'});
+        }
+        
+        var content = include.content;
+        
+        // If fillTemplate is enabled, process template variables
+        if (include.fillTemplate) {
+            var payload = {
+                WEBCONFIG: JOE.webconfig,
+                SETTING: JOE.Cache.settings,
+                REQUEST: req,
+                FORM: formId ? JOE.Cache.findByID('form', formId) : null
+            };
+            try {
+                var fillTemplate = require('../modules/Sites.js').fillTemplate || function(str, data) {
+                    return str.replace(/\$\{this\.([^}]+)\}/g, function(match, path) {
+                        var parts = path.split('.');
+                        var val = data;
+                        for (var i = 0; i < parts.length; i++) {
+                            if (val && typeof val === 'object') {
+                                val = val[parts[i]];
+                            } else {
+                                return match;
+                            }
+                        }
+                        return val != null ? val : match;
+                    });
+                };
+                content = fillTemplate(content, payload);
+            } catch(e) {
+                console.error('[formBuilder.definition] template fill error:', e);
+            }
+        }
+        
+        // Parse and return JSON
+        try {
+            var jsonData = typeof content === 'string' ? JSON.parse(content) : content;
+            res.set('Content-Type', 'application/json');
+            return res.json(jsonData);
+        } catch(e) {
+            return res.status(500).json({errors: 'invalid JSON: ' + e.message, failedat: 'formBuilder'});
+        }
+    };
+    
     return self; 
 }
 module.exports = new FormBuilder();