@xuda.io/ai_module 1.1.5617 → 1.1.5618

package/un-used.mjs

@@ -0,0 +1,1437 @@
+// const add_data_to_table_tool = tool({
+//   name: 'Add data to table tool',
+//   description: 'Add data to a table by receiving a table_id and a data object containing key–value pairs representing the fields and their values.',
+//   parameters: z.object({
+//     table_id: z.string(),
+//   }),
+//   async execute(e, RunContext) {
+//     const { table_id } = e;
+//     const { app_id, uid } = RunContext.context;
+
+//     if (!table_id) throw new Error('got null in the table_id parameter');
+
+//     try {
+//       const project_db = await get_project_db(app_id);
+
+//       let response = '';
+
+//       try {
+//         response = await project_db.get(table_id);
+//       } catch (error) {
+//         throw new Error(`table doc ${table_id} not found`);
+//       }
+
+//       if (_.isEmpty(response)) {
+//         throw new Error(`table with table_id ${table_id} not found`);
+//       }
+
+//       return response;
+//     } catch (err) {
+//       return err.message;
+//     }
+//   },
+// });
+
+// let table_tools = {};
+// const build_studio_table_tools = async function (app_id) {
+//   try {
+//     if (!table_tools[app_id]) {
+//       table_tools[app_id] = {};
+//     }
+
+//     const project_db = await get_project_db(app_id);
+//     let tables = await project_db.view('xuda', 'studio_docs_by_type', { key: 'table' });
+
+//     for (const table_header_doc of tables.rows) {
+//       try {
+//         const doc = await project_db.get(table_header_doc.id);
+
+//         let data_parameters = {};
+
+//         for (const field of doc.tableFields || []) {
+//           const fieldId = field.data.field_id;
+//           const fieldType = field.props.fieldType;
+//           const mandatory = field.props.mandatory;
+//           const comment = field.props.fieldComment;
+
+//           let zodType;
+//           switch (fieldType) {
+//             case 'string':
+//               zodType = z.string();
+//               break;
+//             case 'number':
+//               zodType = z.number();
+//               break;
+//             case 'array':
+//               zodType = z.array(z.any());
+//               break;
+//             case 'object':
+//               zodType = z.object({});
+//               break;
+//             default:
+//               zodType = z.any();
+//           }
+
+//           // Add description if comment exists
+//           if (comment) {
+//             zodType = zodType.describe(comment);
+//           }
+
+//           // Make optional if not mandatory
+//           if (!mandatory) {
+//             zodType = zodType.optional();
+//           }
+
+//           data_parameters[fieldId] = zodType;
+//         }
+
+//         const _tool = {
+//           name: `add data to table tool`,
+//           description: `Add data to ${doc.properties.menuName} table by receiving key–value pairs representing the fields and their values.`,
+//           parameters: z.object(data_parameters),
+//           // parameters: z.object({ ...{ table_id: z.string() }, ...data_parameters }),
+//           // parameters: z.object({
+//           //   table_id: z.string(),
+//           //   data: z.object(data_parameters),
+//           // }),
+//           async execute(e, RunContext) {
+//             debugger;
+//             const {} = e;
+//             const { app_id, uid, table_id } = RunContext.context;
+//             let response = '';
+//             try {
+//               if (!table_id) throw new Error('got null in the table_id parameter');
+
+//               const FlexibleDatabaseSchema = z.object({
+//                 _id: z.string(),
+//                 _rev: z.string(),
+//                 docType: z.literal('database'),
+//                 stat: z.number(),
+//                 udfData: z.object({
+//                   udffileid: z.string(),
+//                   data: z.record(z.union([z.string(), z.number()])), // Any key-value pairs
+//                   error: z.object({}).optional(),
+//                 }),
+//                 ts: z.number(),
+//                 date: z.number(),
+//                 udfIndex: z.array(
+//                   z.object({
+//                     fileId: z.string(),
+//                     indexId: z.string(),
+//                     keySegments: z.array(z.any()),
+//                     keyValue: z.array(z.any()),
+//                   }),
+//                 ),
+//               });
+
+//               // for (const [key, val] of Object.entities(data || {})) {
+//               // }
+
+//               let udfIndex = [];
+
+//               for (const [key, val] of Object.entities(doc.tableIndexes || [])) {
+//                 let keyValue = [];
+//                 for (const key_val of val?.data?.keys || []) {
+//                   keyValue.push(data[key_val]);
+//                 }
+
+//                 udfIndex.push({
+//                   fileId: table_id,
+//                   indexId: val.id,
+//                   keySegments: val?.data?.keys || [],
+//                   keyValue,
+//                 });
+//               }
+//               const t = Date.now();
+//               let table_obj = {
+//                 _id: 'dbs-' + crypto.randomUUID(),
+
+//                 docType: 'database',
+//                 stat: 3,
+//                 udfData: {
+//                   udffileid: table_id,
+//                   data: e,
+//                   error: {},
+//                 },
+//                 ts: t,
+//                 date: t,
+//                 udfIndex,
+//               };
+
+//               return response;
+//             } catch (err) {
+//               return err.message;
+//             }
+//           },
+//         };
+
+//         // const jsonSchema = zodToJsonSchema(_tool.parameters, 'UserSchema');
+
+//         // console.log(JSON.stringify(jsonSchema, null, 2));
+
+//         table_tools[app_id][doc._id] = tool(_tool);
+//       } catch (error) {
+//         throw new Error(`table doc ${doc.id} not found`);
+//       }
+//     }
+//     // console.log(studio_units);
+//   } catch (err) {
+//     console.error(err);
+//   }
+
+//   return table_tools[app_id];
+// };
+
+const get_mock_data_generator_agent = function (app_id, table_id) {
+  return new Agent({
+    name: `Mock Data Generator Agent`,
+    instructions: `You are a Mock Data Generator Assistant for table id ${table_id}. Help the user populate data for table id ${table_id} with sample data. First, retrieve the table field definitions using get_studio_table_info_by_table_id_tool. Then generate five realistic mock data samples based on the table structure and insert each one individually using add data to table tool.`,
+    // instructions: `You are a Mock Data Generator Assistant.
+    // 1. First, get table info using the get_studio_table_info_by_table_id tool with table_id: ${table_id}
+    // 2. Analyze the table structure and field types
+    // 3. Generate 5 realistic mock data samples based on the fields
+    // 4. Return the generated data in a structured format`,
+    model,
+    tools: [get_studio_table_info_by_table_id_tool, table_tools[app_id][table_id]], //, table_tools[app_id][table_id]
+    // modelSettings: { toolChoice: 'required' },
+  });
+};
+
+// const get_add_data_to_table_agent = function (app_id, table_id) {
+//   return new Agent({
+//     name: `Add Data to Table Agent`,
+//     instructions: `You are an Add Data Assistant for table ID ${table_id}. Your job is to help the user provide valid data for this table. First, retrieve the table field definitions using get_studio_table_info_by_table_id_tool. Then prepare the data as key–value pairs (e.g., { first_name: "John", last_name: "Smith" }) and submit it using the add_data_to_table_tool. Generate UUID values for required unique fields when necessary. Ignore extra fields from the user prompt, and skip optional fields if not provided.`,
+//     model,
+//     tools: [get_studio_table_info_by_table_id_tool, table_tools[app_id][table_id]],
+//     modelSettings: { toolChoice: 'required' },
+//   });
+// };
+
+// const add_data_to_table = tool({
+//   name: 'Create table agent',
+//   description: 'crete agent that handles table tools by providing table_id',
+//   parameters: z.object({
+//     table_id: z.string(),
+//   }),
+//   async execute(e, RunContext) {
+//     const { table_id } = e;
+//     const { app_id, uid } = RunContext.context;
+//     let response = '';
+//     try {
+//       if (!table_id) throw new Error('got null in the table_id parameter');
+//       let table_doc = {};
+//       const project_db = await get_project_db(app_id);
+//       try {
+//         table_doc = await project_db.get(table_id);
+//       } catch (error) {
+//         throw new Error(`table doc ${table_id} not found`);
+//       }
+
+//       const agent_name = `${table_doc.properties.menuName} table AI Agent`;
+
+//       try {
+//         const ai_agents_ret = await get_ai_agents({ uid });
+//         for (const doc of ai_agents_ret?.data?.docs || []) {
+//           if (doc.properties.menuName === table_doc.properties.menuName) {
+//             throw new Error(`ai agent for ${table_id} already exist with name ${agent_name}`);
+//           }
+//         }
+//       } catch (err) {
+//         debugger;
+//         console.error(err);
+//         // throw new Error(err.message);
+//       }
+
+//       const agentConfig = {
+//         agent_name,
+//         agent_tools: [
+//           {
+//             type: 'table',
+//             prog_id: table_id,
+//           },
+//         ],
+//         agent_instructions: 'You are a Table Data Assistant. Your role is to query, analyze, and modify data within tables as requested. You can retrieve specific information, perform calculations, filter or sort records, and apply updates or structural changes to the data when necessary.',
+//       };
+
+//       response = await create_ai_agent({
+//         uid,
+//         agentConfig,
+//       });
+
+//       return { response, agentConfig };
+//     } catch (err) {
+//       return err.message;
+//     }
+//   },
+// });
+
+// const get_studio_tables = tool({
+//   name: 'Get studio tables',
+//   description: 'fetch all studio tables',
+//   parameters: z.object({
+//     app_id: z.string(),
+//   }),
+//   async execute(e, RunContext) {
+//     const { app_id } = e;
+//     try {
+//       const project_db = await get_project_db(app_id);
+//       let tables = await project_db.view('xuda', 'studio_docs_by_type', { key: 'table', include_docs: true });
+
+//       return tables;
+//     } catch (err) {
+//       return 'unknown';
+//     }
+//   },
+// });
+
+const get_plugins = tool({
+  name: 'Get plugins',
+  description: 'I am responsible to return all available plugins',
+  parameters: z.object({}),
+  async execute(e, RunContext) {
+    try {
+      const db = get_xuda_couch('xuda_plugins');
+      const plugins_res = await db.view('xuda', 'all-active-plugins', {});
+      let data = [];
+      if (!_.isEmpty(plugins_res.rows)) {
+        for (val of plugins_res.rows) {
+          if (!['library', 'widget', 'ui'].includes(val.plugin_type)) continue;
+          var plugin = _.clone(val.value);
+          delete plugin.account_id;
+          delete plugin.manifest;
+          delete plugin.plugin_name;
+          delete plugin.setup;
+          delete plugin.stat;
+          delete plugin.updated_date;
+          plugin._id = plugin._id.replace('@xuda.io/', '');
+
+          data.push(plugin);
+        }
+      }
+
+      console.log('get_plugins >>>> ', data);
+      return data;
+    } catch (err) {
+      console.error(err);
+      return err.message;
+    }
+  },
+});
+
+const update_app_doc_tool = tool({
+  name: 'update app doc tool',
+  description: 'update any app doc query by _id and _rev and save to the app db',
+  parameters: z.object({
+    app_id: z.string(),
+    doc: z.object({ _id: z.string(), _rev: z.string() }),
+  }),
+  async execute(e, RunContext) {
+    console.log(`SAVED ${e.app_id}`);
+    return `OK`;
+  },
+});
+
+const get_app_doc_tool = tool({
+  name: 'get app doc tool',
+  description: 'get any doc by id from the app db',
+  parameters: z.object({
+    app_id: z.string(),
+    _id: z.string(),
+  }),
+  async execute(e, RunContext) {
+    try {
+      const project_db = await get_project_db(e.app_id);
+
+      const result = await project_db.get(_id);
+      console.log('get_app_doc_tool >>>> ', result);
+      return result;
+    } catch (err) {
+      console.error(err);
+      return err.message;
+    }
+  },
+});
+
+const get_additional_information_tool = tool({
+  name: 'get additional information tool',
+  description: 'get additional information for object unit tool',
+  parameters: z.object({
+    object_unit: z.enum(['table']),
+  }),
+  async execute(e, RunContext) {
+    let obj = { table: ['studio_table'] };
+    const db = get_xuda_couch('xuda_website');
+    try {
+      if (!obj[e.object_unit]) throw new Error(`${e.object_unit} not found`);
+      const doc = await db.get(obj[e.object_unit]);
+      // console.log(`INFO ${doc}`);
+      return doc.md || doc;
+    } catch (err) {
+      console.error(err);
+      return err.message;
+    }
+  },
+});
+
+async function fineTuneWithMultipleFiles(datasetPaths, baseModel = 'gpt-4o-mini') {
+  try {
+    console.log('📤 Uploading datasets...');
+
+    // Step 1. Upload all JSONL files in parallel
+    const uploadedFiles = await Promise.all(
+      datasetPaths.map(async (filePath) => {
+        const file = await openai.files.create({
+          file: fs.createReadStream(path.resolve(filePath)),
+          purpose: 'fine-tune',
+        });
+        console.log(`✅ Uploaded: ${file.filename} → ID: ${file.id}`);
+        return file.id;
+      }),
+    );
+
+    // Step 2. Create fine-tuning job with multiple training files
+    console.log(`🚀 Creating fine-tuning job for ${baseModel}...`);
+    const fineTuneJob = await openai.fineTuning.jobs.create({
+      training_files: uploadedFiles.map((id) => ({ file_id: id })), // <-- multiple files here
+      model: baseModel,
+    });
+
+    console.log(`📌 Fine-tuning job created: ${fineTuneJob.id}`);
+
+    // Step 3. Poll job status until training completes
+    let status = fineTuneJob.status;
+    while (status === 'pending' || status === 'running') {
+      console.log(`⏳ Job status: ${status}...`);
+      await new Promise((resolve) => setTimeout(resolve, 10000));
+
+      const updatedJob = await openai.fineTuning.jobs.retrieve(fineTuneJob.id);
+      status = updatedJob.status;
+    }
+
+    // Step 4. Check final status
+    if (status === 'succeeded') {
+      console.log('🎉 Fine-tuning completed successfully!');
+      console.log(`🆕 Fine-tuned model: ${fineTuneJob.fine_tuned_model}`);
+      return fineTuneJob.fine_tuned_model;
+    } else {
+      throw new Error(`❌ Fine-tuning failed: ${status}`);
+    }
+  } catch (error) {
+    console.error('⚠️ Fine-tuning error:', error);
+    throw error;
+  }
+}
+
+const upload_file_to_openai_storage = async function (file_path) {
+  try {
+    const stream = await get_drive_file_stream(file_path);
+
+    const uploadedFile = await client.files.create({
+      file: stream,
+      purpose: 'assistants',
+    });
+
+    console.log('📂 Uploaded file:', uploadedFile.id);
+    unlink(tmp_file);
+    return uploadedFile.id;
+  } catch (err) {
+    console.error(err);
+  }
+};
+
+const get_account_object = async function (uid) {
+  const db = get_xuda_couch('xuda_accounts');
+  const account_obj = await db.get(uid);
+  return account_obj;
+};
+
+const get_account_project_id = async function (uid) {
+  const account_obj = await account_module.get_account_default_project_obj(uid);
+  return account_obj.account_project_id;
+};
+
+const getExtension = function (filename) {
+  return path.extname(filename).toLowerCase();
+};
+
+setTimeout(async () => {
+  const app_id = 'prj712ffdf5aa8adce6cedef988f9c12392'; //'prj3937cb6f9a31c8c7dea25055bba845b1'; //
+  const uid = 'd39126e0e2c51ffbd1aad10709fc8335';
+
+  // await init_studio_units();
+  let ret;
+  // await build_studio_table_tools(app_id);
+  // testTool(app_id, uid);
+  // ret = await download_yt_audio('https://www.youtube.com/watch?v=uSt5yLOWiNo');
+  // ret = await update_ai_agent({
+  //   uid: 'd39126e0e2c51ffbd1aad10709fc8335',
+  //   agent_id: '392_agn_ddbca7c25455',
+  //   agentConfig: {
+  //     agent_instructions: 'use app builder to build tables and programs',
+  //     agent_tools: [
+  //       {
+  //         type: 'app_builder',
+  //         name: 'app_builder',
+  //       },
+  //     ],
+  //     agent_name: 'Xuda App Builder',
+  //   },
+  // });
+  // ret = await client.vectorStores.files.create(_conf.OPENAI_VECTOR_STORE_ID, {
+  //   file_id: 'file-Rf9UzT1f7NVhTELypnbsBU',
+  // });
+  // ret = upload_file_to_openai_storage('/docxsite-bp.xlsx.pdf');
+  // ret = await transcribe('/AAA.mp3');
+  // ret = await transcribe('/B.mp4');
+  // ret = await upload_file_to_openai_storage('/AAA.mp3');
+  // ret = await create_image("'Generate an image of gray tabby cat hugging an otter with an orange scarf'");
+  // ret = await submit_chat_conversation({ uid, prompt: 'use xuda.io agent to require info about xuda.io', ai_agents: [] });
+  // ret = await submit_chat_conversation({ prompt: 'what is my name', conversation_id: ret.data.conversation_doc._id });
+  // const ret = await create_studio_app('crm');
+  //   ret = await submit_chat_gpt_prompt({
+  //     prompt: `Use the formatting rules from the Xuda fine-tuned model.
+  // Generate a new Table Object JSON with the same schema learned in training.
+  // Don't modify _rev. Preserve screenshots array structure.`,
+  //     model: 'ft:gpt-4o-2024-08-06:xuda-io:xuda-table:CdYWfx0l', //'ft:gpt-4.1-nano-2025-04-14:xuda-io:xuda-table:CdXWxgir'
+  //   });
+  // const ret = await submit_agentic_studio_chat_gpt_prompt({ user_prompt: 'crm app', object_unit: 'app', uid, app_id });
+  // ret = await submit_agentic_studio_chat_gpt_prompt({ user_prompt: 'products', object_unit: 'table', uid, app_id });
+  // ret = await submit_agentic_studio_chat_gpt_prompt({ user_prompt: 'test3', object_unit: 'folder', uid, app_id });
+  // ret = await submit_agentic_studio_chat_gpt_prompt({ user_prompt: 'contact contact form', object_unit: 'component', uid, app_id });
+  // ret = await submit_agentic_studio_chat_gpt_prompt({ user_prompt: 'get data from tasks by task id', object_unit: 'get_data', uid, app_id });
+
+  // ret = await test_plugin(app_id, uid, '@xuda.io/xuda-library-plugin-studio-ai-app-builder', 'table_builder', { prompt: 'build items table', save: true, parentId: 'programs' });
+  // ret = await test_plugin(app_id, uid, '@xuda.io/xuda-library-plugin-studio-ai-app-builder', 'folder_builder', { prompt: 'build boaz folder', save: true, parentId: 'programs' });
+
+  console.log(ret);
+
+  // init({}, {}, _, true, null, null, UglifyJS, JSON5, _conf, false, z);
+  // const schema = get_zod_schema('component');
+
+  // const db = get_xuda_couch('xuda_master');
+  // const app_obj = await db.get(app_id);
+  // const project_db = get_xuda_couch(app_obj.app_db_name);
+  // const doc = await project_db.get('5b1_cmp_313d53e56c2d');
+
+  ////////////////////
+  // let err_ret = [];
+  // const addValidationError = (code, data) => {
+  //   err_ret.push({
+  //     code,
+  //     data,
+  //     type: 'E',
+  //     category: 'document',
+  //     structure_error: true,
+  //   });
+  // };
+  //////////////////
+
+  // const result = schema.safeParse({});
+  // if (!result.success) {
+  //   console.log(result.error.errors); // Array of error objects
+  //   result.error.errors.forEach((error) => {
+  //     // console.log(`Path: ${error.path}, Message: ${error.message}`);
+  //     const msg = `${error.message}: ${error.path}(${error.expected})`;
+  //     addValidationError('CHK_MSG_OBJ_GEN_000', msg);
+  //   });
+  //   console.log(err_ret);
+  // } else {
+  //   console.log('Validation succeeded:', result.data);
+  // }
+
+  // const jsonSchema = zodToJsonSchema(schema);
+  // // console.log(jsonSchema);
+  // console.log(JSON.stringify(jsonSchema, null, 2));
+
+  // console.log(
+  //   check(
+  //     createDoc({
+  //       uid: '11',
+  //       app_id: 'prj-444',
+  //       menuType: 'folder',
+  //       checkedInUserName: 'boaz',
+  //       parentId: '',
+  //     }),
+  //   ),
+  // );
+
+  // const vectorStore = await client.beta.vector_stores.create({
+  //   name: 'Xuda Docs Vector Store',
+  // });
+  // console.log('Vector Store ID:', vectorStore.id);
+  // console.log('OpenAI version:', (await import('openai/package.json')).version);
+  // console.log('has beta?', !!client.beta);
+  // console.log('has vector_stores?', !!client.beta?.vector_stores);
+  // console.log('keys on vector_stores:', Object.keys(client.beta?.vector_stores || {}));
+
+  // const vectorStore = await client.vectorStores.create({
+  //   name: 'Xuda Docs Vector Store',
+  // });
+  // console.log(vectorStore);
+}, 1000);
+
+async function testTool(app_id, uid) {
+  console.log('Testing tool with:', { app_id, uid, table_id: '392_tbl_47f0ac6059ca' });
+
+  try {
+    const input = { table_id: '392_tbl_47f0ac6059ca' };
+    const runContext = { context: { app_id, uid } };
+    const details = {};
+
+    console.log('Input:', JSON.stringify(input, null, 2));
+    console.log('RunContext:', JSON.stringify(runContext, null, 2));
+    debugger;
+    // CORRECT ORDER: input first, then runContext
+    const result = await get_studio_table_info_by_table_id_tool.invoke(
+      runContext, // 1st: context
+      input, // 2nd: parameters
+      details, // 3rd: optional
+    );
+
+    console.log('Tool test successful:', result);
+    return result;
+  } catch (error) {
+    console.error('Tool test failed:');
+    console.error('Error:', error.message);
+    throw error;
+  }
+}
+
+// export const submit_chat_conversation = async function (req, job_id, headers) {
+//   const { uid, gtp_token } = req;
+//   let { prompt, conversation_id, reference_type, reference_id, attachments = [], ai_agents, output_format = 'md', stream = true, with_context = true } = req;
+
+//   const app_id = await account_module.get_account_default_project_id(uid);
+//   const app_obj = await get_app_obj(app_id);
+//   const userName = await account_module.get_user_name(uid);
+//   const account_info = (await account_module.get_account_name({ uid })).data;
+//   let context = { ...{ app_id, uid, userName, account_info, app_obj, account_id: uid }, ...extractClientInfo(headers) };
+//   const d = new Intl.DateTimeFormat('en-US').format(new Date());
+//   if (!visitors[d]) {
+//     visitors[d] = {};
+//   }
+
+//   let conversation_doc = {
+//     docType: 'chat_conversation',
+//     title: prompt.substr(0, 40),
+//     date_created_ts: Date.now(),
+//     ts: Date.now(),
+//     stat: 3,
+//     uid,
+//     messages: [],
+//     reference_type,
+//     reference_id,
+//   };
+
+//   //////////////////////
+
+//   const emitToDashboard = (type, content) => {
+//     global[`_ai_module_ch`].sendToQueue(
+//       'ws_dashboard_module',
+//       Buffer.from(
+//         JSON.stringify({
+//           method: 'emit_message_to_dashboard',
+//           data: {
+//             service: type,
+//             to: uid,
+//             data: {
+//               conversation_id: conversation_id,
+//               job_id,
+//               delta: content,
+//             },
+//           },
+//         }),
+//       ),
+//     );
+//   };
+//   let emitToDashboard_interval;
+//   const update_job = async function (current_step_name) {
+//     if (emitToDashboard_interval) {
+//       clearInterval(emitToDashboard_interval);
+//       emitToDashboard('stream_delta', '<br><br>');
+//     }
+//     emitToDashboard('stream_delta', _.capitalize(current_step_name) + ' ... ');
+//     // emitToDashboard_interval = setInterval(() => {
+//     //   emitToDashboard('stream_delta', '.');
+//     // }, 500);
+//     await jobs_module.update_job(
+//       {
+//         job_id,
+//         current_step_name: current_step_name,
+//       },
+//       global[`_ai_module_ch`],
+//     );
+//   };
+
+//   //////////////////////
+//   // emitToDashboard('stream_start');
+
+//   // await update_job('initiating conversation');
+//   // await _utils.delay(500);
+
+//   // 1. Ensure we have a conversation doc in Couch
+//   let conversation_exist;
+//   try {
+//     if (!conversation_id) throw new Error('no id');
+//     conversation_doc = await db_module.get_app_couch_doc_native(app_id, conversation_id);
+//     reference_type = conversation_doc.reference_type;
+//     reference_id = conversation_doc.reference_id;
+//     conversation_exist = true;
+//     // ai_agents = conversation_doc.ai_agents || [];
+//   } catch (err) {
+//     // new conversation
+//     const conversation_obj = await client.conversations.create();
+//     conversation_id = conversation_obj.id;
+//     conversation_doc.conversation_obj = conversation_obj;
+//     conversation_doc._id = conversation_id;
+//     // conversation_doc.ai_agents = ai_agents;
+//     await db_module.save_app_couch_doc_native(app_id, conversation_doc);
+//   }
+//   emitToDashboard('stream_start');
+//   await update_job('analyzing request');
+
+//   const init_agent_hooks = function (agent) {
+//     agent.on('agent_start', (context, agent) => {
+//       // emitToDashboard('stream_start');
+//     });
+
+//     agent.on('agent_end', (context, output) => {
+//       emitToDashboard('stream_end');
+//     });
+
+//     agent.on('agent_handoff', (context, nextAgent) => {
+//       emitToDashboard('agent_handoff', nextAgent.name);
+//     });
+
+//     agent.on('agent_tool_start', (context, tool, details) => {
+//       emitToDashboard('agent_tool_start', tool.name);
+//     });
+
+//     agent.on('agent_tool_end', (context, tool, result, details) => {
+//       emitToDashboard('agent_tool_end', tool.name);
+//     });
+//   };
+//   const get_agent_instructions = function (is_agent) {
+//     const get_output_format = function () {
+//       let ret = '';
+//       switch (output_format) {
+//         case 'md':
+//           ret = 'Include mark down (MD) tags where necessary in the response.';
+//           break;
+
+//         default:
+//           break;
+//       }
+
+//       return ret;
+//     };
+
+//     let new_visit = true;
+//     if (visitors?.[d]?.[uid]) {
+//       new_visit = false;
+//     } else {
+//       visitors[d][uid] = prompt;
+//     }
+
+//     const triage_assistant = `
+
+//     You are Nissim AI Xuda assistant — a smart triage router.
+//     Your job is to instantly decide who should handle the user's request:
+//     - If the question is general or can be answered directly, respond yourself.
+//     - If it requires a specific specialist, route it to them.
+//     Only fall back to web search as an absolute last resort — never use it unless strictly necessary.
+
+//      ${new_visit ? 'Always start your reply with a friendly greeting using their actual name based on the context.' : ''}
+//     ${get_output_format()}
+
+//      `;
+
+//     const identity = `
+
+//             SPECIAL IDENTITY RULE (overrides everything else):
+//             When the user asks any variation of:
+//             - "who am I"
+//             - "what is my name"
+//             - "do you know who I am"
+//             - "tell me about myself"
+//             - or any similar identity question
+
+//             You MUST:
+
+//             1. answer directly and personally using the real user context provided below.
+//             2. Never say you don’t know or can’t determine their identity.
+//             3. Show thinking process step by step before providing the final answer.
+
+//             Current user context (use this every time also for any location base prompts):
+//             ${JSON.stringify(context)}
+
+//             Example response (adapt naturally):
+//             "Hi Sarah! You're Sarah Cohen, our VIP client from Tel Aviv, account #89231. You've been with us since 2022 and usually reach out about trading or portfolio updates. How can I help you today?"
+//            `.trim();
+
+//     return is_agent ? identity : triage_assistant + identity;
+//   };
+//   const get_agents = async function () {
+//     if (typeof ai_agents === 'undefined') {
+//       // no agents
+//       ai_agents = [];
+//     } else if (!ai_agents.length) {
+//       // auto agent mode
+//       const ai_agents_ret = await get_ai_agents({ uid });
+//       for (const doc of ai_agents_ret.data.docs) {
+//         ai_agents.push(doc.id);
+//       }
+//     }
+
+//     const list = [];
+
+//     let modelSettings = {};
+
+//     if (reference_type === 'ai_agents') {
+//       ai_agents.push(reference_id);
+//     }
+
+//     for (const agent_id of ai_agents) {
+//       let ai_agent_doc;
+//       try {
+//         ai_agent_doc = await db_module.get_app_couch_doc_native(app_id, agent_id);
+//       } catch (err) {
+//         throw new Error(err.message);
+//       }
+
+//       if (ai_agent_doc.use_tools_only) {
+//         modelSettings.toolChoice = 'required';
+//       }
+
+//       let tools = [];
+//       let tool_resources = {};
+
+//       for (const [key, val] of Object.entries(ai_agent_doc?.agentConfig?.agent_tools || [])) {
+//         const { name, description } = val;
+//         switch (val.type) {
+//           case 'web_search': {
+//             //hosted_web_search
+//             tools.push(webSearchTool());
+//             break;
+//           }
+
+//           case 'mcp': {
+//             tools.push(
+//               hostedMcpTool({
+//                 serverLabel: 'mcp_' + (val?.name?.replaceAll(' ', '_') || key), //ai_agent_doc.properties.menuName,
+//                 serverUrl: val.url,
+//                 headers: {
+//                   Authorization: `Bearer ${val.authentication_key || gtp_token}`,
+//                   'X-gtp-token': 'true',
+//                   'Content-Type': 'application/json',
+//                 },
+//                 require_approval: 'never',
+//               }),
+//             );
+//             break;
+//           }
+
+//           case 'cpi': {
+//             try {
+//               debugger;
+//               const api_allow_methods = Object.keys(_conf.cpi_methods);
+//               for (const method_name of api_allow_methods || []) {
+//                 const method_prop = _conf.cpi_methods[method_name];
+
+//                 if (!method_prop) continue;
+//                 if (method_prop.private) continue;
+//                 if (!method_prop.description) continue;
+
+//                 let inputSchema = {};
+
+//                 const get_z_item = function (key, val) {
+//                   let prop = {};
+//                   if (val.type === 'object' && val.items) {
+//                     for (const [item_key, item_val] of Object.entries(val.items.properties)) {
+//                       prop[item_key] = get_z_item(item_key, item_val);
+//                       if (!val?.items?.required?.includes(item_key)) {
+//                         prop[item_key] = prop[item_key].optional();
+//                       }
+//                     }
+//                   }
+
+//                   if (val.type === 'array') {
+//                     prop = get_z_item('', val.items);
+//                   }
+
+//                   let _z = z[val.type](prop);
+
+//                   if (!val.mandatory) {
+//                     _z = _z.optional();
+//                   }
+
+//                   if (val.enum) {
+//                     _z = z.enum(val.enum);
+//                   }
+
+//                   let valid_values = '';
+//                   if (val.options) {
+//                     valid_values = val.options.join(', ');
+//                   }
+//                   if (val.description) {
+//                     const desc = val.name || key.replace(/_/g, ' ').replace(/\b(\w)/g, (_, char) => char.toUpperCase());
+//                     _z = _z.describe((desc ? desc + ' - ' : '') + val.description + (valid_values ? ' Valid values: ' + valid_values : ''));
+//                   }
+
+//                   return _z;
+//                 };
+
+//                 if (method_prop.fields) {
+//                   try {
+//                     for (const [key, val] of Object.entries(method_prop.fields)) {
+//                       inputSchema[key] = get_z_item(key, val);
+//                     }
+//                   } catch (error) {
+//                     debugger;
+//                   }
+//                 }
+//                 try {
+//                   tools.push(
+//                     tool({
+//                       name: method_name.replace(/_/g, ' ').replace(/\b(\w)/g, (_, char) => char.toUpperCase()),
+//                       description: method_prop.description,
+//                       parameters: z.object(inputSchema),
+//                       async execute(e, RunContext) {
+//                         const params = e;
+//                         try {
+//                           const response = await fetch(`http://localhost:3007/cpi/${method_name}`, {
+//                             method: 'POST',
+//                             headers: {
+//                               Accept: 'application/json',
+//                               'Content-Type': 'application/json',
+//                             },
+//                             body: JSON.stringify({ ...params, ...{ gtp_token } }),
+//                           });
+//                           const ret = await response.json();
+//                           return {
+//                             content: [{ type: 'text', text: JSON.stringify(ret || '', null, 2) }],
+//                           };
+//                         } catch (err) {
+//                           return 'unknown';
+//                         }
+//                       },
+//                     }),
+//                   );
+//                 } catch (error) {
+//                   debugger;
+//                 }
+//               }
+//             } catch (error) {
+//               debugger;
+//             }
+//             break;
+//           }
+
+//           // case 'web_search': {
+//           //   tools.push(
+//           //     tool({
+//           //       name: name || val.type,
+//           //       description: description || val.type,
+//           //       parameters: z.object({
+//           //         query: z.string().describe('The search query text.'),
+//           //         numResults: z.number().int().describe('Number of top search results to return.').default(3),
+//           //       }),
+//           //       async execute(e, RunContext) {
+//           //         const { query, numResults } = e;
+//           //         try {
+//           //           const searchUrl = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_redirect=1`;
+//           //           const response = await fetch(searchUrl);
+//           //           const data = await response.json();
+
+//           //           // Simplify results
+//           //           const results = (data.RelatedTopics || []).slice(0, numResults).map((item) => ({
+//           //             title: item.Text,
+//           //             url: item.FirstURL,
+//           //           }));
+
+//           //           return JSON.stringify(results, null, 2);
+//           //         } catch (err) {
+//           //           return 'unknown';
+//           //         }
+//           //       },
+//           //     }),
+//           //   );
+//           //   break;
+//           // }
+
+//           // case 'image_generate': {
+//           //   tools.push(
+//           //     tool({
+//           //       name,
+//           //       description,
+//           //       parameters: z.object({
+//           //         prompt: z.string().describe('The prompt for the image.'),
+//           //         numGenerations: z.number().int().describe('Number generations to image create.').default(3),
+//           //       }),
+//           //       async execute(e, RunContext) {
+//           //         const { prompt, numGenerations = val?.image_generations } = e;
+//           //         try {
+//           //           const base64 = await create_image(prompt, undefined, undefined, numGenerations);
+//           //           // console.log(base64);
+
+//           //           // return `<img src="data:image/jpg;base64,${base64}">`;
+//           //           return `![Generated Image](data:image/jpeg;base64,${base64})`;
+//           //         } catch (err) {
+//           //           return 'unknown';
+//           //         }
+//           //       },
+//           //     }),
+//           //   );
+//           //   break;
+//           // }
+
+//           case 'image_generate': {
+//             tools.push(
+//               tool({
+//                 name,
+//                 description,
+//                 parameters: z.object({
+//                   prompt: z.string().describe('The prompt for the image.'),
+//                   numGenerations: z.number().int().describe('Number generations to image create.').default(3),
+//                 }),
+//                 async execute(e, RunContext) {
+//                   const { prompt, numGenerations } = e;
+//                   try {
+//                     let imgTags = '';
+//                     const images_arr = await create_and_upload_image_to_drive('workspace', 'Chat Images', prompt, numGenerations, uid, app_id, job_id, headers);
+//                     for (const img_obj of images_arr) {
+//                       imgTags += `<img src="${img_obj.file_url}" alt="${prompt}" style="max-width: 100%; border-radius: 8px;" />`;
+//                     }
+
+//                     return imgTags;
+//                   } catch (err) {
+//                     console.error('Image Tool Error:', err);
+//                     return `Error generating image: ${err.message}`;
+//                   }
+//                 },
+//               }),
+//             );
+//             break;
+//           }
+
+//           case 'table': {
+//             const table_id = val.prog_id;
+//             // prompt += ` for table_id: ${table_id}`;
+//             context.table_id = table_id;
+//             tools.push(get_studio_table_info_by_table_id_tool);
+//             //======== mock data ==========
+//             // const mock_data_generator_agent = get_mock_data_generator_agent(app_id, table_id);
+//             // tools.push(
+//             //   mock_data_generator_agent.asTool({
+//             //     toolName: 'Mock Data Generator',
+//             //     toolDescription: `generate Mock Data to a table`,
+//             //   }),
+//             // );
+//             // tools.push(get_studio_table_info_by_table_id_tool);
+
+//             //======== add data ==========
+//             tools.push(
+//               add_data_to_table_agent.asTool({
+//                 toolName: 'add data to table',
+//                 toolDescription: `add data to table`,
+//               }),
+//             );
+
+//             //======== fetch data ==========
+//             tools.push(
+//               fetch_table_data_agent.asTool({
+//                 toolName: 'Fetch Table Data Agent',
+//                 toolDescription: `fetch data from a table`,
+//               }),
+//             );
+
+//             //======== delete data ==========
+//             tools.push(
+//               delete_table_doc.asTool({
+//                 toolName: 'Delete Table Doc',
+//                 toolDescription: `The tool delete document from the database, need to ask for table`,
+//               }),
+//             );
+
+//             // tools.push(create_table_management_tool);
+
+//             // tools.push(create_landing_page_tool);
+
+//             // const add_data_to_table_agent = get_add_data_to_table_agent(app_id, table_id);
+//             // tools.push(
+//             //   add_data_to_table_agent.asTool({
+//             //     toolName: 'add data to table',
+//             //     toolDescription: `add data to table`,
+//             //   }),
+//             // );
+
+//             break;
+//           }
+
+//           case 'file_search': {
+//             if (val?.file?.vector_store_id) {
+//               tools.push(fileSearchTool(val.file.vector_store_id));
+//             }
+
+//             if (val?.youtube?.vector_store_id) {
+//               tools.push(fileSearchTool(val.youtube.vector_store_id));
+//             }
+
+//             break;
+//           }
+
+//           case 'ai_agent': {
+//             let opt = {};
+//             // add all agents if no agent id
+//             if (val.ai_agent_id) {
+//               opt = {
+//                 _id: val.ai_agent_id,
+//               };
+//             }
+//             const agent_ret = await get_ai_agents(opt);
+//             if (agent_ret.code > -1) {
+//               const agent_doc = agent_ret.data;
+
+//               const agent = new Agent({
+//                 name: agent_doc.agent_name,
+//                 instructions: agent_doc.agent_instructions,
+//               });
+
+//               tools.push(
+//                 agent.asTool({
+//                   toolName: agent_doc.agent_name,
+//                   toolDescription: agent_doc.agent_instructions,
+//                 }),
+//               );
+//             }
+//             // }
+//             break;
+//           }
+//           case 'plugin': {
+//             if (!val.plugin_id) {
+//               throw new Error('undefined plugin_id');
+//             }
+//             if (!val.method) {
+//               throw new Error('undefined method');
+//             }
+//             const tool = await get_plugin_tool(app_id, uid, val.plugin_id, val.method, {});
+//             tools.push(tool);
+//             break;
+//           }
+//           default:
+//             break;
+//         }
+//       }
+//       // if (!tools.length) continue;
+//       const agent = new Agent({
+//         name: ai_agent_doc._id,
+//         instructions: ai_agent_doc.agentConfig.agent_instructions + get_agent_instructions(),
+//         model,
+//         tools,
+//         metadata: {
+//           agent_id: ai_agent_doc._id,
+//           ts: Date.now(),
+//         },
+//         tool_resources,
+//         modelSettings,
+//       });
+
+//       init_agent_hooks(agent);
+
+//       list.push(agent);
+//     }
+//     //tell me more on xuda
+//     // debugger;
+
+//     return list;
+//   };
+
+//   // const get_attachment_agent = async function () {
+//   //   if (!attachments?.length) return;
+//   //   let tools = [];
+//   //   let file_names = [];
+//   //   let system_message_items = [];
+//   //   for (const file of attachments) {
+//   //     tools.push(fileSearchTool(file.id));
+//   //     file_names.push(file.file_name);
+
+//   //     const file_info = await client.files.retrieve(file.id);
+//   //     // const file_content = await client.files.content(file.id);
+//   //     system_message_items.push({
+//   //       type: 'message',
+//   //       role: 'system',
+//   //       content: [{ type: 'input_text', text: `the user attached file name: ${file_info.filename} with content: ${file_content}` }],
+//   //     });
+//   //   }
+
+//   //   const items = await client.conversations.items.create(conversation_id, {
+//   //     items: system_message_items,
+//   //   });
+//   //   console.log(items.data);
+
+//   //   return new Agent({
+//   //     name: 'attachments agent',
+//   //     instructions: `You can see and analyze any files the user attaches:${JSON.stringify(file_names)} `,
+//   //     model,
+//   //     tools,
+//   //   });
+//   // };
+
+//   const generate_XU_markdown = async function (type, data) {
+//     return `XU>${job_id}{type:"${type}",data:${JSON.stringify(data)}}${job_id}<XU`;
+//   };
+
+//   const attachment_handler = async function () {
+//     if (!attachments?.length) return;
+
+//     const get_transcript = async function (filename) {
+//       async function streamToBuffer(fileStream) {
+//         const chunks = [];
+//         for await (const chunk of fileStream) {
+//           chunks.push(chunk);
+//         }
+//         return Buffer.concat(chunks);
+//       }
+
+//       const validation_ret = file_upload_validator(filename);
+//       if (!validation_ret.valid) {
+//         throw new Error(validation_ret.error);
+//       }
+
+//       const fileStream = await get_drive_file_stream(filename);
+
+//       let transcript;
+//       switch (validation_ret.category) {
+//         case 'audio':
+//         case 'video': {
+//           transcript = await transcribe(fileStream);
+
+//           break;
+//         }
+
+//         case 'image': {
+//           const fileBuffer = await streamToBuffer(fileStream);
+//           const base64 = fileBuffer.toString('base64');
+
+//           const ret = await submit_chat_gpt_prompt({
+//             prompt: [
+//               {
+//                 role: 'user',
+//                 content: [
+//                   {
+//                     type: 'input_text',
+//                     text: `1. describe the image in detail. 2. Extract the text from this image into a clean markdown format `,
+//                   },
+//                   {
+//                     type: 'input_image',
+//                     image_url: `data:${validation_ret.mime};base64,${base64}`,
+//                   },
+//                 ],
+//               },
+//             ],
+//           });
+//           if (ret.code < 0) {
+//             throw new Error('something went wrong try again later');
+//           }
+//           transcript = ret.data;
+
+//           break;
+//         }
+
+//         case 'document':
+//         case 'text': {
+//           async function transcript_file(fileBuffer, fileName, mimeType) {
+//             try {
+//               const base64 = fileBuffer.toString('base64');
+
+//               const ret = await submit_chat_gpt_prompt({
+//                 prompt: [
+//                   {
+//                     role: 'user',
+//                     content: [
+//                       {
+//                         type: 'input_text',
+//                         text: 'Please extract all text from this document. Maintain the structure and formatting as much as possible.',
+//                       },
+//                       {
+//                         type: 'input_file',
+//                         filename: fileName || 'document.pdf',
+
+//                         file_data: `data:${mimeType || 'application/pdf'};base64,${base64}`,
+//                       },
+//                     ],
+//                   },
+//                 ],
+//               });
+
+//               return ret;
+//             } catch (error) {
+//               console.error('Error transcribing PDF:', error);
+//               throw error;
+//             }
+//           }
+//           const buffer = await streamToBuffer(fileStream);
+//           transcript = await transcript_file(buffer, filename, validation_ret.mime);
+
+//           break;
+//         }
+
+//         default:
+//           break;
+//       }
+
+//       return transcript;
+//     };
+
+//     let file_names = [];
+//     let system_message_items = [];
+//     for (const file of attachments) {
+//       file_names.push(file.filename);
+//       const transcript = await get_transcript(file.filename);
+
+//       system_message_items.push({
+//         type: 'message',
+//         role: 'system',
+//         content: [{ type: 'input_text', text: `the user attached file name: ${file.filename} with content: ${transcript.data || transcript}` }],
+//       });
+
+//       prompt += generate_XU_markdown('attachment', file); //`[![Alt Text](${file.file_url})]`;
+//     }
+
+//     const items = await client.conversations.items.create(conversation_id, {
+//       items: system_message_items,
+//     });
+//     console.log(items.data);
+//   };
+
+//   const set_ts_to_agent = async function () {
+//     try {
+//       let agent_doc = await db_module.get_app_couch_doc_native(app_id, reference_id);
+//       agent_doc.ts = Date.now();
+//       await db_module.save_app_couch_doc_native(app_id, agent_doc);
+//     } catch (err) {}
+//   };
+
+//   try {
+//     const runner = new Runner();
+//     let _agent = [];
+
+//     await update_job('preparing tools');
+//     const agents = await get_agents();
+
+//     /////////////
+//     if (attachments?.length) {
+//       await update_job('analyzing attachments');
+//       await attachment_handler();
+//     }
+//     ////////////
+
+//     if (reference_type === 'ai_agents') {
+//       _agent = agents[0];
+
+//       set_ts_to_agent();
+//     } else {
+//       // 3. Build triage agent that can hand off
+//       _agent = new Agent({
+//         name: 'Triage Agent',
+//         instructions: get_agent_instructions(),
+//         model,
+//         handoffs: agents, // this lets triageAgent forward to other agents
+//       });
+//     }
+
+//     let opt = {
+//       conversationId: conversation_id,
+//       context,
+//       stream,
+//     };
+
+//     await update_job('submitting chat');
+//     init_agent_hooks(_agent);
+//     const output = await runner.run(_agent, prompt, opt);
+//     const done = async function (output) {
+//       try {
+//         const obj = { id: output?.output?.[0]?.id, ts: Date.now(), usage: { inputTokens: output.state._context.usage.inputTokens, outputTokens: output.state._context.usage.outputTokens }, ai_agent_id: output.state._currentAgent.name, attachments };
+//         conversation_doc = await db_module.get_app_couch_doc_native(app_id, conversation_id);
+
+//         conversation_doc.ts = Date.now();
+//         conversation_doc.messages.push(obj);
+//         await db_module.save_app_couch_doc_native(app_id, conversation_doc);
+
+//         if (!conversation_exist) {
+//           setTimeout(async () => {
+//             // Ask GPT for title
+//             const title = await submit_chat_gpt_prompt({
+//               prompt: `create title for the prompt return 5 words result maximum text only without options : ${prompt}`,
+//               model: 'gpt-5-nano',
+//             });
+
+//             conversation_doc = await db_module.get_app_couch_doc_native(app_id, conversation_id);
+//             conversation_doc.title = title.data;
+
+//             await db_module.save_app_couch_doc_native(app_id, conversation_doc);
+//             update_thumbnail('conversation', conversation_doc, app_id, uid, job_id, headers);
+//           }, 5000);
+//         } else {
+//           if (!conversation_doc.chat_image && (!conversation_doc.thumbnail_request_ts || Date.now() - conversation_doc.thumbnail_request_ts > 120000)) {
+//             update_thumbnail('conversation', conversation_doc, app_id, uid, job_id, headers);
+//           }
+//         }
+//       } catch (error) {
+//         debugger;
+//       }
+//       // }
+
+//       // return {
+//       //   code: 4,
+//       //   data: {
+//       //     conversation_doc,
+//       //     response: output.output, // output.finalOutput ?? output.output ?? output, // depends on SDK shape
+//       //   },
+//       // };
+//     };
+
+//     if (stream) {
+//       const output_stream = output.toTextStream({
+//         compatibleWithNodeStreams: true,
+//       });
+//       let buffer = '';
+
+//       let response_start;
+
+//       // 1. Consume the stream exactly once
+//       let string_debug = '';
+
+//       for await (const chunk of output_stream) {
+//         if (!response_start) {
+//           response_start = true;
+
+//           await update_job('streaming results');
+//           emitToDashboard('response_start');
+//         }
+//         const text = chunk.toString();
+//         // console.log('[CHUNK]', text);
+//         emitToDashboard('stream_delta', text);
+//         string_debug += text;
+//         // Accumulate it
+//         buffer += chunk;
+//       }
+//       // console.log('string_debug', string_debug);
+//     }
+//     // else {
+//     await update_job('finalizing');
+//     await done(output);
+//     return {
+//       code: 4,
+//       data: {
+//         conversation_doc,
+//         response: output.output, // output.finalOutput ?? output.output ?? output, // depends on SDK shape
+//       },
+//     };
+//     // }
+//   } catch (err) {
+//     debugger;
+//     emitToDashboard('stream_end');
+//     return { code: -4, data: err.message || String(err) };
+//   }
+// };