@ducci/jarvis 1.0.66 → 1.0.68

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ducci/jarvis",
-  "version": "1.0.66",
+  "version": "1.0.68",
   "description": "A fully automated agent system that lives on a server.",
   "main": "./src/index.js",
   "type": "module",

package/src/channels/telegram/index.js

@@ -6,6 +6,7 @@ import { handleChat } from '../../server/agent.js';
 import { loadSession } from '../../server/sessions.js';
 import { PATHS } from '../../server/config.js';
 import { load, save } from './sessions.js';
+import { describeImage } from '../../server/vision.js';
 
 function getTelegramChatLogPath(chatId, sessionId) {
   const prefix = sessionId ? String(sessionId).slice(0, 8) : 'unknown';
@@ -123,11 +124,39 @@ export async function startTelegramChannel(config) {
       const combinedText = batch.length === 1
         ? batch[0].text
         : batch.map(m => m.text).join('\n\n');
-      const allAttachments = batch.flatMap(m => m.attachments);
+      const rawAttachments = batch.flatMap(m => m.attachments);
+
+      // If a vision model is configured and the batch contains images, run a one-shot
+      // image analysis first and inject the result as text into the main agent turn.
+      let userText = combinedText;
+      let allAttachments = rawAttachments;
+      if (config.visionModel && config.visionApiKey && rawAttachments.length > 0) {
+        try {
+          const results = await Promise.allSettled(
+            rawAttachments.map(a => describeImage(a, combinedText, config))
+          );
+          const successfulDescs = results
+            .filter(r => r.status === 'fulfilled')
+            .map(r => `[Image analysis: ${r.value}]`);
+          const failedCount = results.filter(r => r.status === 'rejected').length;
+          if (failedCount > 0) {
+            console.error(`[telegram] vision error for ${failedCount}/${results.length} image(s), those will be sent directly to main agent`);
+          }
+          if (successfulDescs.length > 0) {
+            const descBlock = successfulDescs.join('\n\n');
+            userText = combinedText ? `${descBlock}\n\n${combinedText}` : descBlock;
+          }
+          // Only clear attachments for images that were successfully described
+          allAttachments = rawAttachments.filter((_, i) => results[i].status === 'rejected');
+        } catch (e) {
+          console.error(`[telegram] vision error, falling back to direct image: ${e.message}`);
+          // allAttachments and userText stay unchanged — main agent gets the raw image
+        }
+      }
 
       let result;
       try {
-        result = await handleChat(config, sessionId, combinedText, allAttachments);
+        result = await handleChat(config, sessionId, userText, allAttachments);
       } catch (e) {
         console.error(`[telegram] agent error chat_id=${chatId}: ${e.message}`);
         const errText = e.message

package/src/scripts/onboarding.js

@@ -389,6 +389,182 @@ async function run() {
   saveSettings(settings);
   console.log(chalk.green(`\nModel ${chalk.bold(selectedModel)} saved to settings.`));
 
+  // --- VISION MODEL STEP (OPTIONAL) ---
+  const { configureVision } = await inquirer.prompt([
+    {
+      type: 'confirm',
+      name: 'configureVision',
+      message: 'Do you want to configure a separate vision model for image analysis (e.g. for Telegram photos)?',
+      default: !!settings.visionModel,
+    }
+  ]);
+
+  if (configureVision) {
+    const { visionProvider } = await inquirer.prompt([
+      {
+        type: 'list',
+        name: 'visionProvider',
+        message: 'Which provider for the vision model?',
+        choices: [
+          { name: 'OpenRouter (access many models via one key)', value: 'openrouter' },
+          { name: 'Z.AI Direct (GLM models)', value: 'z-ai' },
+        ],
+        default: settings.visionProvider || provider,
+      }
+    ]);
+
+    // Ensure the key for the vision provider is available
+    const visionKeyVar = visionProvider === 'z-ai' ? 'ZAI_API_KEY' : 'OPENROUTER_API_KEY';
+    let visionApiKey = loadEnvVar(visionKeyVar);
+    if (visionApiKey) {
+      const { keepVisionKey } = await inquirer.prompt([
+        {
+          type: 'confirm',
+          name: 'keepVisionKey',
+          message: `A ${visionKeyVar} is already configured. Use it for the vision model?`,
+          default: true,
+        }
+      ]);
+      if (!keepVisionKey) visionApiKey = null;
+    }
+    if (!visionApiKey) {
+      console.log(chalk.yellow(`No ${visionKeyVar} found — needed for the vision model.`));
+      const { newVisionKey } = await inquirer.prompt([
+        {
+          type: 'password',
+          name: 'newVisionKey',
+          message: `Enter your ${visionProvider === 'z-ai' ? 'Z.AI' : 'OpenRouter'} API key:`,
+          validate: (input) => input.length >= 10 || 'API key must be at least 10 characters long.',
+        }
+      ]);
+      visionApiKey = newVisionKey;
+      saveEnvVar(visionKeyVar, visionApiKey);
+      console.log(chalk.green(`${visionKeyVar} saved.`));
+    }
+
+    // Model selection for vision provider
+    let visionModel = settings.visionProvider === visionProvider ? settings.visionModel : null;
+
+    if (visionModel) {
+      const { keepVisionModel } = await inquirer.prompt([
+        {
+          type: 'list',
+          name: 'keepVisionModel',
+          message: `Current vision model is ${chalk.yellow(visionModel)}. Keep it or change it?`,
+          choices: [
+            { name: 'Keep current vision model', value: true },
+            { name: 'Change vision model', value: false },
+          ],
+        }
+      ]);
+      if (!keepVisionModel) visionModel = null;
+    }
+
+    if (!visionModel) {
+      if (visionProvider === 'z-ai') {
+        const models = await fetchZaiModels(visionApiKey);
+        const choices = models.length > 0
+          ? models.map(m => ({ name: m.id, value: m.id }))
+          : [];
+        choices.push({ name: 'Enter model ID manually', value: '__manual__' });
+
+        const { browsedVisionModel } = await inquirer.prompt([
+          {
+            type: 'list',
+            name: 'browsedVisionModel',
+            message: 'Select a Z.AI vision model:',
+            choices,
+            pageSize: 20,
+          }
+        ]);
+        if (browsedVisionModel === '__manual__') {
+          const { manualVisionModel } = await inquirer.prompt([
+            {
+              type: 'input',
+              name: 'manualVisionModel',
+              message: 'Enter Z.AI model ID (e.g., glm-4v):',
+              validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+            }
+          ]);
+          visionModel = manualVisionModel.trim();
+        } else {
+          visionModel = browsedVisionModel;
+        }
+      } else {
+        // OpenRouter
+        const { visionModelSelectionMethod } = await inquirer.prompt([
+          {
+            type: 'list',
+            name: 'visionModelSelectionMethod',
+            message: 'How would you like to select the vision model?',
+            choices: [
+              { name: 'Browse OpenRouter models', value: 'browse' },
+              { name: 'Enter model ID manually', value: 'manual' },
+            ],
+          }
+        ]);
+
+        if (visionModelSelectionMethod === 'manual') {
+          const { manualVisionModel } = await inquirer.prompt([
+            {
+              type: 'input',
+              name: 'manualVisionModel',
+              message: 'Enter OpenRouter model ID (e.g., openai/gpt-4o):',
+              validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+            }
+          ]);
+          visionModel = manualVisionModel.trim();
+        } else {
+          const models = await fetchOpenRouterModels(visionApiKey);
+          if (models.length === 0) {
+            console.log(chalk.yellow('Falling back to manual entry due to fetch failure.'));
+            const { manualVisionModel } = await inquirer.prompt([
+              {
+                type: 'input',
+                name: 'manualVisionModel',
+                message: 'Enter OpenRouter model ID:',
+                validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+              }
+            ]);
+            visionModel = manualVisionModel.trim();
+          } else {
+            models.sort((a, b) => {
+              const isFreeA = a.pricing && parseFloat(a.pricing.prompt) === 0 && parseFloat(a.pricing.completion) === 0;
+              const isFreeB = b.pricing && parseFloat(b.pricing.prompt) === 0 && parseFloat(b.pricing.completion) === 0;
+              if (isFreeA && !isFreeB) return -1;
+              if (!isFreeA && isFreeB) return 1;
+              return a.id.localeCompare(b.id);
+            });
+            const choices = models.map(m => {
+              const isFree = m.pricing && parseFloat(m.pricing.prompt) === 0 && parseFloat(m.pricing.completion) === 0;
+              return { name: `${m.id} ${isFree ? chalk.green('(Free)') : ''}`, value: m.id };
+            });
+            const { browsedVisionModel } = await inquirer.prompt([
+              {
+                type: 'list',
+                name: 'browsedVisionModel',
+                message: 'Select a vision model:',
+                choices,
+                pageSize: 20,
+              }
+            ]);
+            visionModel = browsedVisionModel;
+          }
+        }
+      }
+    }
+
+    settings.visionProvider = visionProvider;
+    settings.visionModel = visionModel;
+    saveSettings(settings);
+    console.log(chalk.green(`Vision model ${chalk.bold(visionModel)} saved.`));
+  } else {
+    // Clear vision config if user opts out
+    delete settings.visionProvider;
+    delete settings.visionModel;
+    saveSettings(settings);
+  }
+
   // --- TELEGRAM CHANNEL STEP (OPTIONAL) ---
   const { configureTelegram } = await inquirer.prompt([
     {

package/src/server/agent.js

@@ -15,6 +15,41 @@ const LOOP_DETECTION_THRESHOLD = 3;
 function stripCodeFence(text) {
   return text.replace(/^```(?:json)?\s*\n([\s\S]*?)\n?```\s*$/, '$1').trim();
 }
+
+// Sanitize raw model output before JSON.parse:
+// 1. Strip code fences
+// 2. Escape literal control characters (e.g. real newlines in <pre> blocks) inside
+//    JSON string values — models sometimes forget to escape \n as \\n
+// 3. Remove trailing commas before } and ] (JS-valid but JSON-invalid)
+function sanitizeJson(text) {
+  let s = stripCodeFence(text);
+
+  // State machine: walk the string and fix unescaped control chars inside strings
+  let result = '';
+  let inString = false;
+  let escaped = false;
+  const controlEscapes = { '\n': '\\n', '\r': '\\r', '\t': '\\t' };
+  for (let i = 0; i < s.length; i++) {
+    const ch = s[i];
+    if (escaped) {
+      result += ch;
+      escaped = false;
+    } else if (ch === '\\' && inString) {
+      result += ch;
+      escaped = true;
+    } else if (ch === '"') {
+      result += ch;
+      inString = !inString;
+    } else if (inString && ch.charCodeAt(0) < 0x20) {
+      result += controlEscapes[ch] ?? `\\u${ch.charCodeAt(0).toString(16).padStart(4, '0')}`;
+    } else {
+      result += ch;
+    }
+  }
+
+  // Remove trailing commas before } and ]
+  return result.replace(/,(\s*[}\]])/g, '$1');
+}
 const CONSECUTIVE_FAILURE_THRESHOLD = 3;
 const MAX_TOOL_RESULT = 4000;
 
@@ -402,20 +437,20 @@ export async function runAgentLoop(client, config, session, prepareMessages, usa
         if (nudgeContent.trim()) {
           content = nudgeContent;
         }
-        parsed = JSON.parse(stripCodeFence(nudgeContent));
+        parsed = JSON.parse(sanitizeJson(nudgeContent));
       } catch {
         // Fall through to !parsed handler; content may now carry the nudge text
       }
     } else {
       try {
-        parsed = JSON.parse(stripCodeFence(content));
+        parsed = JSON.parse(sanitizeJson(content));
       } catch {
         // Step 1: retry with fallback model
         try {
           const fallbackResult = await callModel(client, config.fallbackModel, preparedMessages, toolDefs);
           accumulateUsage(usageAccum, fallbackResult);
           const fallbackContent = fallbackResult.choices[0]?.message?.content || '';
-          parsed = JSON.parse(stripCodeFence(fallbackContent));
+          parsed = JSON.parse(sanitizeJson(fallbackContent));
           content = fallbackContent;
         } catch {
           // Step 2: nudge retry via both models
@@ -424,7 +459,7 @@ export async function runAgentLoop(client, config, session, prepareMessages, usa
             const nudgeResult = await callModelWithFallback(client, config, nudgeMessages, toolDefs);
             accumulateUsage(usageAccum, nudgeResult);
             const nudgeContent = nudgeResult.choices[0]?.message?.content || '';
-            parsed = JSON.parse(stripCodeFence(nudgeContent));
+            parsed = JSON.parse(sanitizeJson(nudgeContent));
             content = nudgeContent;
           } catch {
             // Give up
@@ -495,14 +530,14 @@ export async function runAgentLoop(client, config, session, prepareMessages, usa
 
     // Try JSON parse; if it fails, nudge retry (Layer 2)
     try {
-      parsedWrapUp = JSON.parse(stripCodeFence(wrapUpContent));
+      parsedWrapUp = JSON.parse(sanitizeJson(wrapUpContent));
     } catch {
       try {
         const nudgeMessages = [...wrapUpMessages, { role: 'user', content: FORMAT_NUDGE }];
         const nudgeResult = await callModelWithFallback(client, config, nudgeMessages, []);
         accumulateUsage(usageAccum, nudgeResult);
         const nudgeContent = nudgeResult.choices[0]?.message?.content || '';
-        parsedWrapUp = JSON.parse(stripCodeFence(nudgeContent));
+        parsedWrapUp = JSON.parse(sanitizeJson(nudgeContent));
         wrapUpContent = nudgeContent;
       } catch {
         // Layer 3: use raw text as best-effort response below

package/src/server/config.js

@@ -54,6 +54,16 @@ export function loadConfig() {
     if (!apiKey) throw new Error('OPENROUTER_API_KEY not found. Run `jarvis setup` first.');
   }
 
+  // Vision model (optional) — separate provider/model for one-shot image analysis
+  const visionProvider = settings.visionProvider || null;
+  const visionModel = settings.visionModel || null;
+  let visionApiKey = null;
+  if (visionProvider === 'z-ai') {
+    visionApiKey = process.env.ZAI_API_KEY || null;
+  } else if (visionProvider === 'openrouter') {
+    visionApiKey = process.env.OPENROUTER_API_KEY || null;
+  }
+
   return {
     provider,
     apiKey,
@@ -67,6 +77,9 @@ export function loadConfig() {
       token: process.env.TELEGRAM_BOT_TOKEN || null,
       allowedUserIds: settings.channels?.telegram?.allowedUserIds || [],
     },
+    visionProvider,
+    visionModel,
+    visionApiKey,
   };
 }
 

package/src/server/vision.js

@@ -0,0 +1,40 @@
+import { createClient } from './provider.js';
+
+const DEFAULT_PROMPT = 'Please describe this image in detail. Include all visible text, objects, colors, layout, and any other relevant details.';
+
+/**
+ * Sends a one-shot image analysis request to the configured vision model.
+ * No chat history is included — only the image and an optional caption-based prompt.
+ *
+ * @param {object} attachment - { url: 'data:image/jpeg;base64,...' }
+ * @param {string} caption - optional user-provided caption / question
+ * @param {object} config - full app config (must include visionProvider, visionModel, visionApiKey)
+ * @returns {Promise<string>} - the vision model's description
+ */
+export async function describeImage(attachment, caption, config) {
+  const visionConfig = {
+    provider: config.visionProvider,
+    apiKey: config.visionApiKey,
+  };
+
+  const client = createClient(visionConfig);
+
+  const textPrompt = caption?.trim()
+    ? `The user sent this image with the following message: "${caption.trim()}"\n\nPlease describe what you see in the image in detail, paying special attention to anything relevant to their message.`
+    : DEFAULT_PROMPT;
+
+  const response = await client.chat.completions.create({
+    model: config.visionModel,
+    messages: [
+      {
+        role: 'user',
+        content: [
+          { type: 'image_url', image_url: { url: attachment.url } },
+          { type: 'text', text: textPrompt },
+        ],
+      },
+    ],
+  });
+
+  return response.choices[0]?.message?.content?.trim() || '(no description returned)';
+}