@ducci/jarvis 1.0.67 → 1.0.69

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ducci/jarvis",
-  "version": "1.0.67",
+  "version": "1.0.69",
   "description": "A fully automated agent system that lives on a server.",
   "main": "./src/index.js",
   "type": "module",

package/src/channels/telegram/index.js

@@ -6,6 +6,7 @@ import { handleChat } from '../../server/agent.js';
 import { loadSession } from '../../server/sessions.js';
 import { PATHS } from '../../server/config.js';
 import { load, save } from './sessions.js';
+import { describeImage } from '../../server/vision.js';
 
 function getTelegramChatLogPath(chatId, sessionId) {
   const prefix = sessionId ? String(sessionId).slice(0, 8) : 'unknown';
@@ -123,11 +124,39 @@ export async function startTelegramChannel(config) {
       const combinedText = batch.length === 1
         ? batch[0].text
         : batch.map(m => m.text).join('\n\n');
-      const allAttachments = batch.flatMap(m => m.attachments);
+      const rawAttachments = batch.flatMap(m => m.attachments);
+
+      // If a vision model is configured and the batch contains images, run a one-shot
+      // image analysis first and inject the result as text into the main agent turn.
+      let userText = combinedText;
+      let allAttachments = rawAttachments;
+      if (config.visionModel && config.visionApiKey && rawAttachments.length > 0) {
+        try {
+          const results = await Promise.allSettled(
+            rawAttachments.map(a => describeImage(a, combinedText, config))
+          );
+          const successfulDescs = results
+            .filter(r => r.status === 'fulfilled')
+            .map(r => `[Image analysis: ${r.value}]`);
+          const failedCount = results.filter(r => r.status === 'rejected').length;
+          if (failedCount > 0) {
+            console.error(`[telegram] vision error for ${failedCount}/${results.length} image(s), those will be sent directly to main agent`);
+          }
+          if (successfulDescs.length > 0) {
+            const descBlock = successfulDescs.join('\n\n');
+            userText = combinedText ? `${descBlock}\n\n${combinedText}` : descBlock;
+          }
+          // Only clear attachments for images that were successfully described
+          allAttachments = rawAttachments.filter((_, i) => results[i].status === 'rejected');
+        } catch (e) {
+          console.error(`[telegram] vision error, falling back to direct image: ${e.message}`);
+          // allAttachments and userText stay unchanged — main agent gets the raw image
+        }
+      }
 
       let result;
       try {
-        result = await handleChat(config, sessionId, combinedText, allAttachments);
+        result = await handleChat(config, sessionId, userText, allAttachments);
       } catch (e) {
         console.error(`[telegram] agent error chat_id=${chatId}: ${e.message}`);
         const errText = e.message

package/src/scripts/onboarding.js

@@ -389,6 +389,203 @@ async function run() {
   saveSettings(settings);
   console.log(chalk.green(`\nModel ${chalk.bold(selectedModel)} saved to settings.`));
 
+  // --- VISION MODEL STEP (OPTIONAL) ---
+  let skipVision = false;
+  if (settings.visionModel) {
+    const { visionAction } = await inquirer.prompt([
+      {
+        type: 'list',
+        name: 'visionAction',
+        message: `Vision model is configured (${chalk.yellow(settings.visionModel)}). What do you want to do?`,
+        choices: [
+          { name: 'Keep current vision model', value: 'keep' },
+          { name: 'Change vision model', value: 'change' },
+          { name: 'Disable vision', value: 'disable' },
+        ],
+      }
+    ]);
+    if (visionAction === 'keep') {
+      skipVision = true;
+    } else if (visionAction === 'disable') {
+      delete settings.visionProvider;
+      delete settings.visionModel;
+      saveSettings(settings);
+      console.log(chalk.yellow('Vision model disabled.'));
+      skipVision = true;
+    }
+  } else {
+    const { configureVision } = await inquirer.prompt([
+      {
+        type: 'confirm',
+        name: 'configureVision',
+        message: 'Do you want to configure a separate vision model for image analysis (e.g. for Telegram photos)?',
+        default: false,
+      }
+    ]);
+    if (!configureVision) skipVision = true;
+  }
+
+  if (!skipVision) {
+    const { visionProvider } = await inquirer.prompt([
+      {
+        type: 'list',
+        name: 'visionProvider',
+        message: 'Which provider for the vision model?',
+        choices: [
+          { name: 'OpenRouter (access many models via one key)', value: 'openrouter' },
+          { name: 'Z.AI Direct (GLM models)', value: 'z-ai' },
+        ],
+        default: settings.visionProvider || provider,
+      }
+    ]);
+
+    // Ensure the key for the vision provider is available
+    const visionKeyVar = visionProvider === 'z-ai' ? 'ZAI_API_KEY' : 'OPENROUTER_API_KEY';
+    let visionApiKey = loadEnvVar(visionKeyVar);
+    if (visionApiKey) {
+      const { keepVisionKey } = await inquirer.prompt([
+        {
+          type: 'confirm',
+          name: 'keepVisionKey',
+          message: `A ${visionKeyVar} is already configured. Use it for the vision model?`,
+          default: true,
+        }
+      ]);
+      if (!keepVisionKey) visionApiKey = null;
+    }
+    if (!visionApiKey) {
+      console.log(chalk.yellow(`No ${visionKeyVar} found — needed for the vision model.`));
+      const { newVisionKey } = await inquirer.prompt([
+        {
+          type: 'password',
+          name: 'newVisionKey',
+          message: `Enter your ${visionProvider === 'z-ai' ? 'Z.AI' : 'OpenRouter'} API key:`,
+          validate: (input) => input.length >= 10 || 'API key must be at least 10 characters long.',
+        }
+      ]);
+      visionApiKey = newVisionKey;
+      saveEnvVar(visionKeyVar, visionApiKey);
+      console.log(chalk.green(`${visionKeyVar} saved.`));
+    }
+
+    // Model selection for vision provider
+    let visionModel = settings.visionProvider === visionProvider ? settings.visionModel : null;
+
+    if (visionModel) {
+      const { keepVisionModel } = await inquirer.prompt([
+        {
+          type: 'list',
+          name: 'keepVisionModel',
+          message: `Current vision model is ${chalk.yellow(visionModel)}. Keep it or change it?`,
+          choices: [
+            { name: 'Keep current vision model', value: true },
+            { name: 'Change vision model', value: false },
+          ],
+        }
+      ]);
+      if (!keepVisionModel) visionModel = null;
+    }
+
+    if (!visionModel) {
+      if (visionProvider === 'z-ai') {
+        const models = await fetchZaiModels(visionApiKey);
+        const choices = models.length > 0
+          ? models.map(m => ({ name: m.id, value: m.id }))
+          : [];
+        choices.push({ name: 'Enter model ID manually', value: '__manual__' });
+
+        const { browsedVisionModel } = await inquirer.prompt([
+          {
+            type: 'list',
+            name: 'browsedVisionModel',
+            message: 'Select a Z.AI vision model:',
+            choices,
+            pageSize: 20,
+          }
+        ]);
+        if (browsedVisionModel === '__manual__') {
+          const { manualVisionModel } = await inquirer.prompt([
+            {
+              type: 'input',
+              name: 'manualVisionModel',
+              message: 'Enter Z.AI model ID (e.g., glm-4v):',
+              validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+            }
+          ]);
+          visionModel = manualVisionModel.trim();
+        } else {
+          visionModel = browsedVisionModel;
+        }
+      } else {
+        // OpenRouter
+        const { visionModelSelectionMethod } = await inquirer.prompt([
+          {
+            type: 'list',
+            name: 'visionModelSelectionMethod',
+            message: 'How would you like to select the vision model?',
+            choices: [
+              { name: 'Browse OpenRouter models', value: 'browse' },
+              { name: 'Enter model ID manually', value: 'manual' },
+            ],
+          }
+        ]);
+
+        if (visionModelSelectionMethod === 'manual') {
+          const { manualVisionModel } = await inquirer.prompt([
+            {
+              type: 'input',
+              name: 'manualVisionModel',
+              message: 'Enter OpenRouter model ID (e.g., openai/gpt-4o):',
+              validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+            }
+          ]);
+          visionModel = manualVisionModel.trim();
+        } else {
+          const models = await fetchOpenRouterModels(visionApiKey);
+          if (models.length === 0) {
+            console.log(chalk.yellow('Falling back to manual entry due to fetch failure.'));
+            const { manualVisionModel } = await inquirer.prompt([
+              {
+                type: 'input',
+                name: 'manualVisionModel',
+                message: 'Enter OpenRouter model ID:',
+                validate: (input) => input.trim().length > 0 || 'Model ID cannot be empty.',
+              }
+            ]);
+            visionModel = manualVisionModel.trim();
+          } else {
+            models.sort((a, b) => {
+              const isFreeA = a.pricing && parseFloat(a.pricing.prompt) === 0 && parseFloat(a.pricing.completion) === 0;
+              const isFreeB = b.pricing && parseFloat(b.pricing.prompt) === 0 && parseFloat(b.pricing.completion) === 0;
+              if (isFreeA && !isFreeB) return -1;
+              if (!isFreeA && isFreeB) return 1;
+              return a.id.localeCompare(b.id);
+            });
+            const choices = models.map(m => {
+              const isFree = m.pricing && parseFloat(m.pricing.prompt) === 0 && parseFloat(m.pricing.completion) === 0;
+              return { name: `${m.id} ${isFree ? chalk.green('(Free)') : ''}`, value: m.id };
+            });
+            const { browsedVisionModel } = await inquirer.prompt([
+              {
+                type: 'list',
+                name: 'browsedVisionModel',
+                message: 'Select a vision model:',
+                choices,
+                pageSize: 20,
+              }
+            ]);
+            visionModel = browsedVisionModel;
+          }
+        }
+      }
+    }
+
+    settings.visionProvider = visionProvider;
+    settings.visionModel = visionModel;
+    saveSettings(settings);
+    console.log(chalk.green(`Vision model ${chalk.bold(visionModel)} saved.`));
+  }
+
   // --- TELEGRAM CHANNEL STEP (OPTIONAL) ---
   const { configureTelegram } = await inquirer.prompt([
     {
@@ -464,44 +661,6 @@ async function run() {
     }
   }
 
-  // --- PERPLEXITY STEP (OPTIONAL) ---
-  const existingPerplexityKey = loadEnvVar('PERPLEXITY_API_KEY');
-  const { configurePerplexity } = await inquirer.prompt([
-    {
-      type: 'confirm',
-      name: 'configurePerplexity',
-      message: 'Do you want to configure Perplexity web search?',
-      default: !!existingPerplexityKey
-    }
-  ]);
-
-  if (configurePerplexity) {
-    let keepPerplexityKey = false;
-    if (existingPerplexityKey) {
-      const { keep } = await inquirer.prompt([
-        {
-          type: 'confirm',
-          name: 'keep',
-          message: 'A PERPLEXITY_API_KEY is already configured. Do you want to keep it?',
-          default: true
-        }
-      ]);
-      keepPerplexityKey = keep;
-    }
-    if (!keepPerplexityKey) {
-      const { perplexityKey } = await inquirer.prompt([
-        {
-          type: 'password',
-          name: 'perplexityKey',
-          message: 'Enter your Perplexity API key (from perplexity.ai/settings/api):',
-          validate: (input) => input.trim().length > 0 || 'API key cannot be empty.'
-        }
-      ]);
-      saveEnvVar('PERPLEXITY_API_KEY', perplexityKey.trim());
-      console.log(chalk.green('Perplexity API key saved.'));
-    }
-  }
-
   // --- PM2 + LOG ROTATION STEP ---
   const pm2Check = spawnSync('pm2', ['--version'], { stdio: 'pipe' });
   if (pm2Check.status !== 0) {

package/src/server/config.js

@@ -54,6 +54,16 @@ export function loadConfig() {
     if (!apiKey) throw new Error('OPENROUTER_API_KEY not found. Run `jarvis setup` first.');
   }
 
+  // Vision model (optional) — separate provider/model for one-shot image analysis
+  const visionProvider = settings.visionProvider || null;
+  const visionModel = settings.visionModel || null;
+  let visionApiKey = null;
+  if (visionProvider === 'z-ai') {
+    visionApiKey = process.env.ZAI_API_KEY || null;
+  } else if (visionProvider === 'openrouter') {
+    visionApiKey = process.env.OPENROUTER_API_KEY || null;
+  }
+
   return {
     provider,
     apiKey,
@@ -67,6 +77,9 @@ export function loadConfig() {
       token: process.env.TELEGRAM_BOT_TOKEN || null,
       allowedUserIds: settings.channels?.telegram?.allowedUserIds || [],
     },
+    visionProvider,
+    visionModel,
+    visionApiKey,
   };
 }
 

package/src/server/tools.js

@@ -656,6 +656,71 @@ const SEED_TOOLS = {
       return { status: 'ok', name: args.name, content };
     `,
   },
+  analyze_image: {
+    definition: {
+      type: 'function',
+      function: {
+        name: 'analyze_image',
+        description: 'Fetch an image from a URL and analyze it using the configured vision model. Returns a detailed description of the image. Use this whenever a user shares an image URL and asks about its content.',
+        parameters: {
+          type: 'object',
+          properties: {
+            url: {
+              type: 'string',
+              description: 'The URL of the image to analyze (http or https).',
+            },
+            prompt: {
+              type: 'string',
+              description: 'Optional question or instruction for the vision model, e.g. "What text is visible?" or "Describe the chart". Defaults to a general description.',
+            },
+          },
+          required: ['url'],
+        },
+      },
+    },
+    code: `
+      const settingsPath = path.join(process.env.HOME, '.jarvis/data/config/settings.json');
+      const settings = JSON.parse(await fs.promises.readFile(settingsPath, 'utf8').catch(() => '{}'));
+      const visionModel = settings.visionModel;
+      const visionProvider = settings.visionProvider;
+      if (!visionModel || !visionProvider) {
+        return { status: 'error', message: 'No vision model configured. Set visionModel and visionProvider in settings.' };
+      }
+      let apiKey, baseURL;
+      if (visionProvider === 'z-ai') {
+        apiKey = process.env.ZAI_API_KEY;
+        baseURL = 'https://api.z.ai/api/coding/paas/v4/';
+      } else {
+        apiKey = process.env.OPENROUTER_API_KEY;
+        baseURL = 'https://openrouter.ai/api/v1';
+      }
+      if (!apiKey) return { status: 'error', message: 'No API key found for vision provider: ' + visionProvider };
+      const imgResponse = await fetch(args.url);
+      if (!imgResponse.ok) return { status: 'error', message: 'Failed to fetch image: HTTP ' + imgResponse.status };
+      const buffer = await imgResponse.arrayBuffer();
+      const base64 = Buffer.from(buffer).toString('base64');
+      const contentType = imgResponse.headers.get('content-type') || 'image/jpeg';
+      const dataUrl = 'data:' + contentType + ';base64,' + base64;
+      const textPrompt = args.prompt?.trim()
+        ? 'The user shared this image with the following question/context: "' + args.prompt.trim() + '"\\n\\nPlease describe what you see, paying special attention to anything relevant to their message.'
+        : 'Please describe this image in detail. Include all visible text, objects, colors, layout, and any other relevant details.';
+      const apiResponse = await fetch(baseURL + (baseURL.endsWith('/') ? '' : '/') + 'chat/completions', {
+        method: 'POST',
+        headers: { 'Authorization': 'Bearer ' + apiKey, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: visionModel,
+          messages: [{ role: 'user', content: [
+            { type: 'image_url', image_url: { url: dataUrl } },
+            { type: 'text', text: textPrompt },
+          ]}],
+        }),
+      });
+      const result = await apiResponse.json();
+      if (!apiResponse.ok) return { status: 'error', message: result.error?.message || 'Vision API error' };
+      const description = result.choices?.[0]?.message?.content?.trim() || '(no description returned)';
+      return { status: 'ok', description };
+    `,
+  },
 };
 
 export function seedTools() {

package/src/server/vision.js

@@ -0,0 +1,40 @@
+import { createClient } from './provider.js';
+
+const DEFAULT_PROMPT = 'Please describe this image in detail. Include all visible text, objects, colors, layout, and any other relevant details.';
+
+/**
+ * Sends a one-shot image analysis request to the configured vision model.
+ * No chat history is included — only the image and an optional caption-based prompt.
+ *
+ * @param {object} attachment - { url: 'data:image/jpeg;base64,...' }
+ * @param {string} caption - optional user-provided caption / question
+ * @param {object} config - full app config (must include visionProvider, visionModel, visionApiKey)
+ * @returns {Promise<string>} - the vision model's description
+ */
+export async function describeImage(attachment, caption, config) {
+  const visionConfig = {
+    provider: config.visionProvider,
+    apiKey: config.visionApiKey,
+  };
+
+  const client = createClient(visionConfig);
+
+  const textPrompt = caption?.trim()
+    ? `The user sent this image with the following message: "${caption.trim()}"\n\nPlease describe what you see in the image in detail, paying special attention to anything relevant to their message.`
+    : DEFAULT_PROMPT;
+
+  const response = await client.chat.completions.create({
+    model: config.visionModel,
+    messages: [
+      {
+        role: 'user',
+        content: [
+          { type: 'image_url', image_url: { url: attachment.url } },
+          { type: 'text', text: textPrompt },
+        ],
+      },
+    ],
+  });
+
+  return response.choices[0]?.message?.content?.trim() || '(no description returned)';
+}