nothumanallowed 9.8.1 → 9.8.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nothumanallowed",
-  "version": "9.8.1",
+  "version": "9.8.3",
   "description": "NotHumanAllowed — 38 AI agents, 53 tools. Email, calendar, browser automation, screen capture, canvas, cron/heartbeat, GitHub, Notion, Slack, voice chat, 28 languages. Zero-dependency CLI.",
   "type": "module",
   "bin": {

package/src/commands/chat.mjs

@@ -718,12 +718,31 @@ export async function cmdChat(args) {
           const result = await executeTool(action, params, config);
           process.stdout.write('\r' + ' '.repeat(80) + '\r');
 
-          // Show action-specific result header
-          const resultHeader = formatToolResult(action, params, result);
-          console.log(`  ${resultHeader}`);
-          console.log(`  ${result.split('\n').join('\n  ')}\n`);
-
-          addMessages(conv, input, response + `\n\n[Tool ${action} executed. Result: ${result}]`);
+          // Handle screen capture vision result
+          if (result && typeof result === 'object' && result.__screenshot) {
+            console.log(`  ${G}Screenshot captured${NC} — analyzing with vision...\n`);
+            try {
+              const { callLLMVision } = await import('../services/llm.mjs');
+              const visionResponse = await callLLMVision(config,
+                'Describe EXACTLY and ONLY what you see in this screenshot. NEVER invent or fabricate details.',
+                `The user said: "${input}"\n\n${result.question}`,
+                { base64: result.base64, mimeType: 'image/png' }
+              );
+              console.log(`  ${visionResponse.split('\n').join('\n  ')}\n`);
+              addMessages(conv, input, response + `\n\n[Screenshot: ${result.path}]\n${visionResponse}`);
+            } catch (visionErr) {
+              console.log(`  ${R}Vision failed: ${visionErr.message}${NC}\n`);
+              addMessages(conv, input, response + `\n\n[Screenshot captured but vision failed: ${visionErr.message}]`);
+            }
+          } else {
+            // Show action-specific result header
+            const resultStr = typeof result === 'string' ? result : JSON.stringify(result);
+            const resultHeader = formatToolResult(action, params, resultStr);
+            console.log(`  ${resultHeader}`);
+            console.log(`  ${resultStr.split('\n').join('\n  ')}\n`);
+
+            addMessages(conv, input, response + `\n\n[Tool ${action} executed. Result: ${resultStr}]`);
+          }
         } catch (err) {
           process.stdout.write('\r' + ' '.repeat(80) + '\r');
           console.log(`  ${R}Error executing ${action}: ${err.message}${NC}\n`);

package/src/commands/ui.mjs

@@ -1053,18 +1053,49 @@ export async function cmdUI(args) {
 
           // Execute ALL tool actions and collect results
           const toolResults = [];
+          let screenshotData = null; // For vision: { base64, path, question }
           for (const { action, params } of actions) {
             try {
               const result = await executeTool(action, params, config);
-              toolResults.push({ action, result: typeof result === 'object' ? JSON.stringify(result) : String(result) });
+              // Intercept structured screenshot result for vision flow
+              if (result && typeof result === 'object' && result.__screenshot) {
+                screenshotData = result;
+                toolResults.push({ action, result: 'Screenshot captured. Analyzing with vision...' });
+              } else {
+                toolResults.push({ action, result: typeof result === 'object' ? JSON.stringify(result) : String(result) });
+              }
             } catch (e) {
               toolResults.push({ action, result: `Error: ${e.message}` });
             }
           }
 
           let fullResponse;
-          if (toolResults.length > 0) {
-            // Second LLM call with real tool results — forces the LLM to use actual data
+          if (screenshotData && screenshotData.base64) {
+            // VISION FLOW: send screenshot to LLM as image via callLLMVision
+            try {
+              const { callLLMVision } = await import('../services/llm.mjs');
+              const visionPrompt = enrichedSystemPrompt + '\n\nIMPORTANT: You are looking at a REAL screenshot from the user\'s screen. Describe ONLY what you ACTUALLY see. NEVER invent, guess, or fabricate details. If something is unclear, say so.';
+              const question = `The user said: "${body.message}"\n\n${screenshotData.question}\n\nDescribe ONLY what you see. NEVER make up information.`;
+              fullResponse = await callLLMVision(config, visionPrompt, question, {
+                base64: screenshotData.base64,
+                mimeType: 'image/png',
+              });
+            } catch (visionErr) {
+              fullResponse = `I captured a screenshot but vision analysis failed: ${visionErr.message}. To use screen analysis, configure a vision-capable provider (Claude, GPT-4, Gemini).`;
+            }
+            // Prepend screenshot file marker for the UI to display inline
+            if (screenshotData.path) {
+              const fname = screenshotData.path.split('/').pop();
+              // Copy to NHA screenshots dir for persistence
+              try {
+                const ssDir = path.join(NHA_DIR, 'screenshots');
+                fs.mkdirSync(ssDir, { recursive: true });
+                fs.copyFileSync(screenshotData.path, path.join(ssDir, fname));
+              } catch {}
+              fullResponse = `![Screenshot](/api/screenshots/${fname})\n\n${fullResponse}`;
+            }
+          } else if (toolResults.length > 0) {
+            // Standard tool results flow
             const toolContext = toolResults.map(t => {
               let clean = t.result.replace(/\[Screenshot[^\]]*\]/g, '').replace(/!\[.*?\]\(data:image[^)]+\)/g, '').slice(0, 3000);
               return `[${t.action} result]: ${clean.trim()}`;
@@ -1073,7 +1104,6 @@ export async function cmdUI(args) {
             try {
               fullResponse = await callLLM(config, enrichedSystemPrompt, followUp);
             } catch {
-              // Fallback: show raw results
               fullResponse = toolResults.map(t => `${t.action}: ${t.result}`).join('\n\n');
             }
           } else {
@@ -1325,6 +1355,40 @@ export async function cmdUI(args) {
               }
 
               const result = await executeTool(action, params, config);
+
+              // Intercept screen capture vision result
+              if (result && typeof result === 'object' && result.__screenshot) {
+                sendSSE('tool', { action, status: 'analyzing_screen' });
+                try {
+                  const { callLLMVision } = await import('../services/llm.mjs');
+                  const visionPrompt = enrichedPrompt + '\\n\\nIMPORTANT: You are looking at a REAL screenshot. Describe ONLY what you ACTUALLY see. NEVER invent or fabricate.';
+                  const question = `The user said: "${msg}"\\n\\n${result.question}`;
+                  const visionResponse = await callLLMVision(config, visionPrompt, question, { base64: result.base64, mimeType: 'image/png' });
+
+                  // Save screenshot for display
+                  const fname = result.path.split('/').pop();
+                  const ssDir = path.join(NHA_DIR, 'screenshots');
+                  fs.mkdirSync(ssDir, { recursive: true });
+                  try { fs.copyFileSync(result.path, path.join(ssDir, fname)); } catch {}
+
+                  // Send screenshot to client
+                  sendSSE('screenshot', { filename: fname, format: 'png' });
+                  if (!res._screenshotFiles) res._screenshotFiles = [];
+                  res._screenshotFiles.push(fname);
+
+                  // Send vision analysis as tokens
+                  sendSSE('tool_synthesis', {});
+                  sendSSE('token', { content: visionResponse });
+
+                  toolResults.push({ action, result: `[Screen analyzed] ${visionResponse.slice(0, 500)}` });
+                  sendSSE('tool', { action, status: 'done', result: 'Screen analyzed' });
+                } catch (visionErr) {
+                  toolResults.push({ action, result: `Vision failed: ${visionErr.message}` });
+                  sendSSE('tool', { action, status: 'error', error: visionErr.message });
+                }
+                continue;
+              }
+
               const resultStr = typeof result === 'object' ? JSON.stringify(result) : String(result);
               toolResults.push({ action, result: resultStr });
               sendSSE('tool', { action, status: 'done', result: typeof resultStr === 'string' ? resultStr.slice(0, 500) : '' });

package/src/constants.mjs

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-export const VERSION = '9.8.1';
+export const VERSION = '9.8.3';
 export const BASE_URL = 'https://nothumanallowed.com/cli';
 export const API_BASE = 'https://nothumanallowed.com/api/v1';
 

package/src/services/llm.mjs

@@ -275,6 +275,96 @@ export async function callLLM(config, systemPrompt, userMessage, opts = {}) {
   return callFn(apiKey, model, systemPrompt, userMessage, false);
 }
 
+/**
+ * Call LLM with multimodal (vision) messages — supports image + PDF content.
+ * Unified function for ALL vision calls (web UI, CLI, screen capture).
+ * @param {object} config
+ * @param {string} systemPrompt
+ * @param {string} userMessage — text question about the image
+ * @param {object} media — { base64, mimeType } (image/png, image/jpeg, application/pdf)
+ * @returns {Promise<string>}
+ */
+export async function callLLMVision(config, systemPrompt, userMessage, media) {
+  const provider = config.llm.provider || 'anthropic';
+  const model = config.llm.model || null;
+  const apiKey = getApiKey(config, provider);
+  if (!apiKey) throw new Error(`No API key for ${provider}. Vision requires Claude, GPT-4, or Gemini.`);
+
+  const { base64, mimeType } = media;
+  if (!base64 || !mimeType) throw new Error('media.base64 and media.mimeType are required');
+
+  if (provider === 'anthropic') {
+    const isPdf = mimeType === 'application/pdf';
+    const contentBlock = isPdf
+      ? { type: 'document', source: { type: 'base64', media_type: mimeType, data: base64 } }
+      : { type: 'image', source: { type: 'base64', media_type: mimeType, data: base64 } };
+
+    const body = {
+      model: model || 'claude-sonnet-4-20250514',
+      max_tokens: 4096,
+      system: systemPrompt,
+      messages: [{ role: 'user', content: [contentBlock, { type: 'text', text: userMessage }] }],
+    };
+    const res = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) throw new Error(`Anthropic vision ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    const data = await res.json();
+    return data.content?.[0]?.text || '';
+  }
+
+  if (provider === 'openai' || provider === 'deepseek' || provider === 'grok' || provider === 'mistral') {
+    const url = provider === 'openai' ? 'https://api.openai.com/v1/chat/completions'
+              : provider === 'deepseek' ? 'https://api.deepseek.com/v1/chat/completions'
+              : provider === 'grok' ? 'https://api.x.ai/v1/chat/completions'
+              : 'https://api.mistral.ai/v1/chat/completions';
+
+    const body = {
+      model: model || (provider === 'openai' ? 'gpt-4o' : model),
+      max_tokens: 4096,
+      messages: [
+        { role: 'system', content: systemPrompt },
+        { role: 'user', content: [
+          { type: 'image_url', image_url: { url: `data:${mimeType};base64,${base64}` } },
+          { type: 'text', text: userMessage },
+        ] },
+      ],
+    };
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}` },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) throw new Error(`${provider} vision ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    const data = await res.json();
+    return data.choices?.[0]?.message?.content || '';
+  }
+
+  if (provider === 'gemini') {
+    const geminiModel = model || 'gemini-2.0-flash';
+    const body = {
+      contents: [{ parts: [
+        { inline_data: { mime_type: mimeType, data: base64 } },
+        { text: userMessage },
+      ] }],
+      systemInstruction: { parts: [{ text: systemPrompt }] },
+      generationConfig: { maxOutputTokens: 4096 },
+    };
+    const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${apiKey}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) throw new Error(`Gemini vision ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    const data = await res.json();
+    return data.candidates?.[0]?.content?.parts?.[0]?.text || '';
+  }
+
+  throw new Error(`Vision not supported for provider: ${provider}. Use Claude, GPT-4, or Gemini.`);
+}
+
 /**
  * Call an LLM provider with streaming enabled.
  * Calls onToken(chunk) for each token, returns full text at the end.

package/src/services/web-ui.mjs

@@ -94,7 +94,18 @@ input:focus,textarea:focus{border-color:var(--green3)}
 .msg{margin-bottom:12px}
 .msg--user .msg__bubble{background:var(--bg3);border:1px solid var(--border2);border-radius:8px 8px 2px 8px;padding:10px 14px;max-width:85%;margin-left:auto;color:var(--bright)}
 .msg--assistant .msg__bubble{background:var(--greendim);border:1px solid var(--green3);border-radius:8px 8px 8px 2px;padding:10px 14px;max-width:85%;color:var(--text);white-space:pre-wrap;word-wrap:break-word}
+.msg--assistant .msg__bubble img{max-width:100%;border-radius:8px;margin:8px 0;border:1px solid rgba(0,255,65,0.2)}
 .msg__label{font-size:10px;color:var(--dim);margin-bottom:2px}
+.msg__actions{display:flex;gap:6px;margin-top:4px;opacity:0.15;transition:opacity 0.2s}
+.msg:hover .msg__actions{opacity:1}
+.msg__actions button{background:none;border:none;color:var(--dim);cursor:pointer;font-size:10px;font-family:var(--mono);padding:2px 4px}
+.msg__actions button:hover{color:var(--green)}
+#canvasPanel{position:fixed;top:60px;right:12px;width:480px;max-height:calc(100vh - 80px);background:#0d0d0d;border:1px solid var(--green);border-radius:12px;z-index:1000;overflow:hidden;display:none;flex-direction:column;box-shadow:0 0 30px rgba(0,255,65,0.1)}
+#canvasPanel.open{display:flex}
+#canvasPanel .cvs-header{display:flex;align-items:center;justify-content:space-between;padding:8px 12px;border-bottom:1px solid var(--green);background:rgba(0,255,65,0.05)}
+#canvasPanel .cvs-header span{font-family:var(--mono);color:var(--green);font-size:12px}
+#canvasPanel .cvs-header button{background:none;border:none;color:var(--dim);cursor:pointer;font-size:14px;margin-left:8px}
+#canvasPanel iframe{flex:1;border:none;background:#fff;min-height:350px;width:100%}
 .msg--thinking{color:var(--dim);font-style:italic}
 .tool-indicator{display:inline-block;padding:2px 8px;margin:2px 0;border-radius:4px;font-size:11px;background:var(--bg3);border:1px solid var(--border)}
 .tool-indicator--browser{border-color:#9c27b0;color:#ce93d8}
@@ -433,17 +444,32 @@ function renderMessages(){
     el.innerHTML='<div class="chat__empty"><div class="chat__empty-title">NHA Chat</div><div>Personal Operations Assistant — Streaming + Web Search + Browser</div><div class="chat__empty-hint">Try: Show my unread emails / Search the web for React 19 / Open google.com and take a screenshot</div></div>';
     return;
   }
-  var h='';chatHistory.forEach(function(m){
+  var h='';chatHistory.forEach(function(m,mi){
     var raw=m.content||'';
-    // Strip any raw base64 data that leaked into content (from LLM hallucinations)
+    var isA=m.role==='assistant';
+    // Strip any raw base64 data that leaked into content
     raw=raw.replace(/data:image\\/[a-z]+;base64,[A-Za-z0-9+\\/=]{200,}/g,'[image]');
     raw=raw.replace(/[A-Za-z0-9+\\/=]{500,}/g,'');
+    // Handle canvas markers (assistant only)
+    if(isA){
+      var cm=raw.match(/\\[CANVAS_RENDER\\]([\\s\\S]*?)\\[\\/CANVAS_RENDER\\]/);
+      if(cm){try{var cd=JSON.parse(cm[1]);showCanvas(cd.html,cd.title);}catch(e){} raw=raw.replace(/\\[CANVAS_RENDER\\][\\s\\S]*?\\[\\/CANVAS_RENDER\\]/,'').trim();}
+      if(raw.indexOf('[CANVAS_CLEAR]')!==-1){closeCanvas();raw=raw.replace(/\\[CANVAS_CLEAR\\][\\s\\S]*?\\[\\/CANVAS_CLEAR\\]/,'').trim();}
+      // Handle screenshot file markers
+      var sm=raw.match(/\\[SCREENSHOT_FILE\\](.*?)\\[\\/SCREENSHOT_FILE\\]/);
+      if(sm){var fn=sm[1].split('/').pop();raw=raw.replace(/\\[SCREENSHOT_FILE\\].*?\\[\\/SCREENSHOT_FILE\\]/,'');raw='![Screenshot](/api/screenshots/'+fn+')\\n'+raw;}
+    }
     var imgs=[];var idx=0;
-    // Match both /api/screenshots/ URLs and data:image (short ones only, for inline display)
     var safe=raw.replace(/!\\[([^\\]]*)\\]\\((\\/api\\/screenshots\\/[a-zA-Z0-9._-]+)\\)/g,function(_,alt,src){var ph='__IMG'+idx+'__';imgs.push({ph:ph,alt:alt,src:src});idx++;return ph;});
     var content=esc(safe);
     for(var i=0;i<imgs.length;i++){content=content.replace(imgs[i].ph,'<img class="screenshot-preview" alt="'+esc(imgs[i].alt)+'" src="'+imgs[i].src+'">');}
-    h+='<div class="msg msg--'+esc(m.role)+'"><div class="msg__label">'+esc(m.role==='user'?'You':'NHA')+'</div><div class="msg__bubble">'+content+'</div></div>';
+    // Action buttons
+    var acts='<div class="msg__actions">';
+    acts+='<button onclick="copyMsg('+mi+')">Copy</button>';
+    if(isA){acts+='<button onclick="retryMsg('+mi+')">Retry</button>';}
+    else{acts+='<button onclick="editMsg('+mi+')">Edit</button>';}
+    acts+='</div>';
+    h+='<div class="msg msg--'+esc(m.role)+'"><div class="msg__label">'+esc(m.role==='user'?'You':'NHA')+'</div><div class="msg__bubble">'+content+'</div>'+acts+'</div>';
   });
   el.innerHTML=h;el.scrollTop=el.scrollHeight;
 }
@@ -496,6 +522,42 @@ function clearChatAttach(){
   document.getElementById('chatImageInput').value='';
 }
 
+// ---- CANVAS ----
+function showCanvas(html,title){
+  var p=document.getElementById('canvasPanel');
+  var t=document.getElementById('canvasTitle');
+  var f=document.getElementById('canvasFrame');
+  if(!p||!f)return;
+  if(t)t.textContent=title||'Canvas';
+  p.classList.add('open');
+  var doc=f.contentDocument||f.contentWindow.document;
+  doc.open();doc.write(html);doc.close();
+}
+function closeCanvas(){var p=document.getElementById('canvasPanel');if(p)p.classList.remove('open');}
+function toggleCanvasSize(){
+  var p=document.getElementById('canvasPanel');if(!p)return;
+  if(p.style.width==='80vw'){p.style.width='';p.style.height='';p.style.top='';p.style.right='';}
+  else{p.style.width='80vw';p.style.height='80vh';p.style.top='10vh';p.style.right='10vw';}
+}
+// ---- MSG ACTIONS ----
+function copyMsg(i){
+  var m=chatHistory[i];if(!m)return;
+  var t=(m.content||'').replace(/\\[CANVAS_RENDER\\][\\s\\S]*?\\[\\/CANVAS_RENDER\\]/g,'').replace(/\\[SCREENSHOT_FILE\\].*?\\[\\/SCREENSHOT_FILE\\]/g,'').trim();
+  navigator.clipboard.writeText(t).catch(function(){});
+}
+function retryMsg(i){
+  if(i<1||chatHistory[i].role!=='assistant')return;
+  var userMsg=chatHistory[i-1];if(!userMsg||userMsg.role!=='user')return;
+  chatHistory.splice(i,1);saveChatToStorage();renderMessages();
+  var inp=document.getElementById('chatInput');if(inp){inp.value=userMsg.content;}
+  sendChat();
+}
+function editMsg(i){
+  if(chatHistory[i].role!=='user')return;
+  var inp=document.getElementById('chatInput');if(!inp)return;
+  inp.value=chatHistory[i].content;inp.focus();
+  chatHistory.splice(i);saveChatToStorage();renderMessages();
+}
 function sendChat(){
   var inp=document.getElementById('chatInput');if(!inp)return;
   var msg=inp.value.trim();
@@ -1956,6 +2018,7 @@ init();
   </div>
 </div>
 
+<div id="canvasPanel"><div class="cvs-header"><span id="canvasTitle">Canvas</span><div><button onclick="toggleCanvasSize()" title="Resize">&#x2922;</button><button onclick="closeCanvas()" title="Close">&times;</button></div></div><iframe id="canvasFrame" sandbox="allow-scripts allow-same-origin"></iframe></div>
 <script>${JS}</script>
 </body>
 </html>`;