open-agents-ai 0.187.164 → 0.187.166

package/dist/index.js

@@ -257987,8 +257987,33 @@ var init_multimodal_memory = __esm({
           } catch {
           }
           if (existsSync36(imagePath)) {
-            episode.visual = { faceIds: [], faceNames: [], objects: [], imagePath };
+            episode.visual = { faceIds: [], faceNames: [], objects: [], imagePath, clipEmbedding: null };
             results.push("Photo captured");
+            try {
+              const venvPy = join50(homedir15(), ".open-agents", "vision-ml-venv", "bin", "python3");
+              if (existsSync36(venvPy)) {
+                const clipScript = `
+import json, torch
+from PIL import Image
+from transformers import CLIPProcessor, CLIPModel
+model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+img = Image.open("${imagePath}").convert("RGB")
+inputs = processor(images=img, return_tensors="pt")
+with torch.no_grad():
+    features = model.get_image_features(**inputs)
+    features = features / features.norm(dim=-1, keepdim=True)
+print(json.dumps(features[0].cpu().numpy().tolist()))
+`;
+                const scriptFile = join50(tmpdir14(), `mm-clip-${Date.now()}.py`);
+                writeFileSync15(scriptFile, clipScript);
+                const clipOutput = execSync38(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 12e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
+                const embedding = JSON.parse(clipOutput.trim().split("\n").pop());
+                episode.visual.clipEmbedding = embedding;
+                results.push(`CLIP embedding computed (${embedding.length}d)`);
+              }
+            } catch {
+            }
             try {
               const venvPy = join50(homedir15(), ".open-agents", "vision-ml-venv", "bin", "python3");
               if (existsSync36(venvPy)) {
@@ -258188,6 +258213,28 @@ Recall later: multimodal_memory action=recall query="${personName}"`,
           return { success: true, output: "No multi-modal episodes recorded yet. Use 'capture' or 'meet' first.", durationMs: performance.now() - start2 };
         }
         const queryLower = query.toLowerCase();
+        let queryClipEmbedding = null;
+        try {
+          const venvPy = join50(homedir15(), ".open-agents", "vision-ml-venv", "bin", "python3");
+          if (existsSync36(venvPy)) {
+            const clipTextScript = `
+import json, torch
+from transformers import CLIPProcessor, CLIPModel
+model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+inputs = processor(text=["${query.replace(/"/g, '\\"').replace(/\n/g, " ")}"], return_tensors="pt", padding=True)
+with torch.no_grad():
+    features = model.get_text_features(**inputs)
+    features = features / features.norm(dim=-1, keepdim=True)
+print(json.dumps(features[0].cpu().numpy().tolist()))
+`;
+            const scriptFile = join50(tmpdir14(), `mm-clipq-${Date.now()}.py`);
+            writeFileSync15(scriptFile, clipTextScript);
+            const output = execSync38(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 6e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
+            queryClipEmbedding = JSON.parse(output.trim().split("\n").pop());
+          }
+        } catch {
+        }
         const scored = episodes.map((ep) => {
           let score = 0;
           const matchedModalities = [];
@@ -258219,6 +258266,13 @@ Recall later: multimodal_memory action=recall query="${personName}"`,
             score += 4;
             matchedModalities.push(`location: ${ep.spatial.locationLabel}`);
           }
+          if (queryClipEmbedding && ep.visual?.clipEmbedding) {
+            const clipSim = this.cosineSim(queryClipEmbedding, ep.visual.clipEmbedding);
+            if (clipSim > 0.2) {
+              score += clipSim * 8;
+              matchedModalities.push(`CLIP visual: ${(clipSim * 100).toFixed(0)}%`);
+            }
+          }
           const hoursSince = (Date.now() - ep.timestamp) / 36e5;
           const recency = Math.pow(0.995, hoursSince);
           score *= 0.5 + 0.5 * recency;
@@ -258311,6 +258365,18 @@ ${lines.join("\n")}`,
           return null;
         }
       }
+      cosineSim(a2, b) {
+        if (a2.length !== b.length || a2.length === 0)
+          return 0;
+        let dot = 0, normA = 0, normB = 0;
+        for (let i2 = 0; i2 < a2.length; i2++) {
+          dot += a2[i2] * b[i2];
+          normA += a2[i2] * a2[i2];
+          normB += b[i2] * b[i2];
+        }
+        const denom = Math.sqrt(normA) * Math.sqrt(normB);
+        return denom > 0 ? dot / denom : 0;
+      }
       loadAllEpisodes() {
         const episodes = [];
         try {
@@ -290836,8 +290902,29 @@ function ensureVenv(log22) {
   const venvDir = getVenvDir();
   const isWin2 = process.platform === "win32";
   const pipPath = isWin2 ? join71(venvDir, "Scripts", "pip.exe") : join71(venvDir, "bin", "pip");
+  const venvPyPath = isWin2 ? join71(venvDir, "Scripts", "python.exe") : join71(venvDir, "bin", "python3");
   const pythonCmd = isWin2 ? "python" : "python3";
-  if (existsSync55(pipPath)) return venvDir;
+  if (existsSync55(pipPath)) {
+    try {
+      execSync47(`"${venvPyPath}" -m pip --version`, { stdio: "pipe", timeout: 1e4 });
+      return venvDir;
+    } catch {
+      log22("Python venv pip is broken \u2014 repairing...");
+      try {
+        execSync47(`"${venvPyPath}" -m ensurepip --upgrade`, { stdio: "pipe", timeout: 3e4 });
+        log22("pip repaired via ensurepip.");
+        return venvDir;
+      } catch {
+        try {
+          execSync47(`curl -sS https://bootstrap.pypa.io/get-pip.py | "${venvPyPath}"`, { stdio: "pipe", timeout: 6e4 });
+          log22("pip repaired via get-pip.py.");
+          return venvDir;
+        } catch {
+          log22("pip repair failed \u2014 recreating venv...");
+        }
+      }
+    }
+  }
   log22("Creating Python venv for vision deps...");
   if (!hasCmd(pythonCmd) && !hasCmd("python3")) {
     log22(`${pythonCmd} not found \u2014 cannot create venv.`);
@@ -290850,8 +290937,17 @@ function ensureVenv(log22) {
   try {
     mkdirSync28(join71(homedir22(), ".open-agents"), { recursive: true });
     const pyCmd = hasCmd(pythonCmd) ? pythonCmd : "python3";
-    execSync47(`${pyCmd} -m venv "${venvDir}"`, { stdio: "pipe", timeout: 3e4 });
-    execSync47(`"${pipPath}" install --upgrade pip`, {
+    execSync47(`${pyCmd} -m venv --clear "${venvDir}"`, { stdio: "pipe", timeout: 3e4 });
+    try {
+      execSync47(`"${venvPyPath}" -m pip --version`, { stdio: "pipe", timeout: 1e4 });
+    } catch {
+      try {
+        execSync47(`"${venvPyPath}" -m ensurepip --upgrade`, { stdio: "pipe", timeout: 3e4 });
+      } catch {
+        execSync47(`curl -sS https://bootstrap.pypa.io/get-pip.py | "${venvPyPath}"`, { stdio: "pipe", timeout: 6e4 });
+      }
+    }
+    execSync47(`"${venvPyPath}" -m pip install --upgrade pip`, {
       stdio: "pipe",
       timeout: 6e4
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.164",
+  "version": "0.187.166",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",