open-agents-ai 0.187.344 → 0.187.345

package/dist/index.js

@@ -259301,6 +259301,7 @@ print(classes[top])
           textParts.push(`Speech: "${episode.audio.transcript}"`);
         episode.text = { content: textParts.join(". ") || "Multi-modal capture", entities: [] };
         this.saveEpisode(episode);
+        this.bridgeToEpisodeStore(episode);
         return {
           success: true,
           output: `Multi-modal episode captured (${episodeId}):
@@ -259379,6 +259380,7 @@ else:
             }
           }
           this.saveEpisode(episode);
+          this.bridgeToEpisodeStore(episode);
         }
         return {
           success: true,
@@ -259550,6 +259552,88 @@ ${lines.join("\n")}`,
         });
         writeFileSync17(MM_INDEX, JSON.stringify(index, null, 2));
       }
+      /**
+       * WO-AM-GAP-02: Bridge multimodal episode to SQLite EpisodeStore + TemporalGraph.
+       * Makes multimodal episodes visible to PPR retrieval, zettelkasten linking,
+       * and the orchestrator's context injection system.
+       */
+      bridgeToEpisodeStore(episode) {
+        try {
+          const { EpisodeStore: EpisodeStore3, TemporalGraph: TemporalGraph3 } = __require("@open-agents/memory");
+          const { existsSync: fsExists } = __require("node:fs");
+          const { join: pathJoin } = __require("node:path");
+          const cwd4 = process.cwd();
+          const oaDirs = [pathJoin(cwd4, ".oa_test"), pathJoin(cwd4, ".oa")];
+          const oaDir = oaDirs.find((d2) => fsExists(d2));
+          if (!oaDir)
+            return;
+          const es = new EpisodeStore3(pathJoin(oaDir, "memory.db"));
+          const tg = new TemporalGraph3(pathJoin(oaDir, "kg.db"));
+          const contentParts = [];
+          if (episode.social?.personName)
+            contentParts.push(`Met ${episode.social.personName}`);
+          if (episode.audio?.transcript)
+            contentParts.push(`Said: "${episode.audio.transcript}"`);
+          if (episode.audio?.soundClass)
+            contentParts.push(`Sound: ${episode.audio.soundClass}`);
+          if (episode.visual?.faceNames?.length)
+            contentParts.push(`Faces: ${episode.visual.faceNames.join(", ")}`);
+          if (episode.visual?.objects?.length)
+            contentParts.push(`Objects: ${episode.visual.objects.join(", ")}`);
+          if (episode.spatial?.locationLabel)
+            contentParts.push(`Location: ${episode.spatial.locationLabel}`);
+          const content = contentParts.join(". ") || episode.text?.content || "Multimodal capture";
+          const modality = episode.social?.personName ? "social" : episode.visual?.imagePath ? "visual" : episode.audio?.recordingPath ? "audio" : "text";
+          const epId = es.insert({
+            sessionId: episode.sessionId,
+            modality,
+            toolName: "multimodal_memory",
+            content,
+            importance: episode.social?.personName ? 9 : 7,
+            decayClass: episode.social?.personName ? "permanent" : "daily",
+            metadata: {
+              multimodal_episode_id: episode.id,
+              has_face: (episode.visual?.faceIds?.length ?? 0) > 0,
+              has_audio: !!episode.audio?.recordingPath,
+              has_gps: !!episode.spatial?.gps
+            }
+          });
+          if (episode.visual?.clipEmbedding) {
+            const emb = new Float32Array(episode.visual.clipEmbedding);
+            es.setEmbedding(epId, emb);
+          }
+          if (episode.social?.personName) {
+            const personId = tg.upsertNode({ text: episode.social.personName, nodeType: "person" });
+            tg.addEdge({
+              srcId: personId,
+              dstId: personId,
+              relation: "discovered_during",
+              fact: `Met ${episode.social.personName} via multimodal capture`,
+              edgeType: "triple",
+              sourceEpisodeId: epId,
+              modality
+            });
+          }
+          if (episode.spatial?.locationLabel) {
+            const locId = tg.upsertNode({ text: episode.spatial.locationLabel, nodeType: "location" });
+            if (episode.social?.personName) {
+              const personNode = tg.findNode(episode.social.personName, "person");
+              if (personNode) {
+                tg.addEdge({
+                  srcId: personNode.id,
+                  dstId: locId,
+                  relation: "appears_in",
+                  fact: `${episode.social.personName} seen at ${episode.spatial.locationLabel}`,
+                  sourceEpisodeId: epId
+                });
+              }
+            }
+          }
+          es.close();
+          tg.close();
+        } catch {
+        }
+      }
       loadEpisode(id) {
         const file = join52(MM_DIR, id, "episode.json");
         try {
@@ -267715,12 +267799,14 @@ function autoImportance(toolName, modality, content) {
     return 5;
   if (toolName?.startsWith("memory_"))
     return 6;
-  if (modality === "visual" || modality === "audio")
-    return 7;
   if (modality === "social")
-    return 8;
-  if (modality === "spatial")
+    return 9;
+  if (modality === "visual")
+    return 7;
+  if (modality === "audio")
     return 6;
+  if (modality === "spatial")
+    return 5;
   if (modality === "reflection")
     return 7;
   if (modality === "gist")
@@ -267738,7 +267824,13 @@ function autoDecayClass(toolName, modality, content) {
     return "daily";
   if (content.toLowerCase().includes("error") || content.toLowerCase().includes("failed"))
     return "procedural";
-  if (["visual", "audio", "social", "spatial"].includes(modality))
+  if (modality === "social")
+    return "permanent";
+  if (modality === "visual")
+    return "procedural";
+  if (modality === "spatial")
+    return "procedural";
+  if (modality === "audio")
     return "daily";
   if (modality === "reflection" || modality === "gist")
     return "procedural";
@@ -267793,6 +267885,10 @@ var init_episodeStore = __esm({
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_tool ON episodes(tool_name)`);
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_decay ON episodes(decay_class)`);
         this.db.exec(`CREATE INDEX IF NOT EXISTS idx_ep_hash ON episodes(content_hash, session_id)`);
+        try {
+          this.db.exec(`ALTER TABLE episodes ADD COLUMN clip_embedding BLOB`);
+        } catch {
+        }
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("synchronous = NORMAL");
       }
@@ -267853,7 +267949,37 @@ var init_episodeStore = __esm({
         }
         sql += " ORDER BY timestamp DESC LIMIT ?";
         params.push(Math.min(limit * 5, 500));
-        const rows = this.db.prepare(sql).all(...params);
+        let rows = this.db.prepare(sql).all(...params);
+        if (query.metadataFilter || query.soundClass || query.rmsRange) {
+          rows = rows.filter((row) => {
+            let meta = null;
+            try {
+              meta = row.metadata ? typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata : null;
+            } catch {
+              return false;
+            }
+            if (!meta)
+              return false;
+            if (query.metadataFilter) {
+              for (const [k, v] of Object.entries(query.metadataFilter)) {
+                if (meta[k] !== v)
+                  return false;
+              }
+            }
+            if (query.soundClass && meta.sound_class !== query.soundClass)
+              return false;
+            if (query.rmsRange) {
+              const rms = meta.rms_db ?? meta.rmsDb;
+              if (typeof rms !== "number")
+                return false;
+              if (query.rmsRange.min !== void 0 && rms < query.rmsRange.min)
+                return false;
+              if (query.rmsRange.max !== void 0 && rms > query.rmsRange.max)
+                return false;
+            }
+            return true;
+          });
+        }
         const scored = rows.map((row) => {
           const ep = this.rowToEpisode(row);
           const tau = DECAY_TAU[ep.decayClass] ?? DECAY_TAU.daily;
@@ -267883,7 +268009,8 @@ var init_episodeStore = __esm({
             emb = Math.max(0, Math.min(1, (emb + 1) / 2));
           }
           const relevance = wLex * lex + wEmb * emb;
-          const score = recency + importance + relevance;
+          const strengthBonus = Math.min(1, Math.log2(Math.max(1, ep.strength)));
+          const score = recency + importance + relevance + strengthBonus;
           return { episode: ep, score };
         });
         scored.sort((a2, b) => b.score - a2.score);
@@ -267929,6 +268056,11 @@ var init_episodeStore = __esm({
         const buf = Buffer.from(embedding.buffer);
         this.db.prepare("UPDATE episodes SET embedding = ? WHERE id = ?").run(buf, id);
       }
+      /** WO-AM-GAP-03: Set CLIP embedding for cross-modal retrieval (512d). */
+      setClipEmbedding(id, clipEmbedding) {
+        const buf = Buffer.from(clipEmbedding.buffer);
+        this.db.prepare("UPDATE episodes SET clip_embedding = ? WHERE id = ?").run(buf, id);
+      }
       /** Update gist for an episode (from WO-AM-07 gist compression). */
       setGist(id, gist) {
         this.db.prepare("UPDATE episodes SET gist = ? WHERE id = ?").run(gist, id);
@@ -267961,6 +268093,7 @@ var init_episodeStore = __esm({
           decayClass: row.decay_class,
           strength: typeof row.strength === "number" && Number.isFinite(row.strength) ? row.strength : 1,
           lastRetrieved: row.last_retrieved,
+          clipEmbedding: row.clip_embedding ? new Float32Array(new Uint8Array(row.clip_embedding).buffer) : null,
           gist: row.gist,
           sourceEpisodeId: row.source_episode_id
         };
@@ -269706,6 +269839,19 @@ function repairJson(raw) {
   }
   return found ? result : null;
 }
+function inferEpisodeModality(toolName) {
+  if (VISUAL_TOOLS.has(toolName))
+    return "visual";
+  if (AUDIO_TOOLS.has(toolName))
+    return "audio";
+  if (SOCIAL_TOOLS.has(toolName))
+    return "social";
+  if (SPATIAL_TOOLS.has(toolName))
+    return "spatial";
+  if (CODE_TOOLS.has(toolName))
+    return "code";
+  return "tool_result";
+}
 function getSystemPromptForTier(tier) {
   switch (tier) {
     case "small":
@@ -269746,7 +269892,7 @@ ${todoItems}
 </system-reminder>`;
   return { shouldInject: true, content, reason: "injected" };
 }
-var SYSTEM_PROMPT, SYSTEM_PROMPT_MEDIUM, SYSTEM_PROMPT_SMALL, AgenticRunner, OllamaAgenticBackend;
+var SYSTEM_PROMPT, SYSTEM_PROMPT_MEDIUM, SYSTEM_PROMPT_SMALL, VISUAL_TOOLS, AUDIO_TOOLS, SOCIAL_TOOLS, SPATIAL_TOOLS, CODE_TOOLS, AgenticRunner, OllamaAgenticBackend;
 var init_agenticRunner = __esm({
   "packages/orchestrator/dist/agenticRunner.js"() {
     "use strict";
@@ -269764,6 +269910,11 @@ var init_agenticRunner = __esm({
     SYSTEM_PROMPT = loadPrompt("agentic/system-large.md");
     SYSTEM_PROMPT_MEDIUM = loadPrompt("agentic/system-medium.md");
     SYSTEM_PROMPT_SMALL = loadPrompt("agentic/system-small.md");
+    VISUAL_TOOLS = /* @__PURE__ */ new Set(["vision", "camera_capture", "image_read", "screenshot", "ocr", "ocr_image_advanced", "visual_memory", "desktop_describe", "ocr_pdf", "generate_image"]);
+    AUDIO_TOOLS = /* @__PURE__ */ new Set(["audio_capture", "audio_analyze", "asr_listen", "transcribe_file", "transcribe_url", "audio_playback", "youtube_download"]);
+    SOCIAL_TOOLS = /* @__PURE__ */ new Set(["multimodal_memory", "send_message", "jibberlink"]);
+    SPATIAL_TOOLS = /* @__PURE__ */ new Set(["gps_location", "bluetooth_scan", "wifi_control", "sdr_scan", "meshtastic"]);
+    CODE_TOOLS = /* @__PURE__ */ new Set(["file_write", "file_edit", "file_patch", "batch_edit", "code_sandbox", "repl_exec", "notebook_edit"]);
     AgenticRunner = class {
       backend;
       tools = /* @__PURE__ */ new Map();
@@ -271582,17 +271733,33 @@ ${cachedEntry2.result.slice(0, 500)}` : `[BLOCKED — the observer confirmed thi
               if (this._episodeStore) {
                 try {
                   const episodeContent = result.success ? `${tc.name}: ${(result.output ?? "").slice(0, 500)}` : `${tc.name} ERROR: ${(result.error ?? result.output ?? "").slice(0, 500)}`;
+                  const episodeModality = inferEpisodeModality(tc.name);
+                  let episodeMetadata = {
+                    args_fingerprint: argsKey.slice(0, 200),
+                    success: result.success,
+                    duration_ms: performance.now() - toolStart
+                  };
+                  if (result.success && ["audio_analyze", "audio_capture"].includes(tc.name)) {
+                    try {
+                      const parsed = JSON.parse(result.output ?? "{}");
+                      if (parsed.rms_db !== void 0)
+                        episodeMetadata.rms_db = parsed.rms_db;
+                      if (parsed.classifications)
+                        episodeMetadata.sound_class = parsed.classifications[0]?.class;
+                      if (parsed.peak_frequencies)
+                        episodeMetadata.peak_frequencies = parsed.peak_frequencies.slice(0, 3);
+                      if (parsed.band_energy_db)
+                        episodeMetadata.band_energy = parsed.band_energy_db;
+                    } catch {
+                    }
+                  }
                   const episodeId = this._episodeStore.insert({
                     sessionId: this._sessionId,
                     turnNumber: turn,
-                    modality: "tool_result",
+                    modality: episodeModality,
                     toolName: tc.name,
                     content: episodeContent,
-                    metadata: {
-                      args_fingerprint: argsKey.slice(0, 200),
-                      success: result.success,
-                      duration_ms: performance.now() - toolStart
-                    }
+                    metadata: episodeMetadata
                   });
                   if (this._embeddingAvailable && episodeContent.length > 20) {
                     this._pendingEmbeddings.push({ id: episodeId, content: episodeContent.slice(0, 500) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.344",
+  "version": "0.187.345",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) \u2014 interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",